opensmell 3.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- opensmell/__init__.py +239 -0
- opensmell/calibration.py +313 -0
- opensmell/constants/__init__.py +137 -0
- opensmell/constants/sensors.json +386 -0
- opensmell/csv.py +348 -0
- opensmell/electrochemical/__init__.py +21 -0
- opensmell/features.py +47 -0
- opensmell/hardware.py +148 -0
- opensmell/ingest.py +195 -0
- opensmell/io.py +112 -0
- opensmell/miris/__init__.py +22 -0
- opensmell/mox/__init__.py +8 -0
- opensmell/mox/features.py +681 -0
- opensmell/mox/normalize.py +86 -0
- opensmell/mox/preprocessing.py +81 -0
- opensmell/mox/quality.py +259 -0
- opensmell/mox/smellability/__init__.py +214 -0
- opensmell/mox/smellability/chain.py +647 -0
- opensmell/mox/smellability/composites.py +392 -0
- opensmell/mox/smellability/compounds.py +849 -0
- opensmell/mox/smellability/constants.py +113 -0
- opensmell/mox/smellability/enrichment.py +305 -0
- opensmell/mox/smellability/groups.py +358 -0
- opensmell/mox/smellability/inference.py +603 -0
- opensmell/mox/smellability/ontology.py +265 -0
- opensmell/mox/smellability/provisional.py +87 -0
- opensmell/mox/smellability/search.py +144 -0
- opensmell/mox/smellability/transport.py +78 -0
- opensmell/mox/smellability/types.py +427 -0
- opensmell/mox/smellability/user_dictionary.py +74 -0
- opensmell/normalize.py +36 -0
- opensmell/quality.py +48 -0
- opensmell/result.py +20 -0
- opensmell/types.py +330 -0
- opensmell-3.0.0.dist-info/METADATA +232 -0
- opensmell-3.0.0.dist-info/RECORD +39 -0
- opensmell-3.0.0.dist-info/WHEEL +5 -0
- opensmell-3.0.0.dist-info/licenses/LICENSE +21 -0
- opensmell-3.0.0.dist-info/top_level.txt +1 -0
opensmell/__init__.py
ADDED
|
@@ -0,0 +1,239 @@
|
|
|
1
|
+
"""opensmell — digital olfaction SDK.
|
|
2
|
+
|
|
3
|
+
v3 modular framework: sensor-agnostic interfaces at the top level
|
|
4
|
+
(`io`, `csv`, `normalize`, `quality`, `features`) with sensor-specific
|
|
5
|
+
implementations in `opensmell.mox` (and future `opensmell.miris`,
|
|
6
|
+
`opensmell.electrochemical`). The MOX thermodynamic feasibility chain lives at
|
|
7
|
+
`opensmell.mox.smellability`.
|
|
8
|
+
|
|
9
|
+
Legacy v2 CSV-based API (`process`, `train`, `predict`, `extract_features`,
|
|
10
|
+
`load_recording`, `SmellResult`) is preserved for backwards compatibility.
|
|
11
|
+
|
|
12
|
+
The feasibility chain is re-exported at the top level as `opensmell.smellability`
|
|
13
|
+
so `resolve_and_run`, `chemical_from_smiles`, and the verdicts are reachable
|
|
14
|
+
without importing the internals.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
import numpy as np
|
|
18
|
+
from sklearn.pipeline import Pipeline
|
|
19
|
+
from sklearn.preprocessing import StandardScaler
|
|
20
|
+
from sklearn.ensemble import RandomForestClassifier
|
|
21
|
+
|
|
22
|
+
from . import features as _features
|
|
23
|
+
from .calibration import (
|
|
24
|
+
CalibrationError,
|
|
25
|
+
build_calibration_payload,
|
|
26
|
+
calibrate_precise,
|
|
27
|
+
calibrate_quick,
|
|
28
|
+
concentration_series,
|
|
29
|
+
fit_power_law,
|
|
30
|
+
invert_concentration,
|
|
31
|
+
loocv_power_law,
|
|
32
|
+
normed_to_rr,
|
|
33
|
+
two_point_calibration,
|
|
34
|
+
)
|
|
35
|
+
from .hardware import (
|
|
36
|
+
HardwareInsufficiencyWarning,
|
|
37
|
+
check_rig_sufficiency,
|
|
38
|
+
effective_dims,
|
|
39
|
+
implied_channels,
|
|
40
|
+
min_effective_dimensions,
|
|
41
|
+
)
|
|
42
|
+
from .mox.preprocessing import load_csv, rs_r0_normalize, segment
|
|
43
|
+
from .result import SmellResult
|
|
44
|
+
|
|
45
|
+
# --- MOX thermodynamic feasibility chain (Smellability) ---
|
|
46
|
+
from .mox import smellability
|
|
47
|
+
|
|
48
|
+
# Register the re-export as a real dotted path so `import opensmell.smellability`
|
|
49
|
+
# (and `from opensmell.smellability import ...`) works, not just attribute access.
|
|
50
|
+
import sys as _sys
|
|
51
|
+
|
|
52
|
+
_sys.modules[__name__ + ".smellability"] = smellability
|
|
53
|
+
del _sys
|
|
54
|
+
|
|
55
|
+
# --- New v3 sensor-agnostic API ---
|
|
56
|
+
from .csv import guess_sensor_type, parse_csv
|
|
57
|
+
from .features import process_mox, run_processor
|
|
58
|
+
from .ingest import (
|
|
59
|
+
IngestedCollection,
|
|
60
|
+
IngestedSession,
|
|
61
|
+
build_osmell_file,
|
|
62
|
+
ingest_file,
|
|
63
|
+
ingest_folder,
|
|
64
|
+
)
|
|
65
|
+
from .io import (
|
|
66
|
+
build_osmell,
|
|
67
|
+
csv_from_file,
|
|
68
|
+
default_file_name,
|
|
69
|
+
parse_osmell,
|
|
70
|
+
parse_osmell_file,
|
|
71
|
+
write_osmell,
|
|
72
|
+
)
|
|
73
|
+
from .quality import compute_quality
|
|
74
|
+
from .types import (
|
|
75
|
+
OSMELL_FORMAT_VERSION,
|
|
76
|
+
CalibrationDescriptor,
|
|
77
|
+
ChannelDescriptor,
|
|
78
|
+
ChannelStats,
|
|
79
|
+
OsmellFile,
|
|
80
|
+
OsmellManifest,
|
|
81
|
+
ParsedSample,
|
|
82
|
+
QualityReport,
|
|
83
|
+
SensorDescriptor,
|
|
84
|
+
SessionDescriptor,
|
|
85
|
+
SessionEvent,
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _feature_vector(feature_dict: dict) -> tuple:
|
|
90
|
+
keys = sorted(feature_dict.keys())
|
|
91
|
+
values = [feature_dict[k] for k in keys]
|
|
92
|
+
return np.array(values, dtype=np.float32), keys
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def _features_from_normed(normed: np.ndarray, n_channels: int = None) -> tuple:
|
|
96
|
+
if n_channels is None:
|
|
97
|
+
n_channels = normed.shape[1] if normed.ndim == 2 else 1
|
|
98
|
+
segments = segment(normed)
|
|
99
|
+
all_features = []
|
|
100
|
+
for seg in segments:
|
|
101
|
+
feats = _features.extract_all_framework_features(seg)
|
|
102
|
+
vals, _ = _feature_vector(feats)
|
|
103
|
+
all_features.append(vals)
|
|
104
|
+
arr = np.array(all_features)
|
|
105
|
+
fnames = _features.feature_names(n_channels=n_channels)
|
|
106
|
+
return arr, fnames
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def load_recording(filepath: str) -> np.ndarray:
|
|
110
|
+
raw = load_csv(filepath)
|
|
111
|
+
return rs_r0_normalize(raw)
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def extract_features(filepath: str) -> tuple:
|
|
115
|
+
normed = load_recording(filepath)
|
|
116
|
+
return _features_from_normed(normed, n_channels=normed.shape[1])
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def feature_names(n_channels=None) -> list:
|
|
120
|
+
"""Names of the MOX framework features (in extraction order).
|
|
121
|
+
|
|
122
|
+
Length is a function of channel count (``28·c + c(c−1)/2 + 4``); pass
|
|
123
|
+
``n_channels`` to match a non-6 rig, or omit for the canonical 6.
|
|
124
|
+
"""
|
|
125
|
+
return _features.feature_names(n_channels=n_channels)
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def process(filepath: str, model: Pipeline = None) -> SmellResult:
|
|
129
|
+
normed = load_recording(filepath)
|
|
130
|
+
if model is not None:
|
|
131
|
+
check_rig_sufficiency(normed.shape[1], model)
|
|
132
|
+
features_arr, fnames = _features_from_normed(normed)
|
|
133
|
+
if features_arr.shape[0] == 0:
|
|
134
|
+
return SmellResult(features=np.array([]), feature_names=fnames, n_windows=0)
|
|
135
|
+
avg_features = features_arr.mean(axis=0)
|
|
136
|
+
if model is not None:
|
|
137
|
+
pred = model.predict([avg_features])[0]
|
|
138
|
+
proba = model.predict_proba([avg_features]).max()
|
|
139
|
+
warning = ""
|
|
140
|
+
if proba < 0.5:
|
|
141
|
+
warning = "Low confidence"
|
|
142
|
+
elif proba < 0.7:
|
|
143
|
+
warning = "Moderate confidence"
|
|
144
|
+
return SmellResult(
|
|
145
|
+
substance=str(pred),
|
|
146
|
+
confidence=float(proba),
|
|
147
|
+
warning=warning,
|
|
148
|
+
features=avg_features,
|
|
149
|
+
feature_names=fnames,
|
|
150
|
+
n_windows=features_arr.shape[0],
|
|
151
|
+
)
|
|
152
|
+
return SmellResult(
|
|
153
|
+
features=avg_features,
|
|
154
|
+
feature_names=fnames,
|
|
155
|
+
n_windows=features_arr.shape[0],
|
|
156
|
+
)
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def train(X: np.ndarray, y: np.ndarray, n_estimators: int = 200) -> Pipeline:
|
|
160
|
+
model = Pipeline([
|
|
161
|
+
("scaler", StandardScaler()),
|
|
162
|
+
("clf", RandomForestClassifier(
|
|
163
|
+
n_estimators=n_estimators,
|
|
164
|
+
class_weight="balanced",
|
|
165
|
+
random_state=42,
|
|
166
|
+
n_jobs=-1,
|
|
167
|
+
)),
|
|
168
|
+
])
|
|
169
|
+
model.fit(X, y)
|
|
170
|
+
n_ch = implied_channels(X.shape[1])
|
|
171
|
+
if n_ch is not None:
|
|
172
|
+
model.min_effective_dimensions = effective_dims(n_ch)
|
|
173
|
+
return model
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def predict(filepath: str, model: Pipeline) -> SmellResult:
|
|
177
|
+
return process(filepath, model=model)
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
# Backwards-compat alias for the .osmell loader.
|
|
181
|
+
load_osmell = parse_osmell_file
|
|
182
|
+
|
|
183
|
+
__all__ = [
|
|
184
|
+
"extract_features",
|
|
185
|
+
"feature_names",
|
|
186
|
+
"process",
|
|
187
|
+
"train",
|
|
188
|
+
"predict",
|
|
189
|
+
"load_recording",
|
|
190
|
+
"SmellResult",
|
|
191
|
+
# Reference-point calibration (§4.6, §10.10)
|
|
192
|
+
"CalibrationError",
|
|
193
|
+
"two_point_calibration",
|
|
194
|
+
"fit_power_law",
|
|
195
|
+
"invert_concentration",
|
|
196
|
+
"loocv_power_law",
|
|
197
|
+
"build_calibration_payload",
|
|
198
|
+
"concentration_series",
|
|
199
|
+
"normed_to_rr",
|
|
200
|
+
"calibrate_quick",
|
|
201
|
+
"calibrate_precise",
|
|
202
|
+
# Hardware sufficiency gate (§10.10 N→M limit)
|
|
203
|
+
"HardwareInsufficiencyWarning",
|
|
204
|
+
"check_rig_sufficiency",
|
|
205
|
+
"effective_dims",
|
|
206
|
+
"min_effective_dimensions",
|
|
207
|
+
"implied_channels",
|
|
208
|
+
# v3 sensor-agnostic API
|
|
209
|
+
"parse_csv",
|
|
210
|
+
"guess_sensor_type",
|
|
211
|
+
"ingest_file",
|
|
212
|
+
"ingest_folder",
|
|
213
|
+
"build_osmell_file",
|
|
214
|
+
"IngestedSession",
|
|
215
|
+
"IngestedCollection",
|
|
216
|
+
"parse_osmell",
|
|
217
|
+
"parse_osmell_file",
|
|
218
|
+
"load_osmell",
|
|
219
|
+
"build_osmell",
|
|
220
|
+
"write_osmell",
|
|
221
|
+
"csv_from_file",
|
|
222
|
+
"default_file_name",
|
|
223
|
+
"compute_quality",
|
|
224
|
+
"run_processor",
|
|
225
|
+
"process_mox",
|
|
226
|
+
# MOX thermodynamic feasibility chain (Smellability)
|
|
227
|
+
"smellability",
|
|
228
|
+
"OSMELL_FORMAT_VERSION",
|
|
229
|
+
"OsmellFile",
|
|
230
|
+
"OsmellManifest",
|
|
231
|
+
"SensorDescriptor",
|
|
232
|
+
"SessionDescriptor",
|
|
233
|
+
"ChannelDescriptor",
|
|
234
|
+
"SessionEvent",
|
|
235
|
+
"ParsedSample",
|
|
236
|
+
"ChannelStats",
|
|
237
|
+
"QualityReport",
|
|
238
|
+
"CalibrationDescriptor",
|
|
239
|
+
]
|
opensmell/calibration.py
ADDED
|
@@ -0,0 +1,313 @@
|
|
|
1
|
+
"""Reference-point calibration (§4.6, §10.10).
|
|
2
|
+
|
|
3
|
+
The only sanctioned path to quantification: measured resistance ratio
|
|
4
|
+
``rr = R/R0`` at KNOWN concentrations ``C`` on a specific rig, per channel.
|
|
5
|
+
The sensor power law is
|
|
6
|
+
|
|
7
|
+
rr = a · C^b (b < 0 for the classic reducing-gas response)
|
|
8
|
+
|
|
9
|
+
so a channel's ``(a, b)`` are fitted in log-log space and concentration is
|
|
10
|
+
inverted as
|
|
11
|
+
|
|
12
|
+
C = (rr / a) ^ (1 / b)
|
|
13
|
+
|
|
14
|
+
These are per-rig, per-channel, per-substance quantities. They do not transfer
|
|
15
|
+
across rigs (§4.6 proof), they are valid only for the reference substance they
|
|
16
|
+
were measured with, and they must be re-measured as the rig drifts. The
|
|
17
|
+
``sensor.calibration`` manifest contract (§10.10) stores them.
|
|
18
|
+
|
|
19
|
+
Design principles:
|
|
20
|
+
|
|
21
|
+
- *Warn, never silently interpolate.* Every fit reports R², residual spread,
|
|
22
|
+
coverage (ppm span) and a leave-one-out concentration error.
|
|
23
|
+
- *Falsifiable.* ``loocv_power_law`` gives the held-out % error you should
|
|
24
|
+
expect at the fitted concentrations; extrapolation outside the calibrated
|
|
25
|
+
ppm range is explicitly penalized (see ``research/calibration-experiments/
|
|
26
|
+
reference-point-calibration/`` for the quantified numbers).
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
from __future__ import annotations
|
|
30
|
+
|
|
31
|
+
import math
|
|
32
|
+
from datetime import date
|
|
33
|
+
from typing import Dict, Optional
|
|
34
|
+
|
|
35
|
+
import numpy as np
|
|
36
|
+
|
|
37
|
+
from .constants import UnknownSensorError, power_law
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class CalibrationError(ValueError):
|
|
41
|
+
"""A calibration fit is impossible with the given data (e.g. < 2 points,
|
|
42
|
+
non-positive values, or a degenerate reference pair)."""
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def normed_to_rr(normed) -> np.ndarray:
|
|
46
|
+
"""Convert normalized response ``(R - R0)/R0`` to the ratio ``R/R0``.
|
|
47
|
+
|
|
48
|
+
The calibration power law uses ``rr = R/R0``; ``load_recording`` returns the
|
|
49
|
+
normalized form, so ``rr = 1 + normed``.
|
|
50
|
+
"""
|
|
51
|
+
arr = np.asarray(normed, dtype=np.float64)
|
|
52
|
+
return 1.0 + arr
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def invert_concentration(rr, a: float, b: float) -> np.ndarray:
|
|
56
|
+
"""Invert ``rr = a·C^b`` to concentration ``C = (rr/a)^(1/b)``.
|
|
57
|
+
|
|
58
|
+
Returns NaN where ``rr``, ``a``, or ``b`` make the inversion undefined
|
|
59
|
+
(``rr <= 0``, ``a <= 0``, ``b == 0``).
|
|
60
|
+
"""
|
|
61
|
+
arr = np.asarray(rr, dtype=np.float64)
|
|
62
|
+
scalar = arr.ndim == 0
|
|
63
|
+
arr = np.atleast_1d(arr)
|
|
64
|
+
a = float(a)
|
|
65
|
+
b = float(b)
|
|
66
|
+
if a <= 0 or b == 0:
|
|
67
|
+
out = np.full(arr.shape, np.nan)
|
|
68
|
+
else:
|
|
69
|
+
with np.errstate(invalid="ignore", divide="ignore", over="ignore"):
|
|
70
|
+
out = (arr / a) ** (1.0 / b)
|
|
71
|
+
out = np.where(arr > 0, out, np.nan)
|
|
72
|
+
return float(out[0]) if scalar else out
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def two_point_calibration(rr1: float, c1: float, rr2: float, c2: float):
|
|
76
|
+
"""Exact (a, b) from two measured (rr, C) points (§4.6).
|
|
77
|
+
|
|
78
|
+
``b = log(rr1/rr2) / log(C1/C2)``, ``a = rr1 / C1^b``. Raises
|
|
79
|
+
``CalibrationError`` on degenerate inputs (equal concentrations, equal
|
|
80
|
+
responses, non-positive values).
|
|
81
|
+
"""
|
|
82
|
+
if not (rr1 > 0 and rr2 > 0 and c1 > 0 and c2 > 0):
|
|
83
|
+
raise CalibrationError(
|
|
84
|
+
f"Reference points must be positive: (rr1, c1, rr2, c2) = "
|
|
85
|
+
f"({rr1}, {c1}, {rr2}, {c2}).")
|
|
86
|
+
if c1 == c2:
|
|
87
|
+
raise CalibrationError("Two-point calibration needs distinct concentrations.")
|
|
88
|
+
if rr1 == rr2:
|
|
89
|
+
raise CalibrationError("Two-point calibration needs distinct responses.")
|
|
90
|
+
b = math.log(rr1 / rr2) / math.log(c1 / c2)
|
|
91
|
+
a = rr1 / (c1 ** b)
|
|
92
|
+
return float(a), float(b)
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def _valid_mask(rr, c) -> np.ndarray:
|
|
96
|
+
rr = np.asarray(rr, dtype=np.float64)
|
|
97
|
+
c = np.asarray(c, dtype=np.float64)
|
|
98
|
+
return np.isfinite(rr) & np.isfinite(c) & (rr > 0) & (c > 0)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def fit_power_law(rr, c) -> dict:
|
|
102
|
+
"""Multi-point power-law fit in log-log space (recommended over two-point).
|
|
103
|
+
|
|
104
|
+
OLS of ``log(rr) = log(a) + b·log(C)``. Returns a dict with ``a``, ``b``,
|
|
105
|
+
``r2``, ``rmse_log_rr`` (residual spread in log-response, a noise proxy),
|
|
106
|
+
``n_points``, ppm coverage, and ``residuals`` (log domain).
|
|
107
|
+
|
|
108
|
+
Requires at least 2 valid (positive, finite) points; raises
|
|
109
|
+
``CalibrationError`` otherwise.
|
|
110
|
+
"""
|
|
111
|
+
rr = np.asarray(rr, dtype=np.float64)
|
|
112
|
+
c = np.asarray(c, dtype=np.float64)
|
|
113
|
+
if rr.ndim == 0:
|
|
114
|
+
rr = rr.reshape(1)
|
|
115
|
+
c = np.asarray(c, dtype=np.float64).reshape(1)
|
|
116
|
+
mask = _valid_mask(rr, c)
|
|
117
|
+
if mask.sum() < 2:
|
|
118
|
+
raise CalibrationError(
|
|
119
|
+
f"Need >= 2 valid (positive) reference points, got {int(mask.sum())}.")
|
|
120
|
+
log_rr = np.log(rr[mask])
|
|
121
|
+
log_c = np.log(c[mask])
|
|
122
|
+
b, loga = np.polyfit(log_c, log_rr, 1)
|
|
123
|
+
a = float(math.exp(loga))
|
|
124
|
+
resid = log_rr - (b * log_c + loga)
|
|
125
|
+
ss_res = float(np.sum(resid ** 2))
|
|
126
|
+
ss_tot = float(np.sum((log_rr - log_rr.mean()) ** 2))
|
|
127
|
+
r2 = 1.0 - ss_res / ss_tot if ss_tot > 0 else 0.0
|
|
128
|
+
rmse = float(np.sqrt(np.mean(resid ** 2)))
|
|
129
|
+
return {
|
|
130
|
+
"a": float(a),
|
|
131
|
+
"b": float(b),
|
|
132
|
+
"r2": r2,
|
|
133
|
+
"rmse_log_rr": rmse,
|
|
134
|
+
"n_points": int(mask.sum()),
|
|
135
|
+
"min_ppm": float(np.min(c[mask])),
|
|
136
|
+
"max_ppm": float(np.max(c[mask])),
|
|
137
|
+
"decades": float(np.log10(np.max(c[mask]) / np.min(c[mask]))),
|
|
138
|
+
"residuals": resid.tolist(),
|
|
139
|
+
"method": "multi-point-loglog",
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def loocv_power_law(rr, c) -> Optional[dict]:
|
|
144
|
+
"""Leave-one-concentration-out falsification of the power-law fit.
|
|
145
|
+
|
|
146
|
+
For each reference point, fit ``(a, b)`` on the other points and predict
|
|
147
|
+
the held-out concentration; report relative error. Returns ``None`` when
|
|
148
|
+
fewer than 3 valid points exist (a 2-point fit has no independent holdout).
|
|
149
|
+
"""
|
|
150
|
+
rr = np.asarray(rr, dtype=np.float64)
|
|
151
|
+
c = np.asarray(c, dtype=np.float64)
|
|
152
|
+
mask = _valid_mask(rr, c)
|
|
153
|
+
rr_v, c_v = rr[mask], c[mask]
|
|
154
|
+
if len(rr_v) < 3:
|
|
155
|
+
return None
|
|
156
|
+
|
|
157
|
+
folds = []
|
|
158
|
+
for i in range(len(rr_v)):
|
|
159
|
+
tr_rr = np.concatenate([rr_v[:i], rr_v[i + 1:]])
|
|
160
|
+
tr_c = np.concatenate([c_v[:i], c_v[i + 1:]])
|
|
161
|
+
fit = fit_power_law(tr_rr, tr_c)
|
|
162
|
+
pred = invert_concentration(rr_v[i], fit["a"], fit["b"])
|
|
163
|
+
err = (float(pred) - float(c_v[i])) / float(c_v[i])
|
|
164
|
+
folds.append({
|
|
165
|
+
"heldout_ppm": float(c_v[i]),
|
|
166
|
+
"pred_ppm": float(pred),
|
|
167
|
+
"rel_error": float(err),
|
|
168
|
+
"abs_pct_error": abs(err) * 100.0,
|
|
169
|
+
})
|
|
170
|
+
abs_pct = np.array([f["abs_pct_error"] for f in folds])
|
|
171
|
+
rel = np.array([f["rel_error"] for f in folds])
|
|
172
|
+
return {
|
|
173
|
+
"n_folds": len(folds),
|
|
174
|
+
"mean_abs_pct_error": float(np.mean(abs_pct)),
|
|
175
|
+
"median_abs_pct_error": float(np.median(abs_pct)),
|
|
176
|
+
"max_abs_pct_error": float(np.max(abs_pct)),
|
|
177
|
+
"bias_pct": float(np.mean(rel) * 100.0),
|
|
178
|
+
"folds": folds,
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def build_calibration_payload(
|
|
183
|
+
fits: Dict[str, dict],
|
|
184
|
+
reference_substance: str,
|
|
185
|
+
calibration_date: Optional[str] = None,
|
|
186
|
+
method: str = "multi-point-loglog",
|
|
187
|
+
) -> Dict[str, dict]:
|
|
188
|
+
"""Build a ``sensor.calibration`` manifest payload (§10.10 contract).
|
|
189
|
+
|
|
190
|
+
``fits`` maps channel id -> ``fit_power_law`` result. The returned dict is
|
|
191
|
+
directly consumable by ``CalibrationDescriptor.from_dict`` and round-trips
|
|
192
|
+
through ``.osmell``. ``reference_ppm`` is the geometric-mean concentration
|
|
193
|
+
of the calibration range (the contract stores a single scalar).
|
|
194
|
+
"""
|
|
195
|
+
if calibration_date is None:
|
|
196
|
+
calibration_date = date.today().isoformat()
|
|
197
|
+
payload: Dict[str, dict] = {}
|
|
198
|
+
for ch, fit in fits.items():
|
|
199
|
+
a = fit.get("a")
|
|
200
|
+
b = fit.get("b")
|
|
201
|
+
if a is None or b is None:
|
|
202
|
+
continue
|
|
203
|
+
ref_ppm = math.sqrt(max(fit.get("min_ppm", 1.0), 1.0)
|
|
204
|
+
* max(fit.get("max_ppm", 1.0), 1.0))
|
|
205
|
+
payload[ch] = {
|
|
206
|
+
"a": float(a),
|
|
207
|
+
"b": float(b),
|
|
208
|
+
"referenceSubstance": reference_substance,
|
|
209
|
+
"referencePpm": float(ref_ppm),
|
|
210
|
+
"date": calibration_date,
|
|
211
|
+
"method": method,
|
|
212
|
+
}
|
|
213
|
+
return payload
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def concentration_series(normed, params: Dict[int, tuple]) -> np.ndarray:
|
|
217
|
+
"""Per-channel concentration time series from a normalized recording.
|
|
218
|
+
|
|
219
|
+
``normed`` is ``(R - R0)/R0`` per channel (``load_recording`` output);
|
|
220
|
+
``params`` maps channel index -> ``(a, b)``. Uncalibrated channels yield
|
|
221
|
+
NaN. Use with the ``sensor.calibration`` manifest contract via
|
|
222
|
+
``extract_all_framework_features(calibration=...)`` for the scalar feature
|
|
223
|
+
path.
|
|
224
|
+
"""
|
|
225
|
+
normed = np.asarray(normed, dtype=np.float64)
|
|
226
|
+
if normed.ndim == 1:
|
|
227
|
+
normed = normed.reshape(-1, 1)
|
|
228
|
+
rr = normed_to_rr(normed)
|
|
229
|
+
out = np.full_like(rr, np.nan)
|
|
230
|
+
for ch, (a, b) in params.items():
|
|
231
|
+
if 0 <= ch < rr.shape[1]:
|
|
232
|
+
out[:, ch] = invert_concentration(rr[:, ch], a, b)
|
|
233
|
+
return out
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
def calibrate_quick(sensor: str, gas: str, channel: str = "ch0",
|
|
237
|
+
reference_ppm: Optional[float] = None,
|
|
238
|
+
override: Optional[Dict[str, float]] = None) -> dict:
|
|
239
|
+
"""Datasheet-derived single-channel calibration (no measured points).
|
|
240
|
+
|
|
241
|
+
Looks up the converted ``(a, b)`` power-law constants for the given
|
|
242
|
+
``sensor`` model responding to ``gas`` from the embedded ``sensors.json``
|
|
243
|
+
(§10.10). It is a *starting point* for quantification, not a substitute for
|
|
244
|
+
measured reference points: the constants are single-reference-substance
|
|
245
|
+
datasheet estimates that do not transfer across rigs (§4.6).
|
|
246
|
+
|
|
247
|
+
``override`` may supply ``{"a": ..., "b": ...}`` to adjust the tabulated
|
|
248
|
+
values for a specific rig. Returns a one-channel ``sensor.calibration``
|
|
249
|
+
payload conforming to ``CalibrationDescriptor.from_dict``.
|
|
250
|
+
"""
|
|
251
|
+
if override is None:
|
|
252
|
+
override = {}
|
|
253
|
+
try:
|
|
254
|
+
params = dict(power_law(sensor, gas))
|
|
255
|
+
except UnknownSensorError:
|
|
256
|
+
raise
|
|
257
|
+
params.update(override)
|
|
258
|
+
b = float(params["b"])
|
|
259
|
+
a = float(params["a"])
|
|
260
|
+
if a <= 0 or b == 0:
|
|
261
|
+
raise CalibrationError(
|
|
262
|
+
f"`calibrate_quick` requires a > 0 and b != 0 for a usable power law; "
|
|
263
|
+
f"got (a={a}, b={b}) sensor={sensor} gas={gas}.")
|
|
264
|
+
payload = {
|
|
265
|
+
"a": a,
|
|
266
|
+
"b": b,
|
|
267
|
+
"referenceSubstance": gas,
|
|
268
|
+
"date": date.today().isoformat(),
|
|
269
|
+
"method": "datasheet",
|
|
270
|
+
}
|
|
271
|
+
if reference_ppm is not None:
|
|
272
|
+
payload["referencePpm"] = float(reference_ppm)
|
|
273
|
+
return {channel: payload}
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def calibrate_precise(sensor: str, gas: str, rr, c,
|
|
277
|
+
channel: str = "ch0",
|
|
278
|
+
reference_ppm: Optional[float] = None) -> dict:
|
|
279
|
+
"""Measured multi-point power-law calibration wrapper (§4.6).
|
|
280
|
+
|
|
281
|
+
Fits ``(a, b)`` in log-log space from measured ``(rr, C)`` reference points
|
|
282
|
+
(reusing ``fit_power_law``), reports LOOCV falsification (``loocv_power_law``),
|
|
283
|
+
and returns both a ``sensor.calibration`` payload (for ``CalibrationDescriptor``)
|
|
284
|
+
and the full fit + diagnostics under ``"diagnostics"``.
|
|
285
|
+
|
|
286
|
+
``sensor``/``gas`` are descriptive metadata; the ``(a, b)`` come entirely
|
|
287
|
+
from the measured points, never from the datasheet table.
|
|
288
|
+
"""
|
|
289
|
+
fit = fit_power_law(rr, c)
|
|
290
|
+
loocv = loocv_power_law(rr, c)
|
|
291
|
+
if reference_ppm is None:
|
|
292
|
+
reference_ppm = math.sqrt(max(fit.get("min_ppm", 1.0), 1.0)
|
|
293
|
+
* max(fit.get("max_ppm", 1.0), 1.0))
|
|
294
|
+
payload = {
|
|
295
|
+
channel: {
|
|
296
|
+
"a": fit["a"],
|
|
297
|
+
"b": fit["b"],
|
|
298
|
+
"referenceSubstance": gas,
|
|
299
|
+
"referencePpm": float(reference_ppm),
|
|
300
|
+
"date": date.today().isoformat(),
|
|
301
|
+
"method": "multi-point-loglog",
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
return {
|
|
305
|
+
"channel": channel,
|
|
306
|
+
"sensor": sensor,
|
|
307
|
+
"gas": gas,
|
|
308
|
+
"calibration": payload,
|
|
309
|
+
"diagnostics": {key: fit[key] for key in (
|
|
310
|
+
"a", "b", "r2", "rmse_log_rr", "n_points", "min_ppm",
|
|
311
|
+
"max_ppm", "decades", "method")},
|
|
312
|
+
"loocv": loocv,
|
|
313
|
+
}
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
"""Offline sensor constants for MOX power-law response (§4.6, §10.10).
|
|
2
|
+
|
|
3
|
+
Power-law channels. Every ``(a, b)`` pair satisfies the SDK power law
|
|
4
|
+
|
|
5
|
+
rr = a · C^b (C in ppm, rr = R/R0)
|
|
6
|
+
|
|
7
|
+
with concentration inverted as ``C = (rr / a) ^ (1 / b)``. Values are the
|
|
8
|
+
**converted** datasheet-derived constants (source ``MQSensorsLib``, referenced
|
|
9
|
+
by each sensor's ``sources``), so they are directly consumable by the
|
|
10
|
+
reference-point calibration machinery. They are single-reference-substance
|
|
11
|
+
estimates; real quantification still needs per-rig, per-channel measured
|
|
12
|
+
reference points (§4.6).
|
|
13
|
+
|
|
14
|
+
Relative-response channels. Some MEMS entries (``SGP30``, ``SGP40``,
|
|
15
|
+
``TGS8100``, ``BME680``) carry no ``gases`` table and
|
|
16
|
+
``power_law_calibratable = False``: there is no authoritative, publicly
|
|
17
|
+
available power-law mapping to a target gas concentration for them, so the
|
|
18
|
+
SDK deliberately refuses to fabricate one. They are usable as relative
|
|
19
|
+
response/VOC-index channels, but ``power_law``/``clean_air_ratio``/etc. raise
|
|
20
|
+
``UnknownSensorError`` for them. Use ``is_power_law_sensor`` to distinguish.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
import json
|
|
26
|
+
from pathlib import Path
|
|
27
|
+
from typing import Dict, List, Optional
|
|
28
|
+
|
|
29
|
+
_SENSORS_PATH = Path(__file__).with_name("sensors.json")
|
|
30
|
+
|
|
31
|
+
_load = None
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _load_sensors() -> dict:
|
|
35
|
+
global _load
|
|
36
|
+
if _load is None:
|
|
37
|
+
with _SENSORS_PATH.open("r", encoding="utf-8") as f:
|
|
38
|
+
_load = json.load(f)
|
|
39
|
+
return _load
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class UnknownSensorError(KeyError):
|
|
43
|
+
"""The requested sensor model is not in the offline constants table."""
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def sensor_models() -> List[str]:
|
|
47
|
+
"""Sorted list of sensor models present in the constants table."""
|
|
48
|
+
return sorted(_load_sensors()["sensors"].keys())
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _gases_for(sensor: str) -> Dict[str, Dict[str, float]]:
|
|
52
|
+
"""Gas-to-constants map for ``sensor`` ([] if none tabulated).
|
|
53
|
+
|
|
54
|
+
A ``KeyError`` here means either the sensor is unknown, or it is a known
|
|
55
|
+
non-power-law channel (e.g. relative-response MEMS like ``TGS8100`` /
|
|
56
|
+
``SGP30`` / ``SGP40`` / ``BME680``) that has no ``gases`` table. Both cases
|
|
57
|
+
surface as ``UnknownSensorError`` so callers never see a raw ``KeyError``.
|
|
58
|
+
"""
|
|
59
|
+
try:
|
|
60
|
+
gases = _load_sensors()["sensors"][sensor]["gases"]
|
|
61
|
+
except KeyError:
|
|
62
|
+
raise UnknownSensorError(sensor) from None
|
|
63
|
+
return gases
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def sensor_gases(sensor: str) -> List[str]:
|
|
67
|
+
"""Gases (substances) a sensor model has response constants for."""
|
|
68
|
+
return list(_gases_for(sensor).keys())
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def clean_air_ratio(sensor: str) -> float:
|
|
72
|
+
"""Clean-air resistance ratio ``Rs/R0`` reported for the sensor model."""
|
|
73
|
+
try:
|
|
74
|
+
return float(_load_sensors()["sensors"][sensor]["clean_air_ratio"])
|
|
75
|
+
except KeyError:
|
|
76
|
+
raise UnknownSensorError(sensor) from None
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def power_law(sensor: str, gas: str) -> Dict[str, float]:
|
|
80
|
+
"""Return ``{"a": a, "b": b}`` for ``sensor`` responding to ``gas``.
|
|
81
|
+
|
|
82
|
+
Raises ``UnknownSensorError`` for an unknown sensor (or a known
|
|
83
|
+
non-power-law channel with no ``gases`` table) and ``KeyError`` for a gas
|
|
84
|
+
the sensor has no tabulated response for.
|
|
85
|
+
"""
|
|
86
|
+
gases = _gases_for(sensor)
|
|
87
|
+
try:
|
|
88
|
+
return dict(gases[gas])
|
|
89
|
+
except KeyError:
|
|
90
|
+
raise KeyError(
|
|
91
|
+
f"'{sensor}' has no tabulated response for gas '{gas}'. "
|
|
92
|
+
f"Known gases: {sorted(gases)}") from None
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def all_power_laws(sensor: str) -> Dict[str, Dict[str, float]]:
|
|
96
|
+
"""All ``{gas: {"a": a, "b": b}}`` responses for a sensor model."""
|
|
97
|
+
return {g: dict(v) for g, v in _gases_for(sensor).items()}
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def is_power_law_sensor(sensor: str) -> bool:
|
|
101
|
+
"""Whether ``sensor`` has a tabulated power-law ``gases`` table.
|
|
102
|
+
|
|
103
|
+
Power-law channels (e.g. the MQ family) return True and can feed
|
|
104
|
+
``power_law``/``clean_air_ratio``. Relative-response MEMS entries
|
|
105
|
+
(``TGS8100``, ``SGP30``, ``SGP40``, ``BME680``) and unknown sensors
|
|
106
|
+
return False -- they cannot be quantified from offline constants.
|
|
107
|
+
"""
|
|
108
|
+
try:
|
|
109
|
+
return "gases" in _load_sensors()["sensors"][sensor]
|
|
110
|
+
except KeyError:
|
|
111
|
+
return False
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def sensor_sources(sensor: str) -> List[str]:
|
|
115
|
+
"""Verification/reference URLs for ``sensor`` (datasheets, source code).
|
|
116
|
+
|
|
117
|
+
These are the provenance links backing the tabulated constants (or, for
|
|
118
|
+
non-power-law MEMS entries, the official documentation confirming no
|
|
119
|
+
authoritative power-law exists). Raises ``UnknownSensorError`` if the
|
|
120
|
+
sensor is not in the table.
|
|
121
|
+
"""
|
|
122
|
+
try:
|
|
123
|
+
return list(_load_sensors()["sensors"][sensor].get("sources", []))
|
|
124
|
+
except KeyError:
|
|
125
|
+
raise UnknownSensorError(sensor) from None
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
__all__ = [
|
|
129
|
+
"UnknownSensorError",
|
|
130
|
+
"sensor_models",
|
|
131
|
+
"sensor_gases",
|
|
132
|
+
"clean_air_ratio",
|
|
133
|
+
"power_law",
|
|
134
|
+
"all_power_laws",
|
|
135
|
+
"is_power_law_sensor",
|
|
136
|
+
"sensor_sources",
|
|
137
|
+
]
|