single-sensor-outage-imputer 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- single_sensor_outage_imputer-0.1.0.dist-info/METADATA +161 -0
- single_sensor_outage_imputer-0.1.0.dist-info/RECORD +10 -0
- single_sensor_outage_imputer-0.1.0.dist-info/WHEEL +5 -0
- single_sensor_outage_imputer-0.1.0.dist-info/top_level.txt +1 -0
- ssoi/__init__.py +24 -0
- ssoi/bundle.py +235 -0
- ssoi/model.py +114 -0
- ssoi/predict.py +145 -0
- ssoi/train.py +744 -0
- ssoi/train_jobs.py +351 -0
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: single-sensor-outage-imputer
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: TCDR runtime for single-sensor outage target imputation
|
|
5
|
+
Project-URL: Repository, https://github.com/romulobrito/single-sensor-outage-imputer
|
|
6
|
+
Requires-Python: >=3.9
|
|
7
|
+
Description-Content-Type: text/markdown
|
|
8
|
+
Requires-Dist: numpy>=1.21.0
|
|
9
|
+
Requires-Dist: pandas>=1.3.0
|
|
10
|
+
Requires-Dist: scikit-learn>=1.0.0
|
|
11
|
+
Requires-Dist: torch>=1.9.0
|
|
12
|
+
Requires-Dist: joblib>=1.0.0
|
|
13
|
+
Requires-Dist: PyYAML>=6.0
|
|
14
|
+
Provides-Extra: hdf5
|
|
15
|
+
Requires-Dist: tables>=3.8; extra == "hdf5"
|
|
16
|
+
Provides-Extra: dev
|
|
17
|
+
Requires-Dist: pytest>=7.0; extra == "dev"
|
|
18
|
+
|
|
19
|
+
# Single-sensor outage imputer
|
|
20
|
+
|
|
21
|
+
Runtime package for **single-sensor outage recovery**: reconstruct one target tag
|
|
22
|
+
when it is unavailable while auxiliary sensors remain online.
|
|
23
|
+
|
|
24
|
+
The model is a **target-conditional denoising regressor (TCDR)**:
|
|
25
|
+
|
|
26
|
+
```text
|
|
27
|
+
input = [x ; t_tilde ; m_t]
|
|
28
|
+
output = t_hat (scalar)
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
At deployment, outage mode is forced as:
|
|
32
|
+
|
|
33
|
+
```text
|
|
34
|
+
[t_tilde, m_t] = [0, 1] (0 in standardized target space = train mean)
|
|
35
|
+
t_hat = f_theta(x, 0, 1)
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
This repository is the **offline training and inference** surface (`VirtualSensor`,
|
|
39
|
+
bundle I/O). The **host** for outage recovery is BibMon: it consumes SSOI as an
|
|
40
|
+
optional extra. Canonical integration spec:
|
|
41
|
+
|
|
42
|
+
- `../BibMon/doc/SSOI_BIBMON.md` (workspace) or BibMon `doc/SSOI_BIBMON.md`
|
|
43
|
+
- Roadmap in this repo: `doc/PLANO_INTEGRACAO_BIBMON.md`
|
|
44
|
+
|
|
45
|
+
The research manuscript, controlled MCAR/MAR/MNAR validation, and baselines live
|
|
46
|
+
in a separate lab repo.
|
|
47
|
+
|
|
48
|
+
## Install
|
|
49
|
+
|
|
50
|
+
```bash
|
|
51
|
+
git clone https://github.com/romulobrito/single-sensor-outage-imputer.git
|
|
52
|
+
cd single-sensor-outage-imputer
|
|
53
|
+
python -m venv .venv
|
|
54
|
+
source .venv/bin/activate
|
|
55
|
+
pip install -r requirements.txt
|
|
56
|
+
pip install -e ".[dev]"
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
HDF5 training needs PyTables:
|
|
60
|
+
|
|
61
|
+
```bash
|
|
62
|
+
pip install -e ".[hdf5,dev]"
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
## Quick start (synthetic smoke)
|
|
66
|
+
|
|
67
|
+
```bash
|
|
68
|
+
python examples/smoke_synthetic.py
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
This fits scalers on synthetic data, trains a tiny TCDR, writes a bundle under
|
|
72
|
+
`examples/synthetic_bundle/`, reloads it, and runs `predict`.
|
|
73
|
+
|
|
74
|
+
## Train from scratch on local industrial data
|
|
75
|
+
|
|
76
|
+
Keep the historian file **outside git** (or under ignored `data/`). Example:
|
|
77
|
+
|
|
78
|
+
```bash
|
|
79
|
+
python examples/train_from_h5.py \
|
|
80
|
+
--data /path/to/sulfatos_dados_concatenados_formatados_2021.h5 \
|
|
81
|
+
--target 1251_FIT_801C_2 \
|
|
82
|
+
--bundle-dir artifacts/bundle_target_a \
|
|
83
|
+
--hdf-key optional_table_name
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
Omit `--cpu` to use CUDA when available. `--hdf-key` is only required when the
|
|
87
|
+
HDF5 file contains more than one table.
|
|
88
|
+
|
|
89
|
+
To keep an auxiliary out of automatic screening even if correlation is high:
|
|
90
|
+
|
|
91
|
+
```bash
|
|
92
|
+
python examples/train_from_h5.py \
|
|
93
|
+
--data /path/to/data.h5 \
|
|
94
|
+
--target 1251_FIT_801C_2 \
|
|
95
|
+
--bundle-dir artifacts/bundle_target_a \
|
|
96
|
+
--exclude-features TAG_INSTAVEL \
|
|
97
|
+
--exclude-features-file plant_exclude.json
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
`--exclude-features` and the JSON file are merged. The JSON may be a list or
|
|
101
|
+
`{"exclude_features": ["TAG_INSTAVEL"]}`. Screening still ranks the remaining
|
|
102
|
+
columns. The banned names are stored in the bundle as `excluded_features`
|
|
103
|
+
(audit only; inference uses `selected_features`).
|
|
104
|
+
|
|
105
|
+
HDF5 screening tests skip unless `SSOI_H5_PATH` or `data/*.h5` exists. They
|
|
106
|
+
do not train a full TCDR and do not print process values.
|
|
107
|
+
|
|
108
|
+
This runs blocked chronological split with adaptive head-skip, train-only
|
|
109
|
+
feature screening, mask-weighted TCDR training, bundle export, and held-out
|
|
110
|
+
outage-mode metrics via `VirtualSensor.predict`.
|
|
111
|
+
|
|
112
|
+
Several targets at once: one process per target, started together by
|
|
113
|
+
`examples/train_parallel.py` (JSON job list, `max_parallel` cap). See
|
|
114
|
+
`examples/train_parallel.example.json`. Each job still needs its own
|
|
115
|
+
`bundle_dir`.
|
|
116
|
+
|
|
117
|
+
## Ecosystem integration
|
|
118
|
+
|
|
119
|
+
```python
|
|
120
|
+
from ssoi import VirtualSensor
|
|
121
|
+
|
|
122
|
+
sensor = VirtualSensor.from_bundle("path/to/bundle")
|
|
123
|
+
y_hat = sensor.predict(X_dataframe_or_ndarray)
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
### Bundle contract
|
|
127
|
+
|
|
128
|
+
A bundle directory must contain:
|
|
129
|
+
|
|
130
|
+
| File | Role |
|
|
131
|
+
|------|------|
|
|
132
|
+
| `manifest.json` | metadata / schema version |
|
|
133
|
+
| `feature_selection_config.json` | ordered auxiliary feature names |
|
|
134
|
+
| `train_feature_means.json` | train-only fill values for NaNs in `x` |
|
|
135
|
+
| `scaler_X.joblib` | feature scaler fitted on train |
|
|
136
|
+
| `scaler_y.joblib` | target scaler fitted on train |
|
|
137
|
+
| `model_config.json` | architecture hyperparameters |
|
|
138
|
+
| `best_model.pth` | trained weights |
|
|
139
|
+
|
|
140
|
+
### Input / output
|
|
141
|
+
|
|
142
|
+
- **Input:** auxiliaries in the feature order from the bundle (DataFrame columns
|
|
143
|
+
or `ndarray` with shape `(n, n_features)`).
|
|
144
|
+
- **Output:** `t_hat` in engineering units (inverse-scaled).
|
|
145
|
+
- **Missing auxiliaries:** filled with train means, then `0.0` as fallback.
|
|
146
|
+
- **Target channel:** always treated as missing at inference (`m_t=1`).
|
|
147
|
+
- **Batching:** `predict(..., batch_size=1024)` slices the forward pass (override as needed).
|
|
148
|
+
|
|
149
|
+
## Not in scope here
|
|
150
|
+
|
|
151
|
+
- Industrial raw historian dumps
|
|
152
|
+
- Paper LaTeX / figure pipelines
|
|
153
|
+
- Full Phase-2 mechanism stress tests
|
|
154
|
+
|
|
155
|
+
Train offline in your environment, publish the **bundle**, and call this package
|
|
156
|
+
only for inference (or for lightweight retrain scripts you add later).
|
|
157
|
+
|
|
158
|
+
## License / authorship
|
|
159
|
+
|
|
160
|
+
Research context: UFRJ industrial virtual-sensing work. Adapt target names and
|
|
161
|
+
feature lists to your plant tags.
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
ssoi/__init__.py,sha256=cyk84kTQIsyDy5-ok-YfE5jizpVhieHJcXeqB3fbdQU,661
|
|
2
|
+
ssoi/bundle.py,sha256=S82o3buxAeX3MauHd2g9grFa1L7Z3XLhfoF0K8-TBJY,7615
|
|
3
|
+
ssoi/model.py,sha256=D_AaSa_3cCvGzd5LdGelxAtYPFq9Wb3q3juYmLnGxZo,3598
|
|
4
|
+
ssoi/predict.py,sha256=DuflyYd4tqw1CsNw6uJ3yhOzrQd8m6sHAtsDe4dttFU,5228
|
|
5
|
+
ssoi/train.py,sha256=4UsTcIKgi-qgedjqCeAMPHj4Jio3oa9ExIu9LJM7EfM,23727
|
|
6
|
+
ssoi/train_jobs.py,sha256=NSDjBzjbOKrxCK02Z1v4w2rB-4sMhxjDTzPRVAnAEN8,11406
|
|
7
|
+
single_sensor_outage_imputer-0.1.0.dist-info/METADATA,sha256=N_oUv9pBM3nOxELWoqs8rKcLwrd49s2sBhxsHKTuR-Q,5168
|
|
8
|
+
single_sensor_outage_imputer-0.1.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
|
|
9
|
+
single_sensor_outage_imputer-0.1.0.dist-info/top_level.txt,sha256=UrrLdnOMSq7TIpY0LPQZAyUuwJgpfBsWBWE84YO6Wkk,5
|
|
10
|
+
single_sensor_outage_imputer-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
ssoi
|
ssoi/__init__.py
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Single-sensor outage imputer (SSOI).
|
|
3
|
+
|
|
4
|
+
Runtime package for target-conditional denoising regressor (TCDR) inference
|
|
5
|
+
under the deployment pattern: auxiliaries observed, target unavailable.
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from ssoi.bundle import ImputerBundle, load_bundle, save_bundle
|
|
10
|
+
from ssoi.model import TargetConditionalDenoisingRegressor
|
|
11
|
+
from ssoi.predict import VirtualSensor
|
|
12
|
+
from ssoi.train import train_from_dataframe, train_from_h5
|
|
13
|
+
|
|
14
|
+
__all__ = [
|
|
15
|
+
"ImputerBundle",
|
|
16
|
+
"TargetConditionalDenoisingRegressor",
|
|
17
|
+
"VirtualSensor",
|
|
18
|
+
"load_bundle",
|
|
19
|
+
"save_bundle",
|
|
20
|
+
"train_from_dataframe",
|
|
21
|
+
"train_from_h5",
|
|
22
|
+
]
|
|
23
|
+
|
|
24
|
+
__version__ = "0.1.0"
|
ssoi/bundle.py
ADDED
|
@@ -0,0 +1,235 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Bundle I/O for deployment artifacts.
|
|
3
|
+
|
|
4
|
+
A bundle is a directory containing:
|
|
5
|
+
- manifest.json
|
|
6
|
+
- feature_selection_config.json (feature column order)
|
|
7
|
+
- train_feature_means.json (per-feature fill values)
|
|
8
|
+
- scaler_X.joblib / scaler_y.joblib
|
|
9
|
+
- model_config.json
|
|
10
|
+
- best_model.pth
|
|
11
|
+
"""
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import json
|
|
15
|
+
from dataclasses import dataclass
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
from typing import Any, Dict, List, Mapping, Optional, Sequence, Union
|
|
18
|
+
|
|
19
|
+
import joblib
|
|
20
|
+
import numpy as np
|
|
21
|
+
import torch
|
|
22
|
+
from sklearn.preprocessing import StandardScaler
|
|
23
|
+
|
|
24
|
+
from ssoi.model import TargetConditionalDenoisingRegressor
|
|
25
|
+
|
|
26
|
+
PathLike = Union[str, Path]
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@dataclass
|
|
30
|
+
class ImputerBundle:
|
|
31
|
+
"""In-memory deployment bundle for single-sensor outage imputation."""
|
|
32
|
+
|
|
33
|
+
features: List[str]
|
|
34
|
+
train_feature_means: Dict[str, float]
|
|
35
|
+
scaler_x: Any
|
|
36
|
+
scaler_y: Any
|
|
37
|
+
model: TargetConditionalDenoisingRegressor
|
|
38
|
+
target_name: str
|
|
39
|
+
manifest: Dict[str, Any]
|
|
40
|
+
model_config: Dict[str, Any]
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _read_json(path: Path) -> Dict[str, Any]:
|
|
44
|
+
with path.open("r", encoding="utf-8") as fh:
|
|
45
|
+
data = json.load(fh)
|
|
46
|
+
if not isinstance(data, dict):
|
|
47
|
+
raise ValueError(f"Expected JSON object in {path}")
|
|
48
|
+
return data
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _write_json(path: Path, payload: Mapping[str, Any]) -> None:
|
|
52
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
53
|
+
with path.open("w", encoding="utf-8") as fh:
|
|
54
|
+
json.dump(dict(payload), fh, indent=2, sort_keys=True)
|
|
55
|
+
fh.write("\n")
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _extract_feature_list(feature_cfg: Mapping[str, Any]) -> List[str]:
|
|
59
|
+
"""Accept common feature-config layouts used in the research pipeline."""
|
|
60
|
+
if "selected_features" in feature_cfg:
|
|
61
|
+
feats = feature_cfg["selected_features"]
|
|
62
|
+
elif "features" in feature_cfg:
|
|
63
|
+
feats = feature_cfg["features"]
|
|
64
|
+
else:
|
|
65
|
+
raise KeyError(
|
|
66
|
+
"feature_selection_config.json must contain "
|
|
67
|
+
"'selected_features' or 'features'"
|
|
68
|
+
)
|
|
69
|
+
if not isinstance(feats, list) or not feats:
|
|
70
|
+
raise ValueError("Feature list must be a non-empty list of strings")
|
|
71
|
+
out: List[str] = []
|
|
72
|
+
for item in feats:
|
|
73
|
+
if not isinstance(item, str) or not item:
|
|
74
|
+
raise ValueError(f"Invalid feature name: {item!r}")
|
|
75
|
+
out.append(item)
|
|
76
|
+
return out
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _means_to_dict(
|
|
80
|
+
features: List[str],
|
|
81
|
+
means_raw: Mapping[str, Any],
|
|
82
|
+
) -> Dict[str, float]:
|
|
83
|
+
"""Normalize means JSON to feature -> float."""
|
|
84
|
+
# Allow {"means": {...}} or flat {feature: value}
|
|
85
|
+
if "means" in means_raw and isinstance(means_raw["means"], dict):
|
|
86
|
+
source = means_raw["means"]
|
|
87
|
+
else:
|
|
88
|
+
source = means_raw
|
|
89
|
+
out: Dict[str, float] = {}
|
|
90
|
+
for name in features:
|
|
91
|
+
if name not in source:
|
|
92
|
+
raise KeyError(f"Missing train mean for feature '{name}'")
|
|
93
|
+
out[name] = float(source[name])
|
|
94
|
+
return out
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def save_bundle(
|
|
98
|
+
bundle_dir: PathLike,
|
|
99
|
+
*,
|
|
100
|
+
features: List[str],
|
|
101
|
+
train_feature_means: Mapping[str, float],
|
|
102
|
+
scaler_x: Any,
|
|
103
|
+
scaler_y: Any,
|
|
104
|
+
model: TargetConditionalDenoisingRegressor,
|
|
105
|
+
target_name: str,
|
|
106
|
+
manifest: Optional[Mapping[str, Any]] = None,
|
|
107
|
+
excluded_features: Optional[Sequence[str]] = None,
|
|
108
|
+
) -> Path:
|
|
109
|
+
"""
|
|
110
|
+
Persist a trainable/deployable bundle to disk.
|
|
111
|
+
|
|
112
|
+
Returns the bundle directory path.
|
|
113
|
+
"""
|
|
114
|
+
root = Path(bundle_dir)
|
|
115
|
+
root.mkdir(parents=True, exist_ok=True)
|
|
116
|
+
|
|
117
|
+
excluded = [str(name) for name in (excluded_features or []) if str(name)]
|
|
118
|
+
feature_cfg = {
|
|
119
|
+
"selected_features": list(features),
|
|
120
|
+
"target": target_name,
|
|
121
|
+
"excluded_features": excluded,
|
|
122
|
+
}
|
|
123
|
+
means_payload = {"means": {k: float(v) for k, v in train_feature_means.items()}}
|
|
124
|
+
model_cfg = model.config_dict()
|
|
125
|
+
man = {
|
|
126
|
+
"schema_version": "1.0",
|
|
127
|
+
"package": "single-sensor-outage-imputer",
|
|
128
|
+
"target_name": target_name,
|
|
129
|
+
"n_features": len(features),
|
|
130
|
+
"inference_mode": "missing_target",
|
|
131
|
+
"inference_channels": {"t_tilde": 0.0, "m_t": 1.0},
|
|
132
|
+
"excluded_features": excluded,
|
|
133
|
+
}
|
|
134
|
+
if manifest:
|
|
135
|
+
man.update(dict(manifest))
|
|
136
|
+
man["excluded_features"] = excluded
|
|
137
|
+
|
|
138
|
+
_write_json(root / "feature_selection_config.json", feature_cfg)
|
|
139
|
+
_write_json(root / "train_feature_means.json", means_payload)
|
|
140
|
+
_write_json(root / "model_config.json", model_cfg)
|
|
141
|
+
_write_json(root / "manifest.json", man)
|
|
142
|
+
joblib.dump(scaler_x, root / "scaler_X.joblib")
|
|
143
|
+
joblib.dump(scaler_y, root / "scaler_y.joblib")
|
|
144
|
+
torch.save(model.state_dict(), root / "best_model.pth")
|
|
145
|
+
return root
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def load_bundle(bundle_dir: PathLike, device: str = "cpu") -> ImputerBundle:
|
|
149
|
+
"""
|
|
150
|
+
Load a deployment bundle from disk.
|
|
151
|
+
|
|
152
|
+
Args:
|
|
153
|
+
bundle_dir: directory with manifest, scalers, and weights
|
|
154
|
+
device: torch device string for the loaded model
|
|
155
|
+
"""
|
|
156
|
+
root = Path(bundle_dir)
|
|
157
|
+
if not root.is_dir():
|
|
158
|
+
raise FileNotFoundError(f"Bundle directory not found: {root}")
|
|
159
|
+
|
|
160
|
+
required = [
|
|
161
|
+
"manifest.json",
|
|
162
|
+
"feature_selection_config.json",
|
|
163
|
+
"train_feature_means.json",
|
|
164
|
+
"model_config.json",
|
|
165
|
+
"scaler_X.joblib",
|
|
166
|
+
"scaler_y.joblib",
|
|
167
|
+
"best_model.pth",
|
|
168
|
+
]
|
|
169
|
+
missing = [name for name in required if not (root / name).exists()]
|
|
170
|
+
if missing:
|
|
171
|
+
raise FileNotFoundError(f"Bundle incomplete at {root}: missing {missing}")
|
|
172
|
+
|
|
173
|
+
manifest = _read_json(root / "manifest.json")
|
|
174
|
+
feature_cfg = _read_json(root / "feature_selection_config.json")
|
|
175
|
+
means_raw = _read_json(root / "train_feature_means.json")
|
|
176
|
+
model_cfg = _read_json(root / "model_config.json")
|
|
177
|
+
|
|
178
|
+
features = _extract_feature_list(feature_cfg)
|
|
179
|
+
means = _means_to_dict(features, means_raw)
|
|
180
|
+
target_name = str(
|
|
181
|
+
manifest.get("target_name")
|
|
182
|
+
or feature_cfg.get("target")
|
|
183
|
+
or "target"
|
|
184
|
+
)
|
|
185
|
+
|
|
186
|
+
n_features = int(model_cfg.get("n_features", len(features)))
|
|
187
|
+
if n_features != len(features):
|
|
188
|
+
raise ValueError(
|
|
189
|
+
f"model_config n_features={n_features} != len(features)={len(features)}"
|
|
190
|
+
)
|
|
191
|
+
latent_dim = int(model_cfg.get("latent_dim", 16))
|
|
192
|
+
dropout = float(model_cfg.get("dropout", 0.2))
|
|
193
|
+
|
|
194
|
+
model = TargetConditionalDenoisingRegressor(
|
|
195
|
+
n_features=n_features,
|
|
196
|
+
latent_dim=latent_dim,
|
|
197
|
+
dropout=dropout,
|
|
198
|
+
)
|
|
199
|
+
weights_path = root / "best_model.pth"
|
|
200
|
+
try:
|
|
201
|
+
state = torch.load(weights_path, map_location=device)
|
|
202
|
+
model.load_state_dict(state)
|
|
203
|
+
except FileNotFoundError:
|
|
204
|
+
raise
|
|
205
|
+
except Exception as exc:
|
|
206
|
+
raise RuntimeError(
|
|
207
|
+
f"Failed to load model weights from {weights_path}: {exc}"
|
|
208
|
+
) from exc
|
|
209
|
+
model.to(device)
|
|
210
|
+
model.eval()
|
|
211
|
+
|
|
212
|
+
scaler_x = joblib.load(root / "scaler_X.joblib")
|
|
213
|
+
scaler_y = joblib.load(root / "scaler_y.joblib")
|
|
214
|
+
if not isinstance(scaler_x, StandardScaler):
|
|
215
|
+
# Accept RobustScaler or other sklearn transformers with transform()
|
|
216
|
+
if not hasattr(scaler_x, "transform"):
|
|
217
|
+
raise TypeError("scaler_X must expose .transform()")
|
|
218
|
+
n_in = getattr(scaler_x, "n_features_in_", None)
|
|
219
|
+
if n_in is not None and int(n_in) != len(features):
|
|
220
|
+
raise ValueError(
|
|
221
|
+
f"scaler_X n_features_in_={int(n_in)} != len(features)={len(features)}"
|
|
222
|
+
)
|
|
223
|
+
if not hasattr(scaler_y, "inverse_transform"):
|
|
224
|
+
raise TypeError("scaler_y must expose .inverse_transform()")
|
|
225
|
+
|
|
226
|
+
return ImputerBundle(
|
|
227
|
+
features=features,
|
|
228
|
+
train_feature_means=means,
|
|
229
|
+
scaler_x=scaler_x,
|
|
230
|
+
scaler_y=scaler_y,
|
|
231
|
+
model=model,
|
|
232
|
+
target_name=target_name,
|
|
233
|
+
manifest=manifest,
|
|
234
|
+
model_config=model_cfg,
|
|
235
|
+
)
|
ssoi/model.py
ADDED
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
"""
|
|
2
|
+
TCDR network: target-conditional denoising regressor.
|
|
3
|
+
|
|
4
|
+
Maps observability-aware input [x; t_tilde; m_t] to scalar target estimate.
|
|
5
|
+
Extracted from the paper research implementation (ConditionalDenoisingAE).
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from typing import Dict, Tuple
|
|
10
|
+
|
|
11
|
+
import torch
|
|
12
|
+
import torch.nn as nn
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def tcdr_hidden_widths(n_features: int, latent_dim: int = 16) -> Tuple[int, int, int]:
|
|
16
|
+
"""
|
|
17
|
+
Compute hidden widths from auxiliary feature count.
|
|
18
|
+
|
|
19
|
+
Capacity uses in_dim = n_features + 2 (x plus t_tilde and m_t), matching
|
|
20
|
+
the manuscript sizing rule.
|
|
21
|
+
"""
|
|
22
|
+
if n_features < 1:
|
|
23
|
+
raise ValueError(f"n_features must be >= 1, got {n_features}")
|
|
24
|
+
capacity_in_dim = int(n_features) + 2
|
|
25
|
+
h1 = max(64, 8 * capacity_in_dim)
|
|
26
|
+
h2 = max(32, 4 * capacity_in_dim)
|
|
27
|
+
return h1, h2, int(latent_dim)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class TargetConditionalDenoisingRegressor(nn.Module):
|
|
31
|
+
"""
|
|
32
|
+
Supervised scalar regressor for single-sensor outage recovery.
|
|
33
|
+
|
|
34
|
+
Input channels:
|
|
35
|
+
- x: auxiliary sensors (n_features)
|
|
36
|
+
- t_tilde: observed target or 0 when missing (scaled space)
|
|
37
|
+
- m_t: 1 if target unavailable, 0 if observed
|
|
38
|
+
|
|
39
|
+
Output:
|
|
40
|
+
- t_hat: predicted target in the same scaled space as training
|
|
41
|
+
"""
|
|
42
|
+
|
|
43
|
+
def __init__(
|
|
44
|
+
self,
|
|
45
|
+
n_features: int,
|
|
46
|
+
latent_dim: int = 16,
|
|
47
|
+
dropout: float = 0.2,
|
|
48
|
+
) -> None:
|
|
49
|
+
super().__init__()
|
|
50
|
+
if n_features < 1:
|
|
51
|
+
raise ValueError(f"n_features must be >= 1, got {n_features}")
|
|
52
|
+
self.n_features = int(n_features)
|
|
53
|
+
self.latent_dim = int(latent_dim)
|
|
54
|
+
self.dropout = float(dropout)
|
|
55
|
+
h1, h2, latent = tcdr_hidden_widths(self.n_features, self.latent_dim)
|
|
56
|
+
self.h1 = h1
|
|
57
|
+
self.h2 = h2
|
|
58
|
+
in_dim = self.n_features + 2
|
|
59
|
+
self.net = nn.Sequential(
|
|
60
|
+
nn.Linear(in_dim, h1),
|
|
61
|
+
nn.BatchNorm1d(h1),
|
|
62
|
+
nn.LeakyReLU(0.1),
|
|
63
|
+
nn.Dropout(self.dropout),
|
|
64
|
+
nn.Linear(h1, h2),
|
|
65
|
+
nn.BatchNorm1d(h2),
|
|
66
|
+
nn.LeakyReLU(0.1),
|
|
67
|
+
nn.Dropout(self.dropout / 2.0),
|
|
68
|
+
nn.Linear(h2, latent),
|
|
69
|
+
nn.BatchNorm1d(latent),
|
|
70
|
+
nn.LeakyReLU(0.1),
|
|
71
|
+
nn.Linear(latent, h2),
|
|
72
|
+
nn.LeakyReLU(0.1),
|
|
73
|
+
nn.Linear(h2, 1),
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
def forward(
|
|
77
|
+
self,
|
|
78
|
+
x: torch.Tensor,
|
|
79
|
+
t_tilde: torch.Tensor,
|
|
80
|
+
m_t: torch.Tensor,
|
|
81
|
+
) -> torch.Tensor:
|
|
82
|
+
"""
|
|
83
|
+
Forward pass.
|
|
84
|
+
|
|
85
|
+
Args:
|
|
86
|
+
x: shape (batch, n_features)
|
|
87
|
+
t_tilde: shape (batch, 1)
|
|
88
|
+
m_t: shape (batch, 1)
|
|
89
|
+
"""
|
|
90
|
+
if x.ndim != 2:
|
|
91
|
+
raise ValueError(f"x must be 2D (batch, n_features), got shape {tuple(x.shape)}")
|
|
92
|
+
if x.shape[1] != self.n_features:
|
|
93
|
+
raise ValueError(
|
|
94
|
+
f"x has {x.shape[1]} features, model expects {self.n_features}"
|
|
95
|
+
)
|
|
96
|
+
inp = torch.cat([x, t_tilde, m_t], dim=1)
|
|
97
|
+
return self.net(inp)
|
|
98
|
+
|
|
99
|
+
def parameter_count(self) -> int:
|
|
100
|
+
"""Return number of trainable parameters."""
|
|
101
|
+
return int(sum(p.numel() for p in self.parameters() if p.requires_grad))
|
|
102
|
+
|
|
103
|
+
def config_dict(self) -> Dict[str, object]:
|
|
104
|
+
"""Serializable architecture metadata for bundle manifests."""
|
|
105
|
+
return {
|
|
106
|
+
"model_type": "tcdr",
|
|
107
|
+
"n_features": self.n_features,
|
|
108
|
+
"in_dim": self.n_features + 2,
|
|
109
|
+
"h1": self.h1,
|
|
110
|
+
"h2": self.h2,
|
|
111
|
+
"latent_dim": self.latent_dim,
|
|
112
|
+
"dropout": self.dropout,
|
|
113
|
+
"parameter_count": self.parameter_count(),
|
|
114
|
+
}
|
ssoi/predict.py
ADDED
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Inference API for single-sensor outage virtual sensing.
|
|
3
|
+
|
|
4
|
+
Deployment pattern: auxiliaries available, target missing
|
|
5
|
+
t_hat = f_theta(x_scaled, t_tilde=0, m_t=1)
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import Any, Dict, List, Mapping, Optional, Sequence, Union
|
|
11
|
+
|
|
12
|
+
import numpy as np
|
|
13
|
+
import pandas as pd
|
|
14
|
+
import torch
|
|
15
|
+
|
|
16
|
+
from ssoi.bundle import ImputerBundle, load_bundle
|
|
17
|
+
|
|
18
|
+
ArrayLike = Union[np.ndarray, pd.DataFrame, Sequence[Sequence[float]]]
|
|
19
|
+
PathLike = Union[str, Path]
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class VirtualSensor:
|
|
23
|
+
"""
|
|
24
|
+
Load a TCDR bundle and impute the target under outage conditions.
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
def __init__(self, bundle: ImputerBundle, device: str = "cpu") -> None:
|
|
28
|
+
self.bundle = bundle
|
|
29
|
+
self.device = device
|
|
30
|
+
self.bundle.model.to(device)
|
|
31
|
+
self.bundle.model.eval()
|
|
32
|
+
|
|
33
|
+
@classmethod
|
|
34
|
+
def from_bundle(cls, bundle_dir: PathLike, device: str = "cpu") -> "VirtualSensor":
|
|
35
|
+
"""Construct from a bundle directory path."""
|
|
36
|
+
return cls(load_bundle(bundle_dir, device=device), device=device)
|
|
37
|
+
|
|
38
|
+
@property
|
|
39
|
+
def features(self) -> List[str]:
|
|
40
|
+
"""Ordered auxiliary feature names required at inference."""
|
|
41
|
+
return list(self.bundle.features)
|
|
42
|
+
|
|
43
|
+
@property
|
|
44
|
+
def target_name(self) -> str:
|
|
45
|
+
"""Name of the reconstructed target tag."""
|
|
46
|
+
return self.bundle.target_name
|
|
47
|
+
|
|
48
|
+
def _frame_from_input(self, x: ArrayLike) -> pd.DataFrame:
|
|
49
|
+
"""Normalize caller input to a DataFrame with expected columns."""
|
|
50
|
+
if isinstance(x, pd.DataFrame):
|
|
51
|
+
missing = [c for c in self.features if c not in x.columns]
|
|
52
|
+
if missing:
|
|
53
|
+
raise KeyError(f"Input DataFrame missing columns: {missing}")
|
|
54
|
+
return x.loc[:, self.features].copy()
|
|
55
|
+
|
|
56
|
+
arr = np.asarray(x, dtype=np.float64)
|
|
57
|
+
if arr.ndim == 1:
|
|
58
|
+
arr = arr.reshape(1, -1)
|
|
59
|
+
if arr.ndim != 2:
|
|
60
|
+
raise ValueError(f"Expected 2D feature array, got shape {arr.shape}")
|
|
61
|
+
if arr.shape[1] != len(self.features):
|
|
62
|
+
raise ValueError(
|
|
63
|
+
f"Expected {len(self.features)} features, got {arr.shape[1]}"
|
|
64
|
+
)
|
|
65
|
+
return pd.DataFrame(arr, columns=self.features)
|
|
66
|
+
|
|
67
|
+
def _impute_auxiliaries(self, frame: pd.DataFrame) -> np.ndarray:
|
|
68
|
+
"""Fill NaNs in auxiliaries using train-only means, then zeros."""
|
|
69
|
+
means = self.bundle.train_feature_means
|
|
70
|
+
filled = frame.copy()
|
|
71
|
+
for col in self.features:
|
|
72
|
+
filled[col] = filled[col].fillna(means[col])
|
|
73
|
+
filled = filled.fillna(0.0)
|
|
74
|
+
return filled.to_numpy(dtype=np.float64)
|
|
75
|
+
|
|
76
|
+
def predict(
|
|
77
|
+
self,
|
|
78
|
+
x: ArrayLike,
|
|
79
|
+
*,
|
|
80
|
+
return_scaled: bool = False,
|
|
81
|
+
batch_size: int = 1024,
|
|
82
|
+
) -> np.ndarray:
|
|
83
|
+
"""
|
|
84
|
+
Predict target values in outage mode (m_t=1, t_tilde=0).
|
|
85
|
+
|
|
86
|
+
Args:
|
|
87
|
+
x: DataFrame with feature columns, or ndarray (n, n_features)
|
|
88
|
+
return_scaled: if True, return predictions in scaled target space
|
|
89
|
+
batch_size: max rows per forward pass (must be >= 1)
|
|
90
|
+
|
|
91
|
+
Returns:
|
|
92
|
+
1D numpy array of predictions (engineering units by default)
|
|
93
|
+
"""
|
|
94
|
+
if int(batch_size) < 1:
|
|
95
|
+
raise ValueError(f"batch_size must be >= 1, got {batch_size}")
|
|
96
|
+
frame = self._frame_from_input(x)
|
|
97
|
+
x_raw = self._impute_auxiliaries(frame)
|
|
98
|
+
x_scaled = self.bundle.scaler_x.transform(x_raw).astype(np.float32)
|
|
99
|
+
|
|
100
|
+
n = int(x_scaled.shape[0])
|
|
101
|
+
chunks: List[np.ndarray] = []
|
|
102
|
+
step = int(batch_size)
|
|
103
|
+
self.bundle.model.eval()
|
|
104
|
+
with torch.no_grad():
|
|
105
|
+
for start in range(0, n, step):
|
|
106
|
+
stop = min(start + step, n)
|
|
107
|
+
x_t = torch.from_numpy(x_scaled[start:stop]).to(self.device)
|
|
108
|
+
rows = int(x_t.shape[0])
|
|
109
|
+
t_zero = torch.zeros((rows, 1), dtype=torch.float32, device=self.device)
|
|
110
|
+
m_one = torch.ones((rows, 1), dtype=torch.float32, device=self.device)
|
|
111
|
+
y_hat_scaled = self.bundle.model(x_t, t_zero, m_one).cpu().numpy()
|
|
112
|
+
chunks.append(y_hat_scaled.reshape(-1))
|
|
113
|
+
|
|
114
|
+
if chunks:
|
|
115
|
+
y_hat_scaled = np.concatenate(chunks, axis=0)
|
|
116
|
+
else:
|
|
117
|
+
y_hat_scaled = np.empty((0,), dtype=np.float32)
|
|
118
|
+
|
|
119
|
+
if return_scaled:
|
|
120
|
+
return y_hat_scaled.reshape(-1)
|
|
121
|
+
|
|
122
|
+
y_hat = self.bundle.scaler_y.inverse_transform(y_hat_scaled.reshape(-1, 1))
|
|
123
|
+
return y_hat.reshape(-1)
|
|
124
|
+
|
|
125
|
+
def predict_dict_rows(
|
|
126
|
+
self,
|
|
127
|
+
rows: Sequence[Mapping[str, Any]],
|
|
128
|
+
) -> List[float]:
|
|
129
|
+
"""
|
|
130
|
+
Convenience API for ecosystem messages: list of feature dicts -> list of floats.
|
|
131
|
+
"""
|
|
132
|
+
frame = pd.DataFrame(list(rows))
|
|
133
|
+
preds = self.predict(frame)
|
|
134
|
+
return [float(v) for v in preds]
|
|
135
|
+
|
|
136
|
+
def info(self) -> Dict[str, Any]:
|
|
137
|
+
"""Return a small status dict useful for health checks."""
|
|
138
|
+
return {
|
|
139
|
+
"target_name": self.target_name,
|
|
140
|
+
"n_features": len(self.features),
|
|
141
|
+
"features": self.features,
|
|
142
|
+
"device": self.device,
|
|
143
|
+
"manifest": dict(self.bundle.manifest),
|
|
144
|
+
"parameter_count": self.bundle.model.parameter_count(),
|
|
145
|
+
}
|