torchriver 0.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- torchriver/__init__.py +23 -0
- torchriver/cli.py +40 -0
- torchriver/configs.py +130 -0
- torchriver/data.py +410 -0
- torchriver/models.py +180 -0
- torchriver/plotting.py +87 -0
- torchriver/project.py +571 -0
- torchriver/serialization.py +87 -0
- torchriver/training.py +180 -0
- torchriver/ui.py +80 -0
- torchriver-0.1.1.dist-info/METADATA +284 -0
- torchriver-0.1.1.dist-info/RECORD +16 -0
- torchriver-0.1.1.dist-info/WHEEL +5 -0
- torchriver-0.1.1.dist-info/entry_points.txt +3 -0
- torchriver-0.1.1.dist-info/licenses/LICENSE +21 -0
- torchriver-0.1.1.dist-info/top_level.txt +1 -0
torchriver/__init__.py
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
"""TorchRiver: low-flow / drought forecasting for river time-series data."""
|
|
2
|
+
|
|
3
|
+
from .configs import RiverDataConfig, RiverModelConfig, RiverTrainConfig, RiverUIConfig, TorchRiverWorkflow
|
|
4
|
+
from .data import make_synthetic_data
|
|
5
|
+
from .models import MODEL_REGISTRY, RiverCNNLSTM, build_model
|
|
6
|
+
from .project import RiverProject
|
|
7
|
+
from .serialization import load_river_model
|
|
8
|
+
|
|
9
|
+
__version__ = "0.1.1"
|
|
10
|
+
|
|
11
|
+
__all__ = [
|
|
12
|
+
"RiverProject",
|
|
13
|
+
"load_river_model",
|
|
14
|
+
"RiverDataConfig",
|
|
15
|
+
"RiverModelConfig",
|
|
16
|
+
"RiverTrainConfig",
|
|
17
|
+
"RiverUIConfig",
|
|
18
|
+
"TorchRiverWorkflow",
|
|
19
|
+
"RiverCNNLSTM",
|
|
20
|
+
"MODEL_REGISTRY",
|
|
21
|
+
"build_model",
|
|
22
|
+
"make_synthetic_data",
|
|
23
|
+
]
|
torchriver/cli.py
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
"""Command-line helpers for TorchRiver."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import json
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
from . import __version__
|
|
10
|
+
from .project import RiverProject
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def smoke() -> None:
|
|
14
|
+
"""Print package information; useful after pip install."""
|
|
15
|
+
print(f"torchriver {__version__} installed successfully")
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def synthetic_demo() -> None:
|
|
19
|
+
"""Train a tiny synthetic demo and optionally save a .triver model."""
|
|
20
|
+
parser = argparse.ArgumentParser(description="Run a tiny TorchRiver synthetic training demo.")
|
|
21
|
+
parser.add_argument("--days", type=int, default=120)
|
|
22
|
+
parser.add_argument("--epochs", type=int, default=1)
|
|
23
|
+
parser.add_argument("--output", type=str, default="", help="Optional .triver output path")
|
|
24
|
+
args = parser.parse_args()
|
|
25
|
+
|
|
26
|
+
project = RiverProject(title="TorchRiver synthetic demo")
|
|
27
|
+
project.make_synthetic(days=args.days, station_codes=["R001", "R002"])
|
|
28
|
+
project.use_model("river_cnn_lstm", horizons=[7], lookback=14, hidden_dim=16)
|
|
29
|
+
project.train(epochs=args.epochs, batch_size=128, min_windows=3, verbose=False)
|
|
30
|
+
result = project.predict(station="R001", horizon=7)
|
|
31
|
+
print(json.dumps(result, ensure_ascii=False, indent=2))
|
|
32
|
+
|
|
33
|
+
if args.output:
|
|
34
|
+
output = Path(args.output)
|
|
35
|
+
project.save_model(output, include_recent_data=True)
|
|
36
|
+
print(f"saved: {output}")
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
if __name__ == "__main__":
|
|
40
|
+
smoke()
|
torchriver/configs.py
ADDED
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
"""Configuration objects for TorchRiver."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass, field
|
|
6
|
+
from typing import Any, Dict, Iterable, List, Optional
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
@dataclass
|
|
10
|
+
class RiverDataConfig:
|
|
11
|
+
"""Configuration for loading and normalizing river data."""
|
|
12
|
+
|
|
13
|
+
source: str = "synthetic"
|
|
14
|
+
path: Optional[str] = None
|
|
15
|
+
dataset_url: Optional[str] = None
|
|
16
|
+
nrows: Optional[int] = None
|
|
17
|
+
mapping: Optional[Dict[str, str]] = None
|
|
18
|
+
station_names: Optional[Dict[str, str]] = None
|
|
19
|
+
fallback_synthetic: bool = True
|
|
20
|
+
synthetic_days: int = 240
|
|
21
|
+
synthetic_station_codes: List[str] = field(default_factory=lambda: ["R001", "R002", "R003"])
|
|
22
|
+
column_names: Optional[Dict[str, str]] = None
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@dataclass
|
|
26
|
+
class RiverModelConfig:
|
|
27
|
+
"""Configuration for a TorchRiver model family."""
|
|
28
|
+
|
|
29
|
+
name: str = "river_cnn_lstm"
|
|
30
|
+
horizons: List[int] = field(default_factory=lambda: [7, 14, 30])
|
|
31
|
+
lookback: int = 30
|
|
32
|
+
window_mode: str = "per_station"
|
|
33
|
+
device: str = "auto"
|
|
34
|
+
legacy_sigmoid: bool = False
|
|
35
|
+
hidden_dim: int = 64
|
|
36
|
+
dropout: float = 0.0
|
|
37
|
+
balance_classes: bool = True
|
|
38
|
+
model_kwargs: Dict[str, Any] = field(default_factory=dict)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@dataclass
|
|
42
|
+
class RiverTrainConfig:
|
|
43
|
+
"""Configuration for training."""
|
|
44
|
+
|
|
45
|
+
epochs: int = 2
|
|
46
|
+
batch_size: int = 512
|
|
47
|
+
lr: float = 0.001
|
|
48
|
+
validation_split: float = 0.2
|
|
49
|
+
balance_classes: Optional[bool] = None
|
|
50
|
+
seed: int = 42
|
|
51
|
+
min_windows: int = 5
|
|
52
|
+
verbose: bool = True
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
@dataclass
|
|
56
|
+
class RiverUIConfig:
|
|
57
|
+
"""Configuration for the Gradio interface."""
|
|
58
|
+
|
|
59
|
+
title: str = "River Low-Flow Early Warning System"
|
|
60
|
+
lang: str = "en"
|
|
61
|
+
direction: str = "ltr"
|
|
62
|
+
labels: Dict[str, str] = field(default_factory=dict)
|
|
63
|
+
logo: Optional[str] = None
|
|
64
|
+
share: bool = False
|
|
65
|
+
api: bool = True
|
|
66
|
+
output_mode: str = "text"
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
@dataclass
|
|
70
|
+
class TorchRiverWorkflow:
|
|
71
|
+
"""Advanced workflow wrapper combining data, model, training, and UI configs."""
|
|
72
|
+
|
|
73
|
+
data: RiverDataConfig
|
|
74
|
+
model: RiverModelConfig = field(default_factory=RiverModelConfig)
|
|
75
|
+
train_config: RiverTrainConfig = field(default_factory=RiverTrainConfig)
|
|
76
|
+
ui: RiverUIConfig = field(default_factory=RiverUIConfig)
|
|
77
|
+
|
|
78
|
+
def run(self):
|
|
79
|
+
"""Run loading, cleaning, model selection, and training. Returns the project."""
|
|
80
|
+
from .project import RiverProject
|
|
81
|
+
|
|
82
|
+
project = RiverProject(title=self.ui.title, lang=self.ui.lang, direction=self.ui.direction)
|
|
83
|
+
if self.data.source == "kaggle":
|
|
84
|
+
project.load_kaggle(
|
|
85
|
+
self.data.dataset_url or self.data.path or "",
|
|
86
|
+
nrows=self.data.nrows,
|
|
87
|
+
mapping=self.data.mapping,
|
|
88
|
+
station_names=self.data.station_names,
|
|
89
|
+
fallback_synthetic=self.data.fallback_synthetic,
|
|
90
|
+
)
|
|
91
|
+
elif self.data.source == "folder":
|
|
92
|
+
project.load_folder(self.data.path or "", nrows=self.data.nrows, mapping=self.data.mapping, station_names=self.data.station_names)
|
|
93
|
+
elif self.data.source == "file":
|
|
94
|
+
project.load_file(self.data.path or "", nrows=self.data.nrows, mapping=self.data.mapping, station_names=self.data.station_names)
|
|
95
|
+
elif self.data.source == "url":
|
|
96
|
+
project.load_url(self.data.dataset_url or self.data.path or "", nrows=self.data.nrows, mapping=self.data.mapping, station_names=self.data.station_names)
|
|
97
|
+
elif self.data.source == "synthetic":
|
|
98
|
+
project.make_synthetic(
|
|
99
|
+
days=self.data.synthetic_days,
|
|
100
|
+
station_codes=self.data.synthetic_station_codes,
|
|
101
|
+
station_names=self.data.station_names,
|
|
102
|
+
column_names=self.data.column_names,
|
|
103
|
+
)
|
|
104
|
+
else:
|
|
105
|
+
raise ValueError(f"Unknown data source: {self.data.source!r}")
|
|
106
|
+
|
|
107
|
+
project.clean()
|
|
108
|
+
project.use_model(
|
|
109
|
+
self.model.name,
|
|
110
|
+
horizons=self.model.horizons,
|
|
111
|
+
lookback=self.model.lookback,
|
|
112
|
+
device=self.model.device,
|
|
113
|
+
window_mode=self.model.window_mode,
|
|
114
|
+
balance_classes=self.model.balance_classes,
|
|
115
|
+
legacy_sigmoid=self.model.legacy_sigmoid,
|
|
116
|
+
hidden_dim=self.model.hidden_dim,
|
|
117
|
+
dropout=self.model.dropout,
|
|
118
|
+
**self.model.model_kwargs,
|
|
119
|
+
)
|
|
120
|
+
project.train(
|
|
121
|
+
epochs=self.train_config.epochs,
|
|
122
|
+
batch_size=self.train_config.batch_size,
|
|
123
|
+
lr=self.train_config.lr,
|
|
124
|
+
validation_split=self.train_config.validation_split,
|
|
125
|
+
balance_classes=self.train_config.balance_classes,
|
|
126
|
+
seed=self.train_config.seed,
|
|
127
|
+
min_windows=self.train_config.min_windows,
|
|
128
|
+
verbose=self.train_config.verbose,
|
|
129
|
+
)
|
|
130
|
+
return project
|
torchriver/data.py
ADDED
|
@@ -0,0 +1,410 @@
|
|
|
1
|
+
"""Data loading, schema mapping, cleaning, synthetic generation, and windows."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import io
|
|
6
|
+
import os
|
|
7
|
+
import re
|
|
8
|
+
import tempfile
|
|
9
|
+
import zipfile
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from typing import Dict, Iterable, List, Optional, Sequence, Tuple
|
|
12
|
+
from urllib.parse import urlparse
|
|
13
|
+
from urllib.request import urlopen
|
|
14
|
+
|
|
15
|
+
import numpy as np
|
|
16
|
+
import pandas as pd
|
|
17
|
+
from sklearn.preprocessing import StandardScaler
|
|
18
|
+
|
|
19
|
+
INTERNAL_COLUMNS = {
|
|
20
|
+
"date": "date",
|
|
21
|
+
"station": "station_code",
|
|
22
|
+
"station_name": "station",
|
|
23
|
+
"discharge": "discharge_m3s",
|
|
24
|
+
"low_flow_threshold": "low_flow_q10_threshold_m3s",
|
|
25
|
+
"target": "target_lowflow",
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
COLUMN_SYNONYMS = {
|
|
29
|
+
"date": ["date", "day", "timestamp", "time", "datetime", "measurement_date"],
|
|
30
|
+
"station": ["station_code", "station_id", "gauge_id", "site_no", "river_id", "stationcode", "gauge"],
|
|
31
|
+
"station_name": ["station", "station_name", "gauge_name", "river_name", "name", "site_name"],
|
|
32
|
+
"discharge": ["discharge_m3s", "discharge", "flow", "streamflow", "q_m3s", "q", "value", "runoff"],
|
|
33
|
+
"low_flow_threshold": ["low_flow_q10_threshold_m3s", "q10", "threshold", "low_flow_threshold", "critical_flow"],
|
|
34
|
+
"target": ["target_lowflow", "low_flow", "low_flow_flag", "drought", "label"],
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
EXCLUDED_COLS = [
|
|
38
|
+
"date",
|
|
39
|
+
"station_code",
|
|
40
|
+
"station",
|
|
41
|
+
"station_original",
|
|
42
|
+
"river",
|
|
43
|
+
"target_lowflow",
|
|
44
|
+
]
|
|
45
|
+
|
|
46
|
+
SUPPORTED_EXTENSIONS = [".csv", ".xlsx", ".xls", ".parquet", ".json", ".jsonl", ".feather", ".pkl", ".pickle"]
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _norm(name: str) -> str:
|
|
50
|
+
return re.sub(r"[^a-z0-9]+", "_", str(name).strip().lower()).strip("_")
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def infer_mapping(columns: Iterable[str], mapping: Optional[Dict[str, str]] = None) -> Dict[str, str]:
|
|
54
|
+
"""Infer a user-column -> internal-column mapping.
|
|
55
|
+
|
|
56
|
+
The input ``mapping`` follows the public API style, e.g.
|
|
57
|
+
``{"date": "day", "station": "gauge_id"}``. Returned mapping maps actual
|
|
58
|
+
dataframe columns to canonical TorchRiver columns.
|
|
59
|
+
"""
|
|
60
|
+
columns = list(columns)
|
|
61
|
+
normalized_to_original = {_norm(c): c for c in columns}
|
|
62
|
+
result: Dict[str, str] = {}
|
|
63
|
+
|
|
64
|
+
if mapping:
|
|
65
|
+
for semantic_name, actual_col in mapping.items():
|
|
66
|
+
if actual_col is None:
|
|
67
|
+
continue
|
|
68
|
+
target = INTERNAL_COLUMNS.get(semantic_name, semantic_name)
|
|
69
|
+
if actual_col in columns:
|
|
70
|
+
result[actual_col] = target
|
|
71
|
+
else:
|
|
72
|
+
normalized = _norm(actual_col)
|
|
73
|
+
if normalized in normalized_to_original:
|
|
74
|
+
result[normalized_to_original[normalized]] = target
|
|
75
|
+
|
|
76
|
+
already_targets = set(result.values())
|
|
77
|
+
for semantic_name, candidates in COLUMN_SYNONYMS.items():
|
|
78
|
+
target = INTERNAL_COLUMNS[semantic_name]
|
|
79
|
+
if target in already_targets:
|
|
80
|
+
continue
|
|
81
|
+
for candidate in candidates:
|
|
82
|
+
if _norm(candidate) in normalized_to_original:
|
|
83
|
+
result[normalized_to_original[_norm(candidate)]] = target
|
|
84
|
+
already_targets.add(target)
|
|
85
|
+
break
|
|
86
|
+
return result
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def apply_mapping(df: pd.DataFrame, mapping: Optional[Dict[str, str]] = None) -> Tuple[pd.DataFrame, Dict[str, str]]:
|
|
90
|
+
actual_mapping = infer_mapping(df.columns, mapping)
|
|
91
|
+
out = df.copy()
|
|
92
|
+
rename_map = {src: dst for src, dst in actual_mapping.items() if src != dst}
|
|
93
|
+
out = out.rename(columns=rename_map)
|
|
94
|
+
return out, actual_mapping
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def read_data_file(path: str | os.PathLike[str], nrows: Optional[int] = None) -> pd.DataFrame:
|
|
98
|
+
"""Read a CSV, Excel, Parquet, JSON/JSONL, Feather, or Pickle file."""
|
|
99
|
+
path = str(path)
|
|
100
|
+
ext = Path(path).suffix.lower()
|
|
101
|
+
if ext == ".csv":
|
|
102
|
+
return pd.read_csv(path, nrows=nrows, low_memory=False)
|
|
103
|
+
if ext in {".xlsx", ".xls"}:
|
|
104
|
+
return pd.read_excel(path, nrows=nrows)
|
|
105
|
+
if ext == ".parquet":
|
|
106
|
+
df = pd.read_parquet(path)
|
|
107
|
+
return df.head(nrows) if nrows else df
|
|
108
|
+
if ext == ".json":
|
|
109
|
+
df = pd.read_json(path)
|
|
110
|
+
return df.head(nrows) if nrows else df
|
|
111
|
+
if ext == ".jsonl":
|
|
112
|
+
return pd.read_json(path, lines=True, nrows=nrows)
|
|
113
|
+
if ext == ".feather":
|
|
114
|
+
df = pd.read_feather(path)
|
|
115
|
+
return df.head(nrows) if nrows else df
|
|
116
|
+
if ext in {".pkl", ".pickle"}:
|
|
117
|
+
df = pd.read_pickle(path)
|
|
118
|
+
return df.head(nrows) if nrows else df
|
|
119
|
+
raise ValueError(f"Unsupported file type {ext!r}. Supported: {SUPPORTED_EXTENSIONS}")
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def _candidate_score(path: Path) -> int:
|
|
123
|
+
name = path.name.lower()
|
|
124
|
+
score = 0
|
|
125
|
+
if "combined" in name:
|
|
126
|
+
score += 8
|
|
127
|
+
if "elbe" in name:
|
|
128
|
+
score += 5
|
|
129
|
+
if "river" in name:
|
|
130
|
+
score += 4
|
|
131
|
+
if "station" in name or "gauge" in name:
|
|
132
|
+
score += 2
|
|
133
|
+
if path.suffix.lower() == ".csv":
|
|
134
|
+
score += 1
|
|
135
|
+
try:
|
|
136
|
+
score += min(path.stat().st_size // 1_000_000, 5)
|
|
137
|
+
except OSError:
|
|
138
|
+
pass
|
|
139
|
+
return score
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def find_data_file(folder: str | os.PathLike[str]) -> Path:
|
|
143
|
+
"""Recursively find the most likely data file in a folder."""
|
|
144
|
+
root = Path(folder)
|
|
145
|
+
if not root.exists():
|
|
146
|
+
raise FileNotFoundError(f"Folder does not exist: {folder}")
|
|
147
|
+
candidates = [p for p in root.rglob("*") if p.is_file() and p.suffix.lower() in SUPPORTED_EXTENSIONS]
|
|
148
|
+
if not candidates:
|
|
149
|
+
raise FileNotFoundError(f"No supported data file found under {folder}")
|
|
150
|
+
return sorted(candidates, key=_candidate_score, reverse=True)[0]
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def _download_url_to_temp(url: str) -> Path:
|
|
154
|
+
suffix = Path(urlparse(url).path).suffix or ".data"
|
|
155
|
+
fd, temp_path = tempfile.mkstemp(suffix=suffix)
|
|
156
|
+
os.close(fd)
|
|
157
|
+
with urlopen(url) as response, open(temp_path, "wb") as f:
|
|
158
|
+
f.write(response.read())
|
|
159
|
+
return Path(temp_path)
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def load_url(url: str, nrows: Optional[int] = None) -> pd.DataFrame:
|
|
163
|
+
"""Load a direct data URL or a ZIP URL containing a supported file."""
|
|
164
|
+
temp_path = _download_url_to_temp(url)
|
|
165
|
+
try:
|
|
166
|
+
if temp_path.suffix.lower() == ".zip":
|
|
167
|
+
extract_dir = Path(tempfile.mkdtemp(prefix="torchriver_url_"))
|
|
168
|
+
with zipfile.ZipFile(temp_path) as zf:
|
|
169
|
+
zf.extractall(extract_dir)
|
|
170
|
+
return read_data_file(find_data_file(extract_dir), nrows=nrows)
|
|
171
|
+
return read_data_file(temp_path, nrows=nrows)
|
|
172
|
+
finally:
|
|
173
|
+
try:
|
|
174
|
+
temp_path.unlink(missing_ok=True)
|
|
175
|
+
except Exception:
|
|
176
|
+
pass
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def load_kaggle_dataset(dataset_url: str, cache_dir: str | os.PathLike[str] = "~/.cache/torchriver", nrows: Optional[int] = None) -> pd.DataFrame:
|
|
180
|
+
"""Download/read a Kaggle dataset without embedding any credentials.
|
|
181
|
+
|
|
182
|
+
Supported credential paths:
|
|
183
|
+
- ``KAGGLE_USERNAME`` and ``KAGGLE_KEY`` environment variables.
|
|
184
|
+
- Kaggle's standard ``~/.kaggle/kaggle.json``.
|
|
185
|
+
- Interactive prompts through ``opendatasets`` when available.
|
|
186
|
+
"""
|
|
187
|
+
if not dataset_url:
|
|
188
|
+
raise ValueError("dataset_url is required for Kaggle loading")
|
|
189
|
+
|
|
190
|
+
cache_dir = Path(cache_dir).expanduser()
|
|
191
|
+
cache_dir.mkdir(parents=True, exist_ok=True)
|
|
192
|
+
slug = dataset_url.rstrip("/").split("/")[-1]
|
|
193
|
+
dataset_folder = cache_dir / slug
|
|
194
|
+
|
|
195
|
+
if not dataset_folder.exists() or not any(dataset_folder.rglob("*")):
|
|
196
|
+
try:
|
|
197
|
+
import opendatasets as od # type: ignore
|
|
198
|
+
except ImportError as exc:
|
|
199
|
+
raise ImportError("Install opendatasets to use load_kaggle: pip install opendatasets kaggle") from exc
|
|
200
|
+
od.download(dataset_url, data_dir=str(cache_dir))
|
|
201
|
+
|
|
202
|
+
data_path = find_data_file(dataset_folder if dataset_folder.exists() else cache_dir)
|
|
203
|
+
return read_data_file(data_path, nrows=nrows)
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def make_synthetic_data(
|
|
207
|
+
days: int = 600,
|
|
208
|
+
station_codes: Optional[Sequence[str]] = None,
|
|
209
|
+
station_names: Optional[Dict[str, str]] = None,
|
|
210
|
+
column_names: Optional[Dict[str, str]] = None,
|
|
211
|
+
seed: int = 42,
|
|
212
|
+
) -> pd.DataFrame:
|
|
213
|
+
"""Generate station-aware synthetic river data with realistic seasonality."""
|
|
214
|
+
rng = np.random.default_rng(seed)
|
|
215
|
+
station_codes = list(station_codes or ["R001", "R002", "R003"])
|
|
216
|
+
station_names = station_names or {code: f"River {i + 1}" for i, code in enumerate(station_codes)}
|
|
217
|
+
dates = pd.date_range(end=pd.Timestamp.today().normalize(), periods=days, freq="D")
|
|
218
|
+
frames: List[pd.DataFrame] = []
|
|
219
|
+
|
|
220
|
+
for i, code in enumerate(station_codes):
|
|
221
|
+
phase = rng.uniform(0, 2 * np.pi)
|
|
222
|
+
base = rng.uniform(70, 210)
|
|
223
|
+
amplitude = rng.uniform(20, 80)
|
|
224
|
+
trend = np.linspace(rng.uniform(-5, 5), rng.uniform(-10, 10), days)
|
|
225
|
+
day_index = np.arange(days)
|
|
226
|
+
season = np.sin(2 * np.pi * day_index / 365.25 + phase)
|
|
227
|
+
precipitation = np.maximum(0, rng.gamma(shape=1.5 + i * 0.15, scale=3.0, size=days) * (1 + 0.4 * season))
|
|
228
|
+
temperature = 12 + 12 * np.sin(2 * np.pi * (day_index - 80) / 365.25 + phase / 3) + rng.normal(0, 2, days)
|
|
229
|
+
ndwi = np.clip(0.2 + 0.35 * season + rng.normal(0, 0.12, days), -1, 1)
|
|
230
|
+
discharge = base + amplitude * season + 1.8 * precipitation - 0.8 * np.maximum(temperature - 25, 0) + trend + rng.normal(0, 8, days)
|
|
231
|
+
discharge = np.maximum(discharge, 1.0)
|
|
232
|
+
q10 = float(np.quantile(discharge, 0.10))
|
|
233
|
+
sdi_30 = (pd.Series(discharge).rolling(30, min_periods=1).mean() - np.mean(discharge)) / (np.std(discharge) + 1e-8)
|
|
234
|
+
target = (discharge < q10).astype(int)
|
|
235
|
+
frames.append(
|
|
236
|
+
pd.DataFrame(
|
|
237
|
+
{
|
|
238
|
+
"date": dates,
|
|
239
|
+
"station_code": str(code),
|
|
240
|
+
"station": station_names.get(str(code), str(code)),
|
|
241
|
+
"discharge_m3s": discharge,
|
|
242
|
+
"low_flow_q10_threshold_m3s": q10,
|
|
243
|
+
"sdi_30": sdi_30,
|
|
244
|
+
"precipitation_mm": precipitation,
|
|
245
|
+
"temperature_c": temperature,
|
|
246
|
+
"ndwi": ndwi,
|
|
247
|
+
"target_lowflow": target,
|
|
248
|
+
}
|
|
249
|
+
)
|
|
250
|
+
)
|
|
251
|
+
df = pd.concat(frames, ignore_index=True)
|
|
252
|
+
if column_names:
|
|
253
|
+
public_to_internal = {
|
|
254
|
+
"date": "date",
|
|
255
|
+
"station": "station_code",
|
|
256
|
+
"station_name": "station",
|
|
257
|
+
"discharge": "discharge_m3s",
|
|
258
|
+
"low_flow_threshold": "low_flow_q10_threshold_m3s",
|
|
259
|
+
"target": "target_lowflow",
|
|
260
|
+
}
|
|
261
|
+
rename = {internal: column_names[key] for key, internal in public_to_internal.items() if key in column_names}
|
|
262
|
+
df = df.rename(columns=rename)
|
|
263
|
+
return df
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
def clean_river_dataframe(
|
|
267
|
+
df: pd.DataFrame,
|
|
268
|
+
mapping: Optional[Dict[str, str]] = None,
|
|
269
|
+
station_names: Optional[Dict[str, str]] = None,
|
|
270
|
+
) -> Tuple[pd.DataFrame, Dict[str, str], Dict[str, str]]:
|
|
271
|
+
"""Map and clean river data into TorchRiver's internal schema."""
|
|
272
|
+
if df is None or len(df) == 0:
|
|
273
|
+
raise ValueError("Input dataframe is empty")
|
|
274
|
+
out, actual_mapping = apply_mapping(df, mapping)
|
|
275
|
+
|
|
276
|
+
if "date" not in out.columns:
|
|
277
|
+
raise ValueError("No date column found. Provide mapping={'date': '<your_date_column>'}.")
|
|
278
|
+
if "station_code" not in out.columns:
|
|
279
|
+
# Single-station fallback is useful for custom files, but transparent.
|
|
280
|
+
out["station_code"] = "RIVER_001"
|
|
281
|
+
if "station" not in out.columns:
|
|
282
|
+
out["station"] = out["station_code"].astype(str)
|
|
283
|
+
if "discharge_m3s" not in out.columns:
|
|
284
|
+
raise ValueError("No discharge column found. Provide mapping={'discharge': '<your_flow_column>'}.")
|
|
285
|
+
|
|
286
|
+
out["date"] = pd.to_datetime(out["date"], errors="coerce")
|
|
287
|
+
out = out.dropna(subset=["date"]).copy()
|
|
288
|
+
out["station_code"] = out["station_code"].astype(str)
|
|
289
|
+
out["station"] = out["station"].astype(str)
|
|
290
|
+
|
|
291
|
+
# Preserve optional original naming for academic traceability.
|
|
292
|
+
if "station_original" not in out.columns:
|
|
293
|
+
out["station_original"] = out["station"].copy()
|
|
294
|
+
|
|
295
|
+
# Numeric conversion except core text/date fields.
|
|
296
|
+
non_numeric = {"date", "station_code", "station", "station_original", "river"}
|
|
297
|
+
for col in out.columns:
|
|
298
|
+
if col not in non_numeric:
|
|
299
|
+
out[col] = pd.to_numeric(out[col], errors="coerce")
|
|
300
|
+
out = out.replace([np.inf, -np.inf], np.nan)
|
|
301
|
+
|
|
302
|
+
if "low_flow_q10_threshold_m3s" not in out.columns:
|
|
303
|
+
out["low_flow_q10_threshold_m3s"] = out.groupby("station_code")["discharge_m3s"].transform(lambda s: s.quantile(0.10))
|
|
304
|
+
if "target_lowflow" not in out.columns:
|
|
305
|
+
out["target_lowflow"] = (out["discharge_m3s"] < out["low_flow_q10_threshold_m3s"]).astype(int)
|
|
306
|
+
else:
|
|
307
|
+
out["target_lowflow"] = pd.to_numeric(out["target_lowflow"], errors="coerce")
|
|
308
|
+
out["target_lowflow"] = out["target_lowflow"].fillna(0).astype(int).clip(0, 1)
|
|
309
|
+
|
|
310
|
+
out = out.sort_values(["station_code", "date"]).reset_index(drop=True)
|
|
311
|
+
|
|
312
|
+
# Fill numeric missing values inside each station first to prevent leakage.
|
|
313
|
+
numeric_cols = [c for c in out.columns if c not in non_numeric and pd.api.types.is_numeric_dtype(out[c])]
|
|
314
|
+
if numeric_cols:
|
|
315
|
+
out[numeric_cols] = out.groupby("station_code", group_keys=False)[numeric_cols].apply(lambda g: g.ffill().bfill())
|
|
316
|
+
out[numeric_cols] = out[numeric_cols].fillna(out[numeric_cols].median(numeric_only=True)).fillna(0)
|
|
317
|
+
|
|
318
|
+
out = out.replace([np.inf, -np.inf], np.nan)
|
|
319
|
+
if out[numeric_cols].isna().any().any():
|
|
320
|
+
out[numeric_cols] = out[numeric_cols].fillna(0)
|
|
321
|
+
|
|
322
|
+
station_name_map = {str(k): str(v) for k, v in (station_names or {}).items()}
|
|
323
|
+
observed_names = out.groupby("station_code")["station"].first().astype(str).to_dict()
|
|
324
|
+
observed_names.update(station_name_map)
|
|
325
|
+
out["station"] = out["station_code"].map(observed_names).fillna(out["station"].astype(str))
|
|
326
|
+
|
|
327
|
+
return out.reset_index(drop=True), actual_mapping, observed_names
|
|
328
|
+
|
|
329
|
+
|
|
330
|
+
def choose_feature_columns(df: pd.DataFrame, user_features: Optional[Sequence[str]] = None) -> List[str]:
|
|
331
|
+
"""Choose training features like the source notebook: numeric minus excluded columns."""
|
|
332
|
+
if user_features is not None:
|
|
333
|
+
missing = [c for c in user_features if c not in df.columns]
|
|
334
|
+
if missing:
|
|
335
|
+
raise ValueError(f"Requested feature columns not found: {missing}")
|
|
336
|
+
features = list(user_features)
|
|
337
|
+
else:
|
|
338
|
+
features = [c for c in df.columns if c not in EXCLUDED_COLS and pd.api.types.is_numeric_dtype(df[c])]
|
|
339
|
+
if not features:
|
|
340
|
+
raise ValueError("No numeric feature columns are available after exclusions")
|
|
341
|
+
return features
|
|
342
|
+
|
|
343
|
+
|
|
344
|
+
def validate_for_training(df: pd.DataFrame, feature_cols: Sequence[str], lookback: int, horizons: Sequence[int], min_windows: int = 5) -> None:
|
|
345
|
+
if len(df) < lookback + min(horizons):
|
|
346
|
+
raise ValueError(f"Not enough rows for lookback={lookback} and horizons={list(horizons)}")
|
|
347
|
+
for required in ["station_code", "date", "discharge_m3s", "target_lowflow"]:
|
|
348
|
+
if required not in df.columns:
|
|
349
|
+
raise ValueError(f"Required column missing after cleaning: {required}")
|
|
350
|
+
y = df["target_lowflow"].dropna().astype(int)
|
|
351
|
+
if y.nunique() < 2:
|
|
352
|
+
raise ValueError(
|
|
353
|
+
"target_lowflow contains only one class. Check target distribution, threshold, or use more representative data."
|
|
354
|
+
)
|
|
355
|
+
feats = df[list(feature_cols)].replace([np.inf, -np.inf], np.nan)
|
|
356
|
+
if feats.isna().all().all():
|
|
357
|
+
raise ValueError("All feature values are NaN")
|
|
358
|
+
variances = feats.var(numeric_only=True).fillna(0)
|
|
359
|
+
if float(variances.sum()) <= 1e-12:
|
|
360
|
+
raise ValueError("All selected features are constant. Training would produce unreliable or constant predictions.")
|
|
361
|
+
|
|
362
|
+
|
|
363
|
+
def fit_transform_features(df: pd.DataFrame, feature_cols: Sequence[str], scaler: Optional[StandardScaler] = None) -> Tuple[pd.DataFrame, StandardScaler]:
|
|
364
|
+
"""Fit or apply a StandardScaler to feature columns only."""
|
|
365
|
+
out = df.copy()
|
|
366
|
+
if scaler is None:
|
|
367
|
+
scaler = StandardScaler()
|
|
368
|
+
out[list(feature_cols)] = scaler.fit_transform(out[list(feature_cols)])
|
|
369
|
+
else:
|
|
370
|
+
out[list(feature_cols)] = scaler.transform(out[list(feature_cols)])
|
|
371
|
+
return out, scaler
|
|
372
|
+
|
|
373
|
+
|
|
374
|
+
def make_windows(
|
|
375
|
+
df: pd.DataFrame,
|
|
376
|
+
feature_cols: Sequence[str],
|
|
377
|
+
lookback: int = 30,
|
|
378
|
+
horizon: int = 7,
|
|
379
|
+
window_mode: str = "per_station",
|
|
380
|
+
) -> Tuple[np.ndarray, np.ndarray]:
|
|
381
|
+
"""Create sliding windows.
|
|
382
|
+
|
|
383
|
+
``per_station`` prevents mixing rows between stations. ``legacy_global``
|
|
384
|
+
reproduces the original notebook's global row-order behavior.
|
|
385
|
+
"""
|
|
386
|
+
if lookback <= 0 or horizon <= 0:
|
|
387
|
+
raise ValueError("lookback and horizon must be positive integers")
|
|
388
|
+
mode = window_mode.lower().strip()
|
|
389
|
+
Xs: List[np.ndarray] = []
|
|
390
|
+
ys: List[float] = []
|
|
391
|
+
|
|
392
|
+
def append_from_frame(frame: pd.DataFrame) -> None:
|
|
393
|
+
X = frame[list(feature_cols)].to_numpy(dtype=np.float32)
|
|
394
|
+
y = frame["target_lowflow"].to_numpy(dtype=np.float32)
|
|
395
|
+
n = len(frame) - lookback - horizon + 1
|
|
396
|
+
for idx in range(max(0, n)):
|
|
397
|
+
Xs.append(X[idx : idx + lookback])
|
|
398
|
+
ys.append(float(y[idx + lookback + horizon - 1]))
|
|
399
|
+
|
|
400
|
+
if mode == "per_station":
|
|
401
|
+
for _, group in df.sort_values(["station_code", "date"]).groupby("station_code", sort=False):
|
|
402
|
+
append_from_frame(group)
|
|
403
|
+
elif mode == "legacy_global":
|
|
404
|
+
append_from_frame(df.sort_values(["station_code", "date"]))
|
|
405
|
+
else:
|
|
406
|
+
raise ValueError("window_mode must be 'per_station' or 'legacy_global'")
|
|
407
|
+
|
|
408
|
+
if not Xs:
|
|
409
|
+
return np.empty((0, lookback, len(feature_cols)), dtype=np.float32), np.empty((0,), dtype=np.float32)
|
|
410
|
+
return np.stack(Xs).astype(np.float32), np.asarray(ys, dtype=np.float32)
|