jtopmodel 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- jtopmodel/__init__.py +198 -0
- jtopmodel/calibration/__init__.py +1 -0
- jtopmodel/calibration/optimizer.py +259 -0
- jtopmodel/calibration/parameter_manager.py +202 -0
- jtopmodel/calibration/worker.py +415 -0
- jtopmodel/config.py +200 -0
- jtopmodel/extractor.py +133 -0
- jtopmodel/losses.py +209 -0
- jtopmodel/model.py +493 -0
- jtopmodel/parameters.py +261 -0
- jtopmodel/postprocessor.py +59 -0
- jtopmodel/preprocessor.py +277 -0
- jtopmodel/runner.py +352 -0
- jtopmodel-0.1.0.dist-info/METADATA +707 -0
- jtopmodel-0.1.0.dist-info/RECORD +18 -0
- jtopmodel-0.1.0.dist-info/WHEEL +4 -0
- jtopmodel-0.1.0.dist-info/entry_points.txt +2 -0
- jtopmodel-0.1.0.dist-info/licenses/LICENSE +674 -0
jtopmodel/__init__.py
ADDED
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
# SPDX-License-Identifier: GPL-3.0-or-later
|
|
2
|
+
# Copyright (C) 2024-2026 SYMFLUENCE Team <dev@symfluence.org>
|
|
3
|
+
|
|
4
|
+
"""
|
|
5
|
+
TOPMODEL (Beven & Kirkby 1979) -- Standalone Plugin Package.
|
|
6
|
+
|
|
7
|
+
A native Python/JAX implementation of TOPMODEL, enabling:
|
|
8
|
+
- Automatic differentiation for gradient-based calibration
|
|
9
|
+
- JIT compilation for fast execution
|
|
10
|
+
- DDS and evolutionary calibration integration
|
|
11
|
+
|
|
12
|
+
Algorithms:
|
|
13
|
+
- Degree-day snow module
|
|
14
|
+
- Exponential transmissivity baseflow (Beven & Kirkby 1979)
|
|
15
|
+
- Saturation-excess overland flow with topographic index distribution
|
|
16
|
+
- Linear reservoir channel routing
|
|
17
|
+
|
|
18
|
+
Components:
|
|
19
|
+
- TopmodelPreProcessor: Prepares forcing data (P, T, PET)
|
|
20
|
+
- TopmodelRunner: Executes model simulations
|
|
21
|
+
- TopmodelPostprocessor: Extracts streamflow results
|
|
22
|
+
- TopmodelWorker: Handles calibration
|
|
23
|
+
|
|
24
|
+
References:
|
|
25
|
+
Beven, K.J. & Kirkby, M.J. (1979). A physically based, variable
|
|
26
|
+
contributing area model of basin hydrology. Hydrological Sciences
|
|
27
|
+
Bulletin, 24(1), 43-69.
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
from typing import TYPE_CHECKING
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
# Lazy import mapping: attribute name -> (module, attribute)
|
|
34
|
+
_LAZY_IMPORTS = {
|
|
35
|
+
# Configuration
|
|
36
|
+
'TOPMODELConfig': ('.config', 'TOPMODELConfig'),
|
|
37
|
+
'TopmodelConfigAdapter': ('.config', 'TopmodelConfigAdapter'),
|
|
38
|
+
|
|
39
|
+
# Main components
|
|
40
|
+
'TopmodelPreProcessor': ('.preprocessor', 'TopmodelPreProcessor'),
|
|
41
|
+
'TopmodelRunner': ('.runner', 'TopmodelRunner'),
|
|
42
|
+
'TopmodelPostprocessor': ('.postprocessor', 'TopmodelPostprocessor'),
|
|
43
|
+
'TopmodelResultExtractor': ('.extractor', 'TopmodelResultExtractor'),
|
|
44
|
+
|
|
45
|
+
# Parameters
|
|
46
|
+
'PARAM_BOUNDS': ('.parameters', 'PARAM_BOUNDS'),
|
|
47
|
+
'DEFAULT_PARAMS': ('.parameters', 'DEFAULT_PARAMS'),
|
|
48
|
+
'TopmodelParameters': ('.parameters', 'TopmodelParameters'),
|
|
49
|
+
'TopmodelState': ('.parameters', 'TopmodelState'),
|
|
50
|
+
'create_params_from_dict': ('.parameters', 'create_params_from_dict'),
|
|
51
|
+
'create_initial_state': ('.parameters', 'create_initial_state'),
|
|
52
|
+
'generate_ti_distribution': ('.parameters', 'generate_ti_distribution'),
|
|
53
|
+
|
|
54
|
+
# Core model
|
|
55
|
+
'simulate': ('.model', 'simulate'),
|
|
56
|
+
'simulate_jax': ('.model', 'simulate_jax'),
|
|
57
|
+
'simulate_numpy': ('.model', 'simulate_numpy'),
|
|
58
|
+
'snow_step': ('.model', 'snow_step'),
|
|
59
|
+
'topmodel_step': ('.model', 'topmodel_step'),
|
|
60
|
+
'route_step': ('.model', 'route_step'),
|
|
61
|
+
'step': ('.model', 'step'),
|
|
62
|
+
'HAS_JAX': ('.model', 'HAS_JAX'),
|
|
63
|
+
|
|
64
|
+
# Loss functions (for gradient-based calibration)
|
|
65
|
+
'nse_loss': ('.losses', 'nse_loss'),
|
|
66
|
+
'kge_loss': ('.losses', 'kge_loss'),
|
|
67
|
+
'get_nse_gradient_fn': ('.losses', 'get_nse_gradient_fn'),
|
|
68
|
+
'get_kge_gradient_fn': ('.losses', 'get_kge_gradient_fn'),
|
|
69
|
+
|
|
70
|
+
# Calibration
|
|
71
|
+
'TopmodelWorker': ('.calibration.worker', 'TopmodelWorker'),
|
|
72
|
+
'TopmodelParameterManager': ('.calibration.parameter_manager', 'TopmodelParameterManager'),
|
|
73
|
+
'get_topmodel_calibration_bounds': ('.calibration.parameter_manager', 'get_topmodel_calibration_bounds'),
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def __getattr__(name: str):
|
|
78
|
+
"""Lazy import handler for TOPMODEL module components."""
|
|
79
|
+
if name in _LAZY_IMPORTS:
|
|
80
|
+
module_path, attr_name = _LAZY_IMPORTS[name]
|
|
81
|
+
from importlib import import_module
|
|
82
|
+
module = import_module(module_path, package=__name__)
|
|
83
|
+
return getattr(module, attr_name)
|
|
84
|
+
|
|
85
|
+
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def __dir__():
|
|
89
|
+
"""Return available attributes for tab completion."""
|
|
90
|
+
return list(_LAZY_IMPORTS.keys()) + ['register']
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def register() -> None:
|
|
94
|
+
"""Register TOPMODEL components with symfluence plugin registry."""
|
|
95
|
+
from symfluence.core.registry import model_manifest
|
|
96
|
+
from .calibration.optimizer import TopmodelModelOptimizer
|
|
97
|
+
from .calibration.parameter_manager import TopmodelParameterManager
|
|
98
|
+
from .calibration.worker import TopmodelWorker
|
|
99
|
+
from .config import TopmodelConfigAdapter
|
|
100
|
+
from .extractor import TopmodelResultExtractor
|
|
101
|
+
from .postprocessor import TopmodelPostprocessor
|
|
102
|
+
from .preprocessor import TopmodelPreProcessor
|
|
103
|
+
from .runner import TopmodelRunner
|
|
104
|
+
|
|
105
|
+
model_manifest(
|
|
106
|
+
"TOPMODEL",
|
|
107
|
+
preprocessor=TopmodelPreProcessor,
|
|
108
|
+
runner=TopmodelRunner,
|
|
109
|
+
runner_method='run_topmodel',
|
|
110
|
+
postprocessor=TopmodelPostprocessor,
|
|
111
|
+
config_adapter=TopmodelConfigAdapter,
|
|
112
|
+
result_extractor=TopmodelResultExtractor,
|
|
113
|
+
optimizer=TopmodelModelOptimizer,
|
|
114
|
+
worker=TopmodelWorker,
|
|
115
|
+
parameter_manager=TopmodelParameterManager,
|
|
116
|
+
)
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
# Type hints for IDE support
|
|
120
|
+
if TYPE_CHECKING:
|
|
121
|
+
from .calibration.parameter_manager import TopmodelParameterManager, get_topmodel_calibration_bounds
|
|
122
|
+
from .calibration.worker import TopmodelWorker
|
|
123
|
+
from .config import TOPMODELConfig, TopmodelConfigAdapter
|
|
124
|
+
from .extractor import TopmodelResultExtractor
|
|
125
|
+
from .losses import (
|
|
126
|
+
get_kge_gradient_fn,
|
|
127
|
+
get_nse_gradient_fn,
|
|
128
|
+
kge_loss,
|
|
129
|
+
nse_loss,
|
|
130
|
+
)
|
|
131
|
+
from .model import (
|
|
132
|
+
HAS_JAX,
|
|
133
|
+
route_step,
|
|
134
|
+
simulate,
|
|
135
|
+
simulate_jax,
|
|
136
|
+
simulate_numpy,
|
|
137
|
+
snow_step,
|
|
138
|
+
step,
|
|
139
|
+
topmodel_step,
|
|
140
|
+
)
|
|
141
|
+
from .parameters import (
|
|
142
|
+
DEFAULT_PARAMS,
|
|
143
|
+
PARAM_BOUNDS,
|
|
144
|
+
TopmodelParameters,
|
|
145
|
+
TopmodelState,
|
|
146
|
+
create_initial_state,
|
|
147
|
+
create_params_from_dict,
|
|
148
|
+
generate_ti_distribution,
|
|
149
|
+
)
|
|
150
|
+
from .postprocessor import TopmodelPostprocessor
|
|
151
|
+
from .preprocessor import TopmodelPreProcessor
|
|
152
|
+
from .runner import TopmodelRunner
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
__all__ = [
|
|
156
|
+
# Main components
|
|
157
|
+
'TopmodelPreProcessor',
|
|
158
|
+
'TopmodelRunner',
|
|
159
|
+
'TopmodelPostprocessor',
|
|
160
|
+
'TopmodelResultExtractor',
|
|
161
|
+
|
|
162
|
+
# Configuration
|
|
163
|
+
'TOPMODELConfig',
|
|
164
|
+
'TopmodelConfigAdapter',
|
|
165
|
+
|
|
166
|
+
# Parameters
|
|
167
|
+
'PARAM_BOUNDS',
|
|
168
|
+
'DEFAULT_PARAMS',
|
|
169
|
+
'TopmodelParameters',
|
|
170
|
+
'TopmodelState',
|
|
171
|
+
'create_params_from_dict',
|
|
172
|
+
'create_initial_state',
|
|
173
|
+
'generate_ti_distribution',
|
|
174
|
+
|
|
175
|
+
# Core model
|
|
176
|
+
'simulate',
|
|
177
|
+
'simulate_jax',
|
|
178
|
+
'simulate_numpy',
|
|
179
|
+
'snow_step',
|
|
180
|
+
'topmodel_step',
|
|
181
|
+
'route_step',
|
|
182
|
+
'step',
|
|
183
|
+
'HAS_JAX',
|
|
184
|
+
|
|
185
|
+
# Loss functions
|
|
186
|
+
'nse_loss',
|
|
187
|
+
'kge_loss',
|
|
188
|
+
'get_nse_gradient_fn',
|
|
189
|
+
'get_kge_gradient_fn',
|
|
190
|
+
|
|
191
|
+
# Calibration
|
|
192
|
+
'TopmodelWorker',
|
|
193
|
+
'TopmodelParameterManager',
|
|
194
|
+
'get_topmodel_calibration_bounds',
|
|
195
|
+
|
|
196
|
+
# Plugin registration
|
|
197
|
+
'register',
|
|
198
|
+
]
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
# SPDX-License-Identifier: GPL-3.0-or-later
|
|
@@ -0,0 +1,259 @@
|
|
|
1
|
+
# SPDX-License-Identifier: GPL-3.0-or-later
|
|
2
|
+
# Copyright (C) 2024-2026 SYMFLUENCE Team <dev@symfluence.org>
|
|
3
|
+
|
|
4
|
+
"""
|
|
5
|
+
TOPMODEL Optimizer.
|
|
6
|
+
|
|
7
|
+
TOPMODEL-specific optimizer inheriting from BaseModelOptimizer.
|
|
8
|
+
Supports DDS and other iterative optimization algorithms.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
import logging
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
from typing import Any, Dict, Optional
|
|
14
|
+
|
|
15
|
+
import numpy as np
|
|
16
|
+
import pandas as pd
|
|
17
|
+
|
|
18
|
+
from symfluence.evaluation.metrics import calculate_all_metrics
|
|
19
|
+
from symfluence.optimization.optimizers.base_model_optimizer import BaseModelOptimizer
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class TopmodelModelOptimizer(BaseModelOptimizer):
|
|
23
|
+
"""
|
|
24
|
+
TOPMODEL-specific optimizer using the unified BaseModelOptimizer framework.
|
|
25
|
+
|
|
26
|
+
Supports:
|
|
27
|
+
- Standard iterative optimization (DDS, PSO, SCE-UA, DE)
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
def __init__(
|
|
31
|
+
self,
|
|
32
|
+
config: Dict[str, Any],
|
|
33
|
+
logger: logging.Logger,
|
|
34
|
+
optimization_settings_dir: Optional[Path] = None,
|
|
35
|
+
reporting_manager: Optional[Any] = None
|
|
36
|
+
):
|
|
37
|
+
self.experiment_id = config.get('EXPERIMENT_ID')
|
|
38
|
+
self.data_dir = Path(config.get('SYMFLUENCE_DATA_DIR'))
|
|
39
|
+
self.domain_name = config.get('DOMAIN_NAME')
|
|
40
|
+
self.project_dir = self.data_dir / f"domain_{self.domain_name}"
|
|
41
|
+
|
|
42
|
+
self.topmodel_setup_dir = self.project_dir / 'settings' / 'TOPMODEL'
|
|
43
|
+
|
|
44
|
+
super().__init__(config, logger, optimization_settings_dir, reporting_manager=reporting_manager)
|
|
45
|
+
|
|
46
|
+
self.logger.debug("TopmodelModelOptimizer initialized")
|
|
47
|
+
|
|
48
|
+
def _get_model_name(self) -> str:
|
|
49
|
+
"""Return model name."""
|
|
50
|
+
return 'TOPMODEL'
|
|
51
|
+
|
|
52
|
+
def _get_final_file_manager_path(self) -> Path:
|
|
53
|
+
"""Get path to TOPMODEL configuration (placeholder for in-memory model)."""
|
|
54
|
+
return self.topmodel_setup_dir / 'topmodel_config.txt'
|
|
55
|
+
|
|
56
|
+
def _create_parameter_manager(self):
|
|
57
|
+
"""Create TOPMODEL parameter manager."""
|
|
58
|
+
from jtopmodel.calibration.parameter_manager import TopmodelParameterManager
|
|
59
|
+
return TopmodelParameterManager(
|
|
60
|
+
self.config,
|
|
61
|
+
self.logger,
|
|
62
|
+
self.topmodel_setup_dir
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
def _run_model_for_final_evaluation(self, output_dir: Path) -> bool:
|
|
66
|
+
"""Run TOPMODEL for final evaluation using best parameters."""
|
|
67
|
+
best_result = self.get_best_result()
|
|
68
|
+
best_params = best_result.get('params')
|
|
69
|
+
|
|
70
|
+
if not best_params:
|
|
71
|
+
self.logger.warning("No best parameters found for final evaluation")
|
|
72
|
+
return False
|
|
73
|
+
|
|
74
|
+
self.worker.apply_parameters(best_params, self.topmodel_setup_dir)
|
|
75
|
+
|
|
76
|
+
return self.worker.run_model(
|
|
77
|
+
self.config,
|
|
78
|
+
self.topmodel_setup_dir,
|
|
79
|
+
output_dir,
|
|
80
|
+
save_output=True
|
|
81
|
+
)
|
|
82
|
+
|
|
83
|
+
def run_final_evaluation(self, best_params: Dict[str, float]) -> Optional[Dict[str, Any]]:
|
|
84
|
+
"""Run final evaluation with consistent warmup handling for TOPMODEL.
|
|
85
|
+
|
|
86
|
+
Calculates separate metrics for calibration and evaluation periods,
|
|
87
|
+
matching the in-memory warmup handling used during optimization.
|
|
88
|
+
"""
|
|
89
|
+
self.logger.info("=" * 60)
|
|
90
|
+
self.logger.info("RUNNING FINAL EVALUATION")
|
|
91
|
+
self.logger.info("=" * 60)
|
|
92
|
+
|
|
93
|
+
try:
|
|
94
|
+
if not self.worker._initialized:
|
|
95
|
+
if not self.worker.initialize():
|
|
96
|
+
self.logger.error("Failed to initialize TOPMODEL worker for final evaluation")
|
|
97
|
+
return None
|
|
98
|
+
|
|
99
|
+
if not self.worker.apply_parameters(best_params, self.topmodel_setup_dir):
|
|
100
|
+
self.logger.error("Failed to apply best parameters for final evaluation")
|
|
101
|
+
return None
|
|
102
|
+
|
|
103
|
+
final_output_dir = self.results_dir / 'final_evaluation'
|
|
104
|
+
final_output_dir.mkdir(parents=True, exist_ok=True)
|
|
105
|
+
|
|
106
|
+
runoff = self.worker._run_simulation(
|
|
107
|
+
self.worker._forcing,
|
|
108
|
+
best_params
|
|
109
|
+
)
|
|
110
|
+
|
|
111
|
+
self.worker.save_output_files(
|
|
112
|
+
runoff[self.worker.warmup_days:],
|
|
113
|
+
final_output_dir,
|
|
114
|
+
self.worker._time_index[self.worker.warmup_days:] if self.worker._time_index is not None else None
|
|
115
|
+
)
|
|
116
|
+
|
|
117
|
+
# Get time index and observations
|
|
118
|
+
time_index = self.worker._time_index
|
|
119
|
+
observations = self.worker._observations
|
|
120
|
+
|
|
121
|
+
if time_index is None or observations is None:
|
|
122
|
+
self.logger.error("Missing time index or observations for metric calculation")
|
|
123
|
+
return None
|
|
124
|
+
|
|
125
|
+
# Parse calibration and evaluation periods
|
|
126
|
+
calib_period = self._parse_period_config('calibration_period', 'CALIBRATION_PERIOD')
|
|
127
|
+
eval_period = self._parse_period_config('evaluation_period', 'EVALUATION_PERIOD')
|
|
128
|
+
|
|
129
|
+
# Calculate metrics for calibration period (with warmup skip)
|
|
130
|
+
calib_metrics = self._calculate_period_metrics_inmemory(
|
|
131
|
+
runoff, observations, time_index,
|
|
132
|
+
calib_period, 'Calib',
|
|
133
|
+
skip_warmup=True
|
|
134
|
+
)
|
|
135
|
+
|
|
136
|
+
# Calculate metrics for evaluation period (no warmup skip needed)
|
|
137
|
+
eval_metrics = {}
|
|
138
|
+
if eval_period[0] and eval_period[1]:
|
|
139
|
+
eval_metrics = self._calculate_period_metrics_inmemory(
|
|
140
|
+
runoff, observations, time_index,
|
|
141
|
+
eval_period, 'Eval',
|
|
142
|
+
skip_warmup=False
|
|
143
|
+
)
|
|
144
|
+
|
|
145
|
+
# Combine all metrics
|
|
146
|
+
all_metrics = {**calib_metrics, **eval_metrics}
|
|
147
|
+
|
|
148
|
+
# Add unprefixed versions for backward compatibility
|
|
149
|
+
for k, v in calib_metrics.items():
|
|
150
|
+
unprefixed = k.replace('Calib_', '')
|
|
151
|
+
if unprefixed not in all_metrics:
|
|
152
|
+
all_metrics[unprefixed] = v
|
|
153
|
+
|
|
154
|
+
# Log results
|
|
155
|
+
self._log_final_evaluation_results(
|
|
156
|
+
{k: v for k, v in calib_metrics.items()},
|
|
157
|
+
{k: v for k, v in eval_metrics.items()}
|
|
158
|
+
)
|
|
159
|
+
|
|
160
|
+
return {
|
|
161
|
+
'final_metrics': all_metrics,
|
|
162
|
+
'calibration_metrics': calib_metrics,
|
|
163
|
+
'evaluation_metrics': eval_metrics,
|
|
164
|
+
'success': True,
|
|
165
|
+
'best_params': best_params,
|
|
166
|
+
'output_dir': str(final_output_dir),
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
except Exception as e: # noqa: BLE001 — calibration resilience
|
|
170
|
+
self.logger.error(f"Final evaluation failed: {e}")
|
|
171
|
+
import traceback
|
|
172
|
+
self.logger.debug(traceback.format_exc())
|
|
173
|
+
return None
|
|
174
|
+
|
|
175
|
+
def _parse_period_config(self, attr_name: str, dict_key: str):
|
|
176
|
+
"""Parse a period configuration string into start/end timestamps."""
|
|
177
|
+
period_str = self._get_config_value(
|
|
178
|
+
lambda: getattr(self.config.domain, attr_name, ''),
|
|
179
|
+
default='',
|
|
180
|
+
dict_key=dict_key
|
|
181
|
+
)
|
|
182
|
+
if not period_str:
|
|
183
|
+
return (None, None)
|
|
184
|
+
|
|
185
|
+
try:
|
|
186
|
+
dates = [d.strip() for d in period_str.split(',')]
|
|
187
|
+
if len(dates) >= 2:
|
|
188
|
+
return (pd.Timestamp(dates[0]), pd.Timestamp(dates[1]))
|
|
189
|
+
except (ValueError, AttributeError) as e:
|
|
190
|
+
self.logger.debug(f"Could not parse period string '{period_str}': {e}")
|
|
191
|
+
return (None, None)
|
|
192
|
+
|
|
193
|
+
def _calculate_period_metrics_inmemory(
|
|
194
|
+
self,
|
|
195
|
+
runoff: np.ndarray,
|
|
196
|
+
observations: np.ndarray,
|
|
197
|
+
time_index: pd.DatetimeIndex,
|
|
198
|
+
period: tuple,
|
|
199
|
+
prefix: str,
|
|
200
|
+
skip_warmup: bool = True
|
|
201
|
+
) -> Dict[str, float]:
|
|
202
|
+
"""Calculate metrics for a specific period using in-memory data.
|
|
203
|
+
|
|
204
|
+
Warmup is skipped from the full simulation start before filtering
|
|
205
|
+
to the target period, matching the calibration behavior.
|
|
206
|
+
"""
|
|
207
|
+
try:
|
|
208
|
+
if skip_warmup and len(runoff) > self.worker.warmup_days:
|
|
209
|
+
runoff = runoff[self.worker.warmup_days:]
|
|
210
|
+
observations = observations[self.worker.warmup_days:]
|
|
211
|
+
time_index = time_index[self.worker.warmup_days:]
|
|
212
|
+
|
|
213
|
+
sim_series = pd.Series(runoff, index=time_index)
|
|
214
|
+
obs_series = pd.Series(observations, index=time_index)
|
|
215
|
+
|
|
216
|
+
if period[0] and period[1]:
|
|
217
|
+
period_mask = (time_index >= period[0]) & (time_index <= period[1])
|
|
218
|
+
sim_period = sim_series[period_mask]
|
|
219
|
+
obs_period = obs_series[period_mask]
|
|
220
|
+
self.logger.info(
|
|
221
|
+
f"{prefix} period: {period[0].date()} to {period[1].date()}, "
|
|
222
|
+
f"{len(sim_period)} points"
|
|
223
|
+
)
|
|
224
|
+
else:
|
|
225
|
+
sim_period = sim_series
|
|
226
|
+
obs_period = obs_series
|
|
227
|
+
|
|
228
|
+
common_idx = sim_period.index.intersection(obs_period.index)
|
|
229
|
+
if len(common_idx) == 0:
|
|
230
|
+
self.logger.warning(f"No common indices for {prefix} period")
|
|
231
|
+
return {}
|
|
232
|
+
|
|
233
|
+
sim_aligned = sim_period.loc[common_idx].values
|
|
234
|
+
obs_aligned = obs_period.loc[common_idx].values
|
|
235
|
+
|
|
236
|
+
valid_mask = ~(np.isnan(sim_aligned) | np.isnan(obs_aligned))
|
|
237
|
+
sim_valid = sim_aligned[valid_mask]
|
|
238
|
+
obs_valid = obs_aligned[valid_mask]
|
|
239
|
+
|
|
240
|
+
if len(sim_valid) < 10:
|
|
241
|
+
self.logger.warning(f"Insufficient valid points for {prefix} metrics: {len(sim_valid)}")
|
|
242
|
+
return {}
|
|
243
|
+
|
|
244
|
+
metrics_result = calculate_all_metrics(
|
|
245
|
+
pd.Series(obs_valid),
|
|
246
|
+
pd.Series(sim_valid)
|
|
247
|
+
)
|
|
248
|
+
|
|
249
|
+
prefixed_metrics = {}
|
|
250
|
+
for key, value in metrics_result.items():
|
|
251
|
+
prefixed_metrics[f"{prefix}_{key}"] = value
|
|
252
|
+
|
|
253
|
+
return prefixed_metrics
|
|
254
|
+
|
|
255
|
+
except Exception as e: # noqa: BLE001 — calibration resilience
|
|
256
|
+
self.logger.error(f"Error calculating {prefix} metrics: {e}")
|
|
257
|
+
import traceback
|
|
258
|
+
self.logger.debug(traceback.format_exc())
|
|
259
|
+
return {}
|
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
# SPDX-License-Identifier: GPL-3.0-or-later
|
|
2
|
+
# Copyright (C) 2024-2026 SYMFLUENCE Team <dev@symfluence.org>
|
|
3
|
+
|
|
4
|
+
"""
|
|
5
|
+
TOPMODEL Parameter Manager.
|
|
6
|
+
|
|
7
|
+
Provides parameter bounds, transformations, and management for TOPMODEL calibration.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
import logging
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from typing import Dict, List, Optional, Tuple
|
|
13
|
+
|
|
14
|
+
import numpy as np
|
|
15
|
+
|
|
16
|
+
from jtopmodel.parameters import DEFAULT_PARAMS, PARAM_BOUNDS
|
|
17
|
+
from symfluence.optimization.core.base_parameter_manager import BaseParameterManager
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class TopmodelParameterManager(BaseParameterManager):
|
|
21
|
+
"""
|
|
22
|
+
Manages TOPMODEL parameters for calibration.
|
|
23
|
+
|
|
24
|
+
Provides:
|
|
25
|
+
- Parameter bounds retrieval
|
|
26
|
+
- Transformation between normalized [0,1] and physical space
|
|
27
|
+
- Default values
|
|
28
|
+
- Parameter validation
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
def __init__(self, config: Dict, logger: logging.Logger, topmodel_settings_dir: Path):
|
|
32
|
+
"""
|
|
33
|
+
Initialize parameter manager.
|
|
34
|
+
|
|
35
|
+
Args:
|
|
36
|
+
config: Configuration dictionary
|
|
37
|
+
logger: Logger instance
|
|
38
|
+
topmodel_settings_dir: Path to TOPMODEL settings directory
|
|
39
|
+
"""
|
|
40
|
+
super().__init__(config, logger, topmodel_settings_dir)
|
|
41
|
+
|
|
42
|
+
self.domain_name = self._get_config_value(lambda: self.config.domain.name, default=None, dict_key='DOMAIN_NAME')
|
|
43
|
+
self.experiment_id = self._get_config_value(lambda: self.config.domain.experiment_id, default=None, dict_key='EXPERIMENT_ID')
|
|
44
|
+
|
|
45
|
+
# Parse TOPMODEL parameters to calibrate from config
|
|
46
|
+
topmodel_params_str = self._get_config_value(lambda: self.config.model.topmodel.params_to_calibrate, default=None, dict_key='TOPMODEL_PARAMS_TO_CALIBRATE') or self._get_config_value(lambda: None, default=None, dict_key='PARAMS_TO_CALIBRATE')
|
|
47
|
+
|
|
48
|
+
# Handle None, empty string, or 'default' as signal to use ALL available parameters
|
|
49
|
+
if topmodel_params_str is None or topmodel_params_str == '' or topmodel_params_str == 'default':
|
|
50
|
+
self.topmodel_params = list(PARAM_BOUNDS.keys())
|
|
51
|
+
logger.debug(f"Using all available TOPMODEL parameters for calibration: {self.topmodel_params}")
|
|
52
|
+
else:
|
|
53
|
+
self.topmodel_params = [p.strip() for p in str(topmodel_params_str).split(',') if p.strip()]
|
|
54
|
+
logger.debug(f"Using user-specified TOPMODEL parameters: {self.topmodel_params}")
|
|
55
|
+
|
|
56
|
+
# Store internal references
|
|
57
|
+
self.all_bounds = PARAM_BOUNDS.copy()
|
|
58
|
+
self.defaults = DEFAULT_PARAMS.copy()
|
|
59
|
+
self.calibration_params = self.topmodel_params
|
|
60
|
+
|
|
61
|
+
# ========================================================================
|
|
62
|
+
# IMPLEMENT ABSTRACT METHODS
|
|
63
|
+
# ========================================================================
|
|
64
|
+
|
|
65
|
+
def _get_parameter_names(self) -> List[str]:
|
|
66
|
+
"""Return TOPMODEL parameter names from config."""
|
|
67
|
+
return self.topmodel_params
|
|
68
|
+
|
|
69
|
+
def _load_parameter_bounds(self) -> Dict[str, Dict[str, float]]:
|
|
70
|
+
"""Return TOPMODEL parameter bounds from central registry."""
|
|
71
|
+
from symfluence.optimization.core.parameter_bounds_registry import get_topmodel_bounds
|
|
72
|
+
return get_topmodel_bounds()
|
|
73
|
+
|
|
74
|
+
def update_model_files(self, params: Dict[str, float]) -> bool:
|
|
75
|
+
"""TOPMODEL runs in-memory; parameters passed directly to model."""
|
|
76
|
+
return True
|
|
77
|
+
|
|
78
|
+
def get_initial_parameters(self) -> Optional[Dict[str, float]]:
|
|
79
|
+
"""Get initial parameter values from config or defaults."""
|
|
80
|
+
initial_params = self._get_config_value(lambda: None, default='default', dict_key='TOPMODEL_INITIAL_PARAMS')
|
|
81
|
+
|
|
82
|
+
if initial_params == 'default':
|
|
83
|
+
self.logger.debug("Using standard TOPMODEL defaults for initial parameters")
|
|
84
|
+
return {p: self.defaults[p] for p in self.topmodel_params}
|
|
85
|
+
|
|
86
|
+
if isinstance(initial_params, str) and initial_params != 'default':
|
|
87
|
+
try:
|
|
88
|
+
param_dict = {}
|
|
89
|
+
for pair in initial_params.split(','):
|
|
90
|
+
if '=' in pair:
|
|
91
|
+
k, v = pair.split('=')
|
|
92
|
+
param_dict[k.strip()] = float(v.strip())
|
|
93
|
+
return param_dict
|
|
94
|
+
except Exception as e: # noqa: BLE001 — calibration resilience
|
|
95
|
+
self.logger.warning(f"Could not parse TOPMODEL_INITIAL_PARAMS: {e}")
|
|
96
|
+
return {p: self.defaults[p] for p in self.topmodel_params}
|
|
97
|
+
|
|
98
|
+
return {p: self.defaults[p] for p in self.topmodel_params}
|
|
99
|
+
|
|
100
|
+
def get_bounds(self, param_name: str) -> Tuple[float, float]:
|
|
101
|
+
"""Get bounds for a single parameter."""
|
|
102
|
+
if param_name not in self.all_bounds:
|
|
103
|
+
raise KeyError(f"Unknown TOPMODEL parameter: {param_name}")
|
|
104
|
+
return self.all_bounds[param_name]
|
|
105
|
+
|
|
106
|
+
def get_calibration_bounds(self) -> Dict[str, Dict[str, float]]:
|
|
107
|
+
"""Get bounds for all calibration parameters."""
|
|
108
|
+
return {
|
|
109
|
+
name: {'min': self.all_bounds[name][0], 'max': self.all_bounds[name][1]}
|
|
110
|
+
for name in self.calibration_params
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
def get_bounds_array(self) -> Tuple[np.ndarray, np.ndarray]:
|
|
114
|
+
"""Get bounds as arrays for optimization algorithms."""
|
|
115
|
+
lower = np.array([self.all_bounds[p][0] for p in self.calibration_params])
|
|
116
|
+
upper = np.array([self.all_bounds[p][1] for p in self.calibration_params])
|
|
117
|
+
return lower, upper
|
|
118
|
+
|
|
119
|
+
def get_default(self, param_name: str) -> float:
|
|
120
|
+
"""Get default value for a parameter."""
|
|
121
|
+
return self.defaults.get(param_name, 0.0)
|
|
122
|
+
|
|
123
|
+
def get_default_vector(self) -> np.ndarray:
|
|
124
|
+
"""Get default values as array for calibration parameters."""
|
|
125
|
+
return np.array([self.defaults[p] for p in self.calibration_params])
|
|
126
|
+
|
|
127
|
+
def normalize(self, params: Dict[str, float]) -> np.ndarray:
|
|
128
|
+
"""Normalize parameters to [0, 1] range."""
|
|
129
|
+
normalized = []
|
|
130
|
+
for name in self.calibration_params:
|
|
131
|
+
value = params.get(name, self.defaults[name])
|
|
132
|
+
low, high = self.all_bounds[name]
|
|
133
|
+
norm_val = (value - low) / (high - low + 1e-10)
|
|
134
|
+
normalized.append(np.clip(norm_val, 0, 1))
|
|
135
|
+
return np.array(normalized)
|
|
136
|
+
|
|
137
|
+
def denormalize(self, values: np.ndarray) -> Dict[str, float]:
|
|
138
|
+
"""Convert normalized [0, 1] values to physical parameter values."""
|
|
139
|
+
params = {}
|
|
140
|
+
for i, name in enumerate(self.calibration_params):
|
|
141
|
+
low, high = self.all_bounds[name]
|
|
142
|
+
params[name] = low + values[i] * (high - low)
|
|
143
|
+
return params
|
|
144
|
+
|
|
145
|
+
def array_to_dict(self, values: np.ndarray) -> Dict[str, float]:
|
|
146
|
+
"""Convert parameter array to dictionary."""
|
|
147
|
+
return dict(zip(self.calibration_params, values))
|
|
148
|
+
|
|
149
|
+
def dict_to_array(self, params: Dict[str, float]) -> np.ndarray:
|
|
150
|
+
"""Convert parameter dictionary to array."""
|
|
151
|
+
return np.array([params.get(p, self.defaults[p]) for p in self.calibration_params])
|
|
152
|
+
|
|
153
|
+
def validate(self, params: Dict[str, float]) -> Tuple[bool, List[str]]:
|
|
154
|
+
"""Validate parameter values are within bounds."""
|
|
155
|
+
violations = []
|
|
156
|
+
for name, value in params.items():
|
|
157
|
+
if name in self.all_bounds:
|
|
158
|
+
low, high = self.all_bounds[name]
|
|
159
|
+
if value < low:
|
|
160
|
+
violations.append(f"{name}={value} < min={low}")
|
|
161
|
+
elif value > high:
|
|
162
|
+
violations.append(f"{name}={value} > max={high}")
|
|
163
|
+
|
|
164
|
+
return len(violations) == 0, violations
|
|
165
|
+
|
|
166
|
+
def clip_to_bounds(self, params: Dict[str, float]) -> Dict[str, float]:
|
|
167
|
+
"""Clip parameter values to their bounds."""
|
|
168
|
+
clipped = {}
|
|
169
|
+
for name, value in params.items():
|
|
170
|
+
if name in self.all_bounds:
|
|
171
|
+
low, high = self.all_bounds[name]
|
|
172
|
+
clipped[name] = np.clip(value, low, high)
|
|
173
|
+
else:
|
|
174
|
+
clipped[name] = value
|
|
175
|
+
return clipped
|
|
176
|
+
|
|
177
|
+
def get_complete_params(self, partial_params: Dict[str, float]) -> Dict[str, float]:
|
|
178
|
+
"""Complete partial parameter dict with defaults."""
|
|
179
|
+
complete = self.defaults.copy()
|
|
180
|
+
complete.update(partial_params)
|
|
181
|
+
return complete
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def get_topmodel_calibration_bounds(
|
|
185
|
+
params_to_calibrate: Optional[List[str]] = None
|
|
186
|
+
) -> Dict[str, Dict[str, float]]:
|
|
187
|
+
"""
|
|
188
|
+
Convenience function to get TOPMODEL calibration bounds.
|
|
189
|
+
|
|
190
|
+
Args:
|
|
191
|
+
params_to_calibrate: List of parameters. If None, uses all.
|
|
192
|
+
|
|
193
|
+
Returns:
|
|
194
|
+
Dict mapping param_name -> {'min': float, 'max': float}
|
|
195
|
+
"""
|
|
196
|
+
if params_to_calibrate is None:
|
|
197
|
+
params_to_calibrate = list(PARAM_BOUNDS.keys())
|
|
198
|
+
|
|
199
|
+
return {
|
|
200
|
+
name: {'min': PARAM_BOUNDS[name][0], 'max': PARAM_BOUNDS[name][1]}
|
|
201
|
+
for name in params_to_calibrate if name in PARAM_BOUNDS
|
|
202
|
+
}
|