ert 17.1.9__py3-none-any.whl → 18.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- _ert/events.py +19 -2
- ert/__main__.py +8 -7
- ert/analysis/_update_commons.py +12 -3
- ert/cli/main.py +6 -3
- ert/cli/monitor.py +7 -0
- ert/config/__init__.py +13 -3
- ert/config/_create_observation_dataframes.py +60 -12
- ert/config/_observations.py +14 -1
- ert/config/_read_summary.py +8 -6
- ert/config/ensemble_config.py +6 -14
- ert/config/ert_config.py +19 -13
- ert/config/{everest_objective_config.py → everest_response.py} +23 -12
- ert/config/ext_param_config.py +133 -1
- ert/config/field.py +12 -8
- ert/config/forward_model_step.py +108 -6
- ert/config/gen_data_config.py +2 -6
- ert/config/gen_kw_config.py +0 -9
- ert/config/known_response_types.py +14 -0
- ert/config/parameter_config.py +0 -17
- ert/config/parsing/config_keywords.py +1 -0
- ert/config/parsing/config_schema.py +12 -0
- ert/config/parsing/config_schema_deprecations.py +11 -0
- ert/config/parsing/config_schema_item.py +1 -1
- ert/config/queue_config.py +4 -4
- ert/config/response_config.py +0 -7
- ert/config/rft_config.py +230 -0
- ert/config/summary_config.py +2 -6
- ert/config/violations.py +0 -0
- ert/config/workflow_fixtures.py +2 -1
- ert/dark_storage/client/__init__.py +2 -2
- ert/dark_storage/client/_session.py +4 -4
- ert/dark_storage/client/client.py +2 -2
- ert/dark_storage/compute/misfits.py +7 -6
- ert/dark_storage/endpoints/compute/misfits.py +2 -2
- ert/dark_storage/endpoints/observations.py +4 -4
- ert/dark_storage/endpoints/responses.py +15 -1
- ert/ensemble_evaluator/__init__.py +8 -1
- ert/ensemble_evaluator/evaluator.py +81 -29
- ert/ensemble_evaluator/event.py +6 -0
- ert/ensemble_evaluator/snapshot.py +3 -1
- ert/ensemble_evaluator/state.py +1 -0
- ert/field_utils/__init__.py +8 -0
- ert/field_utils/field_utils.py +211 -1
- ert/gui/ertwidgets/__init__.py +23 -16
- ert/gui/ertwidgets/analysismoduleedit.py +2 -2
- ert/gui/ertwidgets/checklist.py +1 -1
- ert/gui/ertwidgets/create_experiment_dialog.py +3 -1
- ert/gui/ertwidgets/ensembleselector.py +2 -2
- ert/gui/ertwidgets/models/__init__.py +2 -0
- ert/gui/ertwidgets/models/activerealizationsmodel.py +2 -1
- ert/gui/ertwidgets/models/path_model.py +1 -1
- ert/gui/ertwidgets/models/targetensemblemodel.py +2 -1
- ert/gui/ertwidgets/models/text_model.py +1 -1
- ert/gui/ertwidgets/searchbox.py +13 -4
- ert/gui/{suggestor → ertwidgets/suggestor}/_suggestor_message.py +13 -4
- ert/gui/main.py +14 -7
- ert/gui/main_window.py +1 -2
- ert/gui/simulation/ensemble_experiment_panel.py +1 -1
- ert/gui/simulation/ensemble_information_filter_panel.py +1 -1
- ert/gui/simulation/ensemble_smoother_panel.py +1 -1
- ert/gui/simulation/evaluate_ensemble_panel.py +1 -1
- ert/gui/simulation/experiment_panel.py +1 -1
- ert/gui/simulation/manual_update_panel.py +31 -8
- ert/gui/simulation/multiple_data_assimilation_panel.py +12 -8
- ert/gui/simulation/run_dialog.py +25 -4
- ert/gui/simulation/single_test_run_panel.py +2 -2
- ert/gui/summarypanel.py +1 -1
- ert/gui/tools/load_results/load_results_panel.py +1 -1
- ert/gui/tools/manage_experiments/storage_info_widget.py +7 -7
- ert/gui/tools/manage_experiments/storage_widget.py +1 -2
- ert/gui/tools/plot/plot_api.py +13 -10
- ert/gui/tools/plot/plot_window.py +12 -0
- ert/gui/tools/plot/plottery/plot_config.py +2 -0
- ert/gui/tools/plot/plottery/plot_context.py +14 -0
- ert/gui/tools/plot/plottery/plots/ensemble.py +9 -2
- ert/gui/tools/plot/plottery/plots/statistics.py +59 -19
- ert/mode_definitions.py +2 -0
- ert/plugins/__init__.py +0 -1
- ert/plugins/hook_implementations/workflows/gen_data_rft_export.py +10 -2
- ert/plugins/hook_specifications/__init__.py +0 -2
- ert/plugins/hook_specifications/jobs.py +0 -9
- ert/plugins/plugin_manager.py +2 -33
- ert/resources/shell_scripts/delete_directory.py +2 -2
- ert/run_models/__init__.py +18 -5
- ert/run_models/_create_run_path.py +33 -21
- ert/run_models/ensemble_experiment.py +10 -4
- ert/run_models/ensemble_information_filter.py +8 -1
- ert/run_models/ensemble_smoother.py +9 -3
- ert/run_models/evaluate_ensemble.py +8 -6
- ert/run_models/event.py +7 -3
- ert/run_models/everest_run_model.py +155 -44
- ert/run_models/initial_ensemble_run_model.py +23 -22
- ert/run_models/manual_update.py +4 -2
- ert/run_models/manual_update_enif.py +37 -0
- ert/run_models/model_factory.py +81 -22
- ert/run_models/multiple_data_assimilation.py +21 -10
- ert/run_models/run_model.py +54 -34
- ert/run_models/single_test_run.py +7 -4
- ert/run_models/update_run_model.py +4 -2
- ert/runpaths.py +5 -6
- ert/sample_prior.py +9 -4
- ert/scheduler/driver.py +37 -0
- ert/scheduler/event.py +3 -1
- ert/scheduler/job.py +23 -13
- ert/scheduler/lsf_driver.py +6 -2
- ert/scheduler/openpbs_driver.py +7 -1
- ert/scheduler/scheduler.py +5 -0
- ert/scheduler/slurm_driver.py +6 -2
- ert/services/__init__.py +2 -2
- ert/services/_base_service.py +31 -15
- ert/services/ert_server.py +317 -0
- ert/shared/_doc_utils/ert_jobs.py +1 -4
- ert/shared/storage/connection.py +3 -3
- ert/shared/version.py +3 -3
- ert/storage/local_ensemble.py +25 -5
- ert/storage/local_experiment.py +6 -14
- ert/storage/local_storage.py +36 -30
- ert/storage/migration/to18.py +12 -0
- ert/storage/migration/to8.py +4 -4
- ert/substitutions.py +12 -28
- ert/validation/active_range.py +7 -7
- ert/validation/rangestring.py +16 -16
- {ert-17.1.9.dist-info → ert-18.0.1.dist-info}/METADATA +8 -7
- {ert-17.1.9.dist-info → ert-18.0.1.dist-info}/RECORD +159 -158
- everest/bin/config_branch_script.py +3 -6
- everest/bin/everconfigdump_script.py +1 -9
- everest/bin/everest_script.py +21 -11
- everest/bin/kill_script.py +2 -2
- everest/bin/monitor_script.py +2 -2
- everest/bin/utils.py +6 -3
- everest/config/__init__.py +4 -1
- everest/config/control_config.py +61 -2
- everest/config/control_variable_config.py +2 -1
- everest/config/everest_config.py +38 -16
- everest/config/forward_model_config.py +5 -3
- everest/config/install_data_config.py +7 -5
- everest/config/install_job_config.py +7 -3
- everest/config/install_template_config.py +3 -3
- everest/config/optimization_config.py +19 -6
- everest/config/output_constraint_config.py +8 -2
- everest/config/server_config.py +6 -49
- everest/config/utils.py +25 -105
- everest/config/validation_utils.py +10 -10
- everest/config_file_loader.py +13 -2
- everest/detached/everserver.py +7 -8
- everest/everest_storage.py +6 -10
- everest/gui/everest_client.py +0 -1
- everest/gui/main_window.py +2 -2
- everest/optimizer/everest2ropt.py +59 -32
- everest/optimizer/opt_model_transforms.py +12 -13
- everest/optimizer/utils.py +0 -29
- everest/strings.py +0 -5
- ert/config/everest_constraints_config.py +0 -95
- ert/services/storage_service.py +0 -127
- everest/config/sampler_config.py +0 -103
- everest/simulator/__init__.py +0 -88
- everest/simulator/everest_to_ert.py +0 -51
- /ert/gui/{suggestor → ertwidgets/suggestor}/__init__.py +0 -0
- /ert/gui/{suggestor → ertwidgets/suggestor}/_colors.py +0 -0
- /ert/gui/{suggestor → ertwidgets/suggestor}/suggestor.py +0 -0
- {ert-17.1.9.dist-info → ert-18.0.1.dist-info}/WHEEL +0 -0
- {ert-17.1.9.dist-info → ert-18.0.1.dist-info}/entry_points.txt +0 -0
- {ert-17.1.9.dist-info → ert-18.0.1.dist-info}/licenses/COPYING +0 -0
- {ert-17.1.9.dist-info → ert-18.0.1.dist-info}/top_level.txt +0 -0
ert/run_models/model_factory.py
CHANGED
|
@@ -21,20 +21,28 @@ from ert.mode_definitions import (
|
|
|
21
21
|
ENSEMBLE_SMOOTHER_MODE,
|
|
22
22
|
ES_MDA_MODE,
|
|
23
23
|
EVALUATE_ENSEMBLE_MODE,
|
|
24
|
+
MANUAL_ENIF_UPDATE_MODE,
|
|
24
25
|
MANUAL_UPDATE_MODE,
|
|
25
26
|
TEST_RUN_MODE,
|
|
26
27
|
)
|
|
27
28
|
from ert.validation import ActiveRange
|
|
28
29
|
|
|
29
|
-
from .ensemble_experiment import EnsembleExperiment
|
|
30
|
-
from .ensemble_information_filter import
|
|
31
|
-
|
|
32
|
-
|
|
30
|
+
from .ensemble_experiment import EnsembleExperiment, EnsembleExperimentConfig
|
|
31
|
+
from .ensemble_information_filter import (
|
|
32
|
+
EnsembleInformationFilter,
|
|
33
|
+
EnsembleInformationFilterConfig,
|
|
34
|
+
)
|
|
35
|
+
from .ensemble_smoother import EnsembleSmoother, EnsembleSmootherConfig
|
|
36
|
+
from .evaluate_ensemble import EvaluateEnsemble, EvaluateEnsembleConfig
|
|
33
37
|
from .initial_ensemble_run_model import DictEncodedDataFrame
|
|
34
|
-
from .manual_update import ManualUpdate
|
|
35
|
-
from .
|
|
38
|
+
from .manual_update import ManualUpdate, ManualUpdateConfig
|
|
39
|
+
from .manual_update_enif import ManualUpdateEnIF
|
|
40
|
+
from .multiple_data_assimilation import (
|
|
41
|
+
MultipleDataAssimilation,
|
|
42
|
+
MultipleDataAssimilationConfig,
|
|
43
|
+
)
|
|
36
44
|
from .run_model import RunModel
|
|
37
|
-
from .single_test_run import SingleTestRun
|
|
45
|
+
from .single_test_run import SingleTestRun, SingleTestRunConfig
|
|
38
46
|
|
|
39
47
|
if TYPE_CHECKING:
|
|
40
48
|
import numpy.typing as npt
|
|
@@ -78,6 +86,8 @@ def create_model(
|
|
|
78
86
|
)
|
|
79
87
|
if args.mode == MANUAL_UPDATE_MODE:
|
|
80
88
|
return _setup_manual_update(config, args, update_settings, status_queue)
|
|
89
|
+
if args.mode == MANUAL_ENIF_UPDATE_MODE:
|
|
90
|
+
return _setup_manual_update_enif(config, args, update_settings, status_queue)
|
|
81
91
|
raise NotImplementedError(f"Run type not supported {args.mode}")
|
|
82
92
|
|
|
83
93
|
|
|
@@ -122,7 +132,7 @@ def _setup_single_test_run(
|
|
|
122
132
|
parameter_configs=config.ensemble_config.parameter_configuration,
|
|
123
133
|
)
|
|
124
134
|
|
|
125
|
-
|
|
135
|
+
runmodel_config = SingleTestRunConfig(
|
|
126
136
|
random_seed=config.random_seed,
|
|
127
137
|
runpath_file=config.runpath_file,
|
|
128
138
|
active_realizations=[True],
|
|
@@ -144,6 +154,10 @@ def _setup_single_test_run(
|
|
|
144
154
|
storage_path=config.ens_path,
|
|
145
155
|
queue_config=config.queue_config.create_local_copy(),
|
|
146
156
|
observations=config.observations,
|
|
157
|
+
)
|
|
158
|
+
|
|
159
|
+
return SingleTestRun(
|
|
160
|
+
**runmodel_config.model_dump(),
|
|
147
161
|
status_queue=status_queue,
|
|
148
162
|
)
|
|
149
163
|
|
|
@@ -177,7 +191,7 @@ def _setup_ensemble_experiment(
|
|
|
177
191
|
parameter_configs=config.ensemble_config.parameter_configuration,
|
|
178
192
|
)
|
|
179
193
|
|
|
180
|
-
|
|
194
|
+
runmodel_config = EnsembleExperimentConfig(
|
|
181
195
|
random_seed=config.random_seed,
|
|
182
196
|
runpath_file=config.runpath_file,
|
|
183
197
|
active_realizations=active_realizations,
|
|
@@ -199,6 +213,10 @@ def _setup_ensemble_experiment(
|
|
|
199
213
|
storage_path=config.ens_path,
|
|
200
214
|
queue_config=config.queue_config,
|
|
201
215
|
observations=config.observations,
|
|
216
|
+
)
|
|
217
|
+
|
|
218
|
+
return EnsembleExperiment(
|
|
219
|
+
**runmodel_config.model_dump(),
|
|
202
220
|
status_queue=status_queue,
|
|
203
221
|
)
|
|
204
222
|
|
|
@@ -210,14 +228,13 @@ def _setup_evaluate_ensemble(
|
|
|
210
228
|
) -> EvaluateEnsemble:
|
|
211
229
|
active_realizations = _get_and_validate_active_realizations_list(args, config)
|
|
212
230
|
validate_minimum_realizations(config, active_realizations)
|
|
213
|
-
|
|
231
|
+
runmodel_config = EvaluateEnsembleConfig(
|
|
214
232
|
random_seed=config.random_seed,
|
|
215
233
|
active_realizations=active_realizations,
|
|
216
234
|
ensemble_id=args.ensemble_id,
|
|
217
235
|
minimum_required_realizations=config.analysis_config.minimum_required_realizations,
|
|
218
236
|
storage_path=config.ens_path,
|
|
219
237
|
queue_config=config.queue_config,
|
|
220
|
-
status_queue=status_queue,
|
|
221
238
|
runpath_file=config.runpath_file,
|
|
222
239
|
user_config_file=Path(config.user_config_file),
|
|
223
240
|
env_vars=config.env_vars,
|
|
@@ -228,6 +245,7 @@ def _setup_evaluate_ensemble(
|
|
|
228
245
|
hooked_workflows=config.hooked_workflows,
|
|
229
246
|
log_path=config.analysis_config.log_path,
|
|
230
247
|
)
|
|
248
|
+
return EvaluateEnsemble(**runmodel_config.model_dump(), status_queue=status_queue)
|
|
231
249
|
|
|
232
250
|
|
|
233
251
|
def _get_and_validate_active_realizations_list(
|
|
@@ -267,7 +285,7 @@ def _setup_manual_update(
|
|
|
267
285
|
active_realizations = _realizations(args, config.runpath_config.num_realizations)
|
|
268
286
|
validate_minimum_realizations(config, active_realizations.tolist())
|
|
269
287
|
|
|
270
|
-
|
|
288
|
+
runmodel_config = ManualUpdateConfig(
|
|
271
289
|
random_seed=config.random_seed,
|
|
272
290
|
active_realizations=active_realizations.tolist(),
|
|
273
291
|
ensemble_id=args.ensemble_id,
|
|
@@ -277,7 +295,6 @@ def _setup_manual_update(
|
|
|
277
295
|
queue_config=config.queue_config,
|
|
278
296
|
analysis_settings=config.analysis_config.es_settings,
|
|
279
297
|
update_settings=update_settings,
|
|
280
|
-
status_queue=status_queue,
|
|
281
298
|
runpath_file=config.runpath_file,
|
|
282
299
|
user_config_file=Path(config.user_config_file),
|
|
283
300
|
env_vars=config.env_vars,
|
|
@@ -290,6 +307,44 @@ def _setup_manual_update(
|
|
|
290
307
|
ert_templates=config.ert_templates,
|
|
291
308
|
observations=config.observations,
|
|
292
309
|
)
|
|
310
|
+
return ManualUpdate(**runmodel_config.model_dump(), status_queue=status_queue)
|
|
311
|
+
|
|
312
|
+
|
|
313
|
+
def _setup_manual_update_enif(
|
|
314
|
+
config: ErtConfig,
|
|
315
|
+
args: Namespace,
|
|
316
|
+
update_settings: ObservationSettings,
|
|
317
|
+
status_queue: SimpleQueue[StatusEvents],
|
|
318
|
+
) -> ManualUpdate:
|
|
319
|
+
active_realizations = _realizations(args, config.runpath_config.num_realizations)
|
|
320
|
+
|
|
321
|
+
return ManualUpdateEnIF(
|
|
322
|
+
random_seed=config.random_seed,
|
|
323
|
+
active_realizations=active_realizations.tolist(),
|
|
324
|
+
ensemble_id=args.ensemble_id,
|
|
325
|
+
minimum_required_realizations=config.analysis_config.minimum_required_realizations,
|
|
326
|
+
target_ensemble=args.target_ensemble,
|
|
327
|
+
config=config,
|
|
328
|
+
storage_path=config.ens_path,
|
|
329
|
+
queue_config=config.queue_config,
|
|
330
|
+
analysis_settings=config.analysis_config.es_settings,
|
|
331
|
+
update_settings=update_settings,
|
|
332
|
+
status_queue=status_queue,
|
|
333
|
+
runpath_file=config.runpath_file,
|
|
334
|
+
design_matrix=config.analysis_config.design_matrix,
|
|
335
|
+
parameter_configuration=config.ensemble_config.parameter_configuration,
|
|
336
|
+
response_configuration=config.ensemble_config.response_configuration,
|
|
337
|
+
ert_templates=config.ert_templates,
|
|
338
|
+
user_config_file=Path(config.user_config_file),
|
|
339
|
+
env_vars=config.env_vars,
|
|
340
|
+
env_pr_fm_step=config.env_pr_fm_step,
|
|
341
|
+
runpath_config=config.runpath_config,
|
|
342
|
+
forward_model_steps=config.forward_model_steps,
|
|
343
|
+
substitutions=config.substitutions,
|
|
344
|
+
hooked_workflows=config.hooked_workflows,
|
|
345
|
+
log_path=config.analysis_config.log_path,
|
|
346
|
+
observations=config.observations,
|
|
347
|
+
)
|
|
293
348
|
|
|
294
349
|
|
|
295
350
|
def _setup_ensemble_smoother(
|
|
@@ -300,7 +355,7 @@ def _setup_ensemble_smoother(
|
|
|
300
355
|
) -> EnsembleSmoother:
|
|
301
356
|
active_realizations = _get_and_validate_active_realizations_list(args, config)
|
|
302
357
|
validate_minimum_realizations(config, active_realizations)
|
|
303
|
-
if
|
|
358
|
+
if sum(active_realizations) < 2:
|
|
304
359
|
raise ConfigValidationError(
|
|
305
360
|
"Number of active realizations must be at least 2 for an update step"
|
|
306
361
|
)
|
|
@@ -311,7 +366,7 @@ def _setup_ensemble_smoother(
|
|
|
311
366
|
require_updateable_param=True,
|
|
312
367
|
)
|
|
313
368
|
|
|
314
|
-
|
|
369
|
+
runmodel_config = EnsembleSmootherConfig(
|
|
315
370
|
target_ensemble=args.target_ensemble,
|
|
316
371
|
experiment_name=getattr(args, "experiment_name", ""),
|
|
317
372
|
active_realizations=active_realizations,
|
|
@@ -321,7 +376,6 @@ def _setup_ensemble_smoother(
|
|
|
321
376
|
queue_config=config.queue_config,
|
|
322
377
|
analysis_settings=config.analysis_config.es_settings,
|
|
323
378
|
update_settings=update_settings,
|
|
324
|
-
status_queue=status_queue,
|
|
325
379
|
runpath_file=config.runpath_file,
|
|
326
380
|
design_matrix=design_matrix,
|
|
327
381
|
parameter_configuration=parameter_configs,
|
|
@@ -337,6 +391,7 @@ def _setup_ensemble_smoother(
|
|
|
337
391
|
log_path=config.analysis_config.log_path,
|
|
338
392
|
observations=config.observations,
|
|
339
393
|
)
|
|
394
|
+
return EnsembleSmoother(**runmodel_config.model_dump(), status_queue=status_queue)
|
|
340
395
|
|
|
341
396
|
|
|
342
397
|
def _setup_ensemble_information_filter(
|
|
@@ -347,7 +402,7 @@ def _setup_ensemble_information_filter(
|
|
|
347
402
|
) -> EnsembleInformationFilter:
|
|
348
403
|
active_realizations = _get_and_validate_active_realizations_list(args, config)
|
|
349
404
|
validate_minimum_realizations(config, active_realizations)
|
|
350
|
-
if
|
|
405
|
+
if sum(active_realizations) < 2:
|
|
351
406
|
raise ConfigValidationError(
|
|
352
407
|
"Number of active realizations must be at least 2 for an update step"
|
|
353
408
|
)
|
|
@@ -357,7 +412,7 @@ def _setup_ensemble_information_filter(
|
|
|
357
412
|
parameter_configs=config.ensemble_config.parameter_configuration,
|
|
358
413
|
)
|
|
359
414
|
|
|
360
|
-
|
|
415
|
+
runmodel_config = EnsembleInformationFilterConfig(
|
|
361
416
|
target_ensemble=args.target_ensemble,
|
|
362
417
|
experiment_name=getattr(args, "experiment_name", ""),
|
|
363
418
|
active_realizations=active_realizations,
|
|
@@ -367,7 +422,6 @@ def _setup_ensemble_information_filter(
|
|
|
367
422
|
queue_config=config.queue_config,
|
|
368
423
|
analysis_settings=config.analysis_config.es_settings,
|
|
369
424
|
update_settings=update_settings,
|
|
370
|
-
status_queue=status_queue,
|
|
371
425
|
runpath_file=config.runpath_file,
|
|
372
426
|
design_matrix=design_matrix,
|
|
373
427
|
parameter_configuration=parameter_configs,
|
|
@@ -383,6 +437,9 @@ def _setup_ensemble_information_filter(
|
|
|
383
437
|
log_path=config.analysis_config.log_path,
|
|
384
438
|
observations=config.observations,
|
|
385
439
|
)
|
|
440
|
+
return EnsembleInformationFilter(
|
|
441
|
+
**runmodel_config.model_dump(), status_queue=status_queue
|
|
442
|
+
)
|
|
386
443
|
|
|
387
444
|
|
|
388
445
|
def _determine_restart_info(args: Namespace) -> tuple[bool, str | None]:
|
|
@@ -413,7 +470,7 @@ def _setup_multiple_data_assimilation(
|
|
|
413
470
|
restart_run, prior_ensemble = _determine_restart_info(args)
|
|
414
471
|
active_realizations = _get_and_validate_active_realizations_list(args, config)
|
|
415
472
|
validate_minimum_realizations(config, active_realizations)
|
|
416
|
-
if
|
|
473
|
+
if sum(active_realizations) < 2:
|
|
417
474
|
raise ConfigValidationError(
|
|
418
475
|
"Number of active realizations must be at least 2 for an update step"
|
|
419
476
|
)
|
|
@@ -424,7 +481,7 @@ def _setup_multiple_data_assimilation(
|
|
|
424
481
|
require_updateable_param=True,
|
|
425
482
|
)
|
|
426
483
|
|
|
427
|
-
|
|
484
|
+
runmodel_config = MultipleDataAssimilationConfig(
|
|
428
485
|
random_seed=config.random_seed,
|
|
429
486
|
active_realizations=active_realizations,
|
|
430
487
|
target_ensemble=_iterative_ensemble_format(args),
|
|
@@ -435,7 +492,6 @@ def _setup_multiple_data_assimilation(
|
|
|
435
492
|
experiment_name=args.experiment_name,
|
|
436
493
|
queue_config=config.queue_config,
|
|
437
494
|
update_settings=update_settings,
|
|
438
|
-
status_queue=status_queue,
|
|
439
495
|
storage_path=config.ens_path,
|
|
440
496
|
analysis_settings=config.analysis_config.es_settings,
|
|
441
497
|
runpath_file=config.runpath_file,
|
|
@@ -453,6 +509,9 @@ def _setup_multiple_data_assimilation(
|
|
|
453
509
|
log_path=config.analysis_config.log_path,
|
|
454
510
|
observations=config.observations,
|
|
455
511
|
)
|
|
512
|
+
return MultipleDataAssimilation(
|
|
513
|
+
**runmodel_config.model_dump(), status_queue=status_queue
|
|
514
|
+
)
|
|
456
515
|
|
|
457
516
|
|
|
458
517
|
def _realizations(
|
|
@@ -14,8 +14,11 @@ from ert.config import (
|
|
|
14
14
|
ResponseConfig,
|
|
15
15
|
)
|
|
16
16
|
from ert.ensemble_evaluator import EvaluatorServerConfig
|
|
17
|
-
from ert.run_models.initial_ensemble_run_model import
|
|
18
|
-
|
|
17
|
+
from ert.run_models.initial_ensemble_run_model import (
|
|
18
|
+
InitialEnsembleRunModel,
|
|
19
|
+
InitialEnsembleRunModelConfig,
|
|
20
|
+
)
|
|
21
|
+
from ert.run_models.update_run_model import UpdateRunModel, UpdateRunModelConfig
|
|
19
22
|
from ert.storage import Ensemble
|
|
20
23
|
from ert.trace import tracer
|
|
21
24
|
|
|
@@ -28,18 +31,25 @@ logger = logging.getLogger(__name__)
|
|
|
28
31
|
MULTIPLE_DATA_ASSIMILATION_GROUP = "Parameter update"
|
|
29
32
|
|
|
30
33
|
|
|
31
|
-
class
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
"""
|
|
35
|
-
|
|
34
|
+
class MultipleDataAssimilationConfig(
|
|
35
|
+
InitialEnsembleRunModelConfig, UpdateRunModelConfig
|
|
36
|
+
):
|
|
36
37
|
default_weights: ClassVar[str] = "4, 2, 1"
|
|
37
38
|
restart_run: bool
|
|
38
39
|
prior_ensemble_id: str | None
|
|
39
40
|
weights: str
|
|
40
41
|
|
|
42
|
+
|
|
43
|
+
class MultipleDataAssimilation(
|
|
44
|
+
UpdateRunModel, InitialEnsembleRunModel, MultipleDataAssimilationConfig
|
|
45
|
+
):
|
|
46
|
+
"""
|
|
47
|
+
Run multiple data assimilation (MDA) ensemble smoother with custom weights.
|
|
48
|
+
"""
|
|
49
|
+
|
|
41
50
|
_parsed_weights: list[float] = PrivateAttr()
|
|
42
51
|
_total_iterations: int = PrivateAttr(default=2)
|
|
52
|
+
_start_iteration: int = PrivateAttr(default=0)
|
|
43
53
|
|
|
44
54
|
def model_post_init(self, ctx: Any) -> None:
|
|
45
55
|
super().model_post_init(ctx)
|
|
@@ -56,7 +66,7 @@ class MultipleDataAssimilation(UpdateRunModel, InitialEnsembleRunModel):
|
|
|
56
66
|
elif not self.experiment_name:
|
|
57
67
|
raise ValueError("For non-restart run, experiment name must be set")
|
|
58
68
|
|
|
59
|
-
self.
|
|
69
|
+
self._start_iteration = start_iteration
|
|
60
70
|
self._total_iterations = total_iterations
|
|
61
71
|
|
|
62
72
|
@tracer.start_as_current_span(f"{__name__}.run_experiment")
|
|
@@ -80,10 +90,10 @@ class MultipleDataAssimilation(UpdateRunModel, InitialEnsembleRunModel):
|
|
|
80
90
|
self.set_env_key("_ERT_EXPERIMENT_ID", str(experiment.id))
|
|
81
91
|
self.set_env_key("_ERT_ENSEMBLE_ID", str(prior.id))
|
|
82
92
|
assert isinstance(prior, Ensemble)
|
|
83
|
-
if self.
|
|
93
|
+
if self._start_iteration != prior.iteration + 1:
|
|
84
94
|
raise ValueError(
|
|
85
95
|
"Experiment misconfigured, got starting "
|
|
86
|
-
f"iteration: {self.
|
|
96
|
+
f"iteration: {self._start_iteration},"
|
|
87
97
|
f"restart iteration = {prior.iteration + 1}"
|
|
88
98
|
)
|
|
89
99
|
target_experiment = self._storage.create_experiment(
|
|
@@ -94,6 +104,7 @@ class MultipleDataAssimilation(UpdateRunModel, InitialEnsembleRunModel):
|
|
|
94
104
|
name=f"Restart from {prior.name}",
|
|
95
105
|
templates=self.ert_templates,
|
|
96
106
|
)
|
|
107
|
+
|
|
97
108
|
except (KeyError, ValueError) as err:
|
|
98
109
|
raise ErtRunError(
|
|
99
110
|
f"Prior ensemble with ID: {id_} does not exists"
|
ert/run_models/run_model.py
CHANGED
|
@@ -3,20 +3,21 @@ from __future__ import annotations
|
|
|
3
3
|
import asyncio
|
|
4
4
|
import concurrent.futures
|
|
5
5
|
import copy
|
|
6
|
-
import dataclasses
|
|
7
6
|
import functools
|
|
8
7
|
import logging
|
|
9
8
|
import os
|
|
10
9
|
import queue
|
|
11
10
|
import shutil
|
|
12
11
|
import threading
|
|
13
|
-
import time
|
|
14
12
|
import traceback
|
|
15
13
|
import uuid
|
|
14
|
+
import warnings
|
|
16
15
|
from abc import ABC, abstractmethod
|
|
17
16
|
from collections import defaultdict
|
|
18
17
|
from collections.abc import Callable, Generator, MutableSequence
|
|
19
18
|
from contextlib import contextmanager
|
|
19
|
+
from dataclasses import dataclass
|
|
20
|
+
from datetime import datetime
|
|
20
21
|
from pathlib import Path
|
|
21
22
|
from typing import TYPE_CHECKING, Any, ClassVar, Protocol, cast
|
|
22
23
|
|
|
@@ -24,8 +25,12 @@ import numpy as np
|
|
|
24
25
|
from pydantic import PrivateAttr, field_validator
|
|
25
26
|
from pydantic_core.core_schema import ValidationInfo
|
|
26
27
|
|
|
27
|
-
from _ert.events import
|
|
28
|
-
|
|
28
|
+
from _ert.events import (
|
|
29
|
+
EEEvent,
|
|
30
|
+
EESnapshot,
|
|
31
|
+
EESnapshotUpdate,
|
|
32
|
+
EnsembleEvaluationWarning,
|
|
33
|
+
)
|
|
29
34
|
from ert.config import (
|
|
30
35
|
ConfigValidationError,
|
|
31
36
|
DesignMatrix,
|
|
@@ -37,7 +42,6 @@ from ert.config import (
|
|
|
37
42
|
PostSimulationFixtures,
|
|
38
43
|
PreSimulationFixtures,
|
|
39
44
|
QueueConfig,
|
|
40
|
-
QueueSystem,
|
|
41
45
|
Workflow,
|
|
42
46
|
create_workflow_fixtures_from_hooked,
|
|
43
47
|
)
|
|
@@ -49,7 +53,7 @@ from ert.ensemble_evaluator import (
|
|
|
49
53
|
Realization,
|
|
50
54
|
WarningEvent,
|
|
51
55
|
)
|
|
52
|
-
from ert.ensemble_evaluator.evaluator import UserCancelled
|
|
56
|
+
from ert.ensemble_evaluator.evaluator import ParallelismViolation, UserCancelled
|
|
53
57
|
from ert.ensemble_evaluator.snapshot import EnsembleSnapshot
|
|
54
58
|
from ert.ensemble_evaluator.state import (
|
|
55
59
|
REALIZATION_STATE_FAILED,
|
|
@@ -67,9 +71,16 @@ from ert.utils import log_duration
|
|
|
67
71
|
from ert.warnings import PostSimulationWarning, capture_specific_warning
|
|
68
72
|
from ert.workflow_runner import WorkflowRunner
|
|
69
73
|
|
|
74
|
+
from ..base_model_context import BaseModelWithContextSupport
|
|
70
75
|
from ..run_arg import RunArg
|
|
71
76
|
from ._create_run_path import create_run_path
|
|
72
|
-
from .event import
|
|
77
|
+
from .event import (
|
|
78
|
+
EndEvent,
|
|
79
|
+
FullSnapshotEvent,
|
|
80
|
+
SnapshotUpdateEvent,
|
|
81
|
+
StartEvent,
|
|
82
|
+
StatusEvents,
|
|
83
|
+
)
|
|
73
84
|
|
|
74
85
|
if TYPE_CHECKING:
|
|
75
86
|
from ert.plugins import ErtRuntimePlugins
|
|
@@ -137,17 +148,16 @@ class StartSimulationsThreadFn(Protocol):
|
|
|
137
148
|
) -> None: ...
|
|
138
149
|
|
|
139
150
|
|
|
140
|
-
@
|
|
151
|
+
@dataclass
|
|
141
152
|
class RunModelAPI:
|
|
142
153
|
experiment_name: str
|
|
143
154
|
supports_rerunning_failed_realizations: bool
|
|
144
155
|
start_simulations_thread: StartSimulationsThreadFn
|
|
145
156
|
cancel: Callable[[], None]
|
|
146
|
-
get_runtime: Callable[[], int]
|
|
147
157
|
has_failed_realizations: Callable[[], bool]
|
|
148
158
|
|
|
149
159
|
|
|
150
|
-
class
|
|
160
|
+
class RunModelConfig(BaseModelWithContextSupport):
|
|
151
161
|
storage_path: str
|
|
152
162
|
runpath_file: Path
|
|
153
163
|
user_config_file: Path
|
|
@@ -165,9 +175,9 @@ class RunModel(BaseModelWithContextSupport, ABC):
|
|
|
165
175
|
minimum_required_realizations: int = 0
|
|
166
176
|
supports_rerunning_failed_realizations: ClassVar[bool] = False
|
|
167
177
|
|
|
178
|
+
|
|
179
|
+
class RunModel(RunModelConfig, ABC):
|
|
168
180
|
# Private attributes initialized in model_post_init
|
|
169
|
-
_start_time: int | None = PrivateAttr(None)
|
|
170
|
-
_stop_time: int | None = PrivateAttr(None)
|
|
171
181
|
_initial_realizations_mask: list[bool] = PrivateAttr()
|
|
172
182
|
_completed_realizations_mask: list[bool] = PrivateAttr(default_factory=list)
|
|
173
183
|
_storage: Storage = PrivateAttr()
|
|
@@ -179,6 +189,8 @@ class RunModel(BaseModelWithContextSupport, ABC):
|
|
|
179
189
|
_is_rerunning_failed_realizations: bool = PrivateAttr(False)
|
|
180
190
|
_run_paths: Runpaths = PrivateAttr()
|
|
181
191
|
_total_iterations: int = PrivateAttr(default=1)
|
|
192
|
+
_start_iteration: int = PrivateAttr(default=0)
|
|
193
|
+
_max_parallelism_violation: ParallelismViolation = ParallelismViolation()
|
|
182
194
|
|
|
183
195
|
def __init__(
|
|
184
196
|
self,
|
|
@@ -198,21 +210,20 @@ class RunModel(BaseModelWithContextSupport, ABC):
|
|
|
198
210
|
self._completed_realizations_mask = [False] * len(self.active_realizations)
|
|
199
211
|
self._storage = open_storage(self.storage_path, mode="w")
|
|
200
212
|
self._rng = np.random.default_rng(self.random_seed)
|
|
201
|
-
self.
|
|
213
|
+
self._start_iteration = self.start_iteration
|
|
202
214
|
|
|
203
215
|
self._run_paths = Runpaths(
|
|
204
|
-
jobname_format=self.
|
|
205
|
-
runpath_format=self.
|
|
216
|
+
jobname_format=self.runpath_config.jobname_format_string,
|
|
217
|
+
runpath_format=self.runpath_config.runpath_format_string,
|
|
206
218
|
filename=str(self.runpath_file),
|
|
207
219
|
substitutions=self.substitutions,
|
|
208
|
-
eclbase=self.
|
|
220
|
+
eclbase=self.runpath_config.eclbase_format_string,
|
|
209
221
|
)
|
|
210
222
|
|
|
211
223
|
@property
|
|
212
224
|
def api(self) -> RunModelAPI:
|
|
213
225
|
return RunModelAPI(
|
|
214
226
|
experiment_name=self.name(),
|
|
215
|
-
get_runtime=self.get_runtime,
|
|
216
227
|
start_simulations_thread=self.start_simulations_thread,
|
|
217
228
|
has_failed_realizations=self.has_failed_realizations,
|
|
218
229
|
supports_rerunning_failed_realizations=self.supports_rerunning_failed_realizations,
|
|
@@ -330,10 +341,6 @@ class RunModel(BaseModelWithContextSupport, ABC):
|
|
|
330
341
|
def send_event(self, event: StatusEvents) -> None:
|
|
331
342
|
self._status_queue.put(event)
|
|
332
343
|
|
|
333
|
-
@property
|
|
334
|
-
def queue_system(self) -> QueueSystem:
|
|
335
|
-
return self.queue_config.queue_system
|
|
336
|
-
|
|
337
344
|
@property
|
|
338
345
|
def ensemble_size(self) -> int:
|
|
339
346
|
return len(self._initial_realizations_mask)
|
|
@@ -396,9 +403,9 @@ class RunModel(BaseModelWithContextSupport, ABC):
|
|
|
396
403
|
def handle_captured_event(message: Warning | str) -> None:
|
|
397
404
|
self.send_event(WarningEvent(msg=str(message)))
|
|
398
405
|
|
|
406
|
+
start_timestamp = datetime.now()
|
|
399
407
|
try:
|
|
400
|
-
self.
|
|
401
|
-
self._stop_time = None
|
|
408
|
+
self.send_event(StartEvent(timestamp=start_timestamp))
|
|
402
409
|
with (
|
|
403
410
|
capture_specific_warning(PostSimulationWarning, handle_captured_event),
|
|
404
411
|
captured_logs(error_messages),
|
|
@@ -453,7 +460,6 @@ class RunModel(BaseModelWithContextSupport, ABC):
|
|
|
453
460
|
finally:
|
|
454
461
|
self._storage.close()
|
|
455
462
|
self._clean_env_context()
|
|
456
|
-
self._stop_time = int(time.time())
|
|
457
463
|
self.send_event(
|
|
458
464
|
EndEvent(
|
|
459
465
|
failed=failed,
|
|
@@ -468,6 +474,10 @@ class RunModel(BaseModelWithContextSupport, ABC):
|
|
|
468
474
|
),
|
|
469
475
|
)
|
|
470
476
|
)
|
|
477
|
+
logger.info(
|
|
478
|
+
"Experiment run finished in: "
|
|
479
|
+
f"{(datetime.now() - start_timestamp).total_seconds():g}s"
|
|
480
|
+
)
|
|
471
481
|
|
|
472
482
|
@abstractmethod
|
|
473
483
|
def run_experiment(
|
|
@@ -489,13 +499,6 @@ class RunModel(BaseModelWithContextSupport, ABC):
|
|
|
489
499
|
return f"{exception}\n{msg}"
|
|
490
500
|
return f"{exception}\n{traceback}\n{msg}"
|
|
491
501
|
|
|
492
|
-
def get_runtime(self) -> int:
|
|
493
|
-
if self._start_time is None:
|
|
494
|
-
return 0
|
|
495
|
-
elif self._stop_time is None:
|
|
496
|
-
return round(time.time() - self._start_time)
|
|
497
|
-
return self._stop_time - self._start_time
|
|
498
|
-
|
|
499
502
|
def get_current_status(self) -> dict[str, int]:
|
|
500
503
|
status: dict[str, int] = defaultdict(int)
|
|
501
504
|
if self._iter_snapshot.keys():
|
|
@@ -609,6 +612,8 @@ class RunModel(BaseModelWithContextSupport, ABC):
|
|
|
609
612
|
snapshot=copy.deepcopy(snapshot),
|
|
610
613
|
)
|
|
611
614
|
)
|
|
615
|
+
elif type(event) is EnsembleEvaluationWarning:
|
|
616
|
+
self.send_event(event)
|
|
612
617
|
|
|
613
618
|
async def run_ensemble_evaluator_async(
|
|
614
619
|
self,
|
|
@@ -639,6 +644,10 @@ class RunModel(BaseModelWithContextSupport, ABC):
|
|
|
639
644
|
finally:
|
|
640
645
|
await evaluator_task
|
|
641
646
|
|
|
647
|
+
self._max_parallelism_violation = max(
|
|
648
|
+
self._max_parallelism_violation, evaluator.max_parallelism_violation
|
|
649
|
+
)
|
|
650
|
+
|
|
642
651
|
logger.debug("tasks complete")
|
|
643
652
|
|
|
644
653
|
if self._end_event.is_set():
|
|
@@ -704,7 +713,8 @@ class RunModel(BaseModelWithContextSupport, ABC):
|
|
|
704
713
|
run_paths = []
|
|
705
714
|
active_realizations = np.where(self.active_realizations)[0]
|
|
706
715
|
for iteration in range(
|
|
707
|
-
self.
|
|
716
|
+
self._start_iteration,
|
|
717
|
+
self._total_iterations + self._start_iteration,
|
|
708
718
|
):
|
|
709
719
|
run_paths.extend(self._run_paths.get_paths(active_realizations, iteration))
|
|
710
720
|
return run_paths
|
|
@@ -772,7 +782,7 @@ class RunModel(BaseModelWithContextSupport, ABC):
|
|
|
772
782
|
env_pr_fm_step=self.env_pr_fm_step,
|
|
773
783
|
forward_model_steps=self.forward_model_steps,
|
|
774
784
|
substitutions=self.substitutions,
|
|
775
|
-
parameters_file=self.
|
|
785
|
+
parameters_file=self.runpath_config.gen_kw_export_name,
|
|
776
786
|
runpaths=self._run_paths,
|
|
777
787
|
context_env=self._context_env,
|
|
778
788
|
)
|
|
@@ -818,7 +828,17 @@ class RunModel(BaseModelWithContextSupport, ABC):
|
|
|
818
828
|
f"Experiment run ended with number of realizations failing: "
|
|
819
829
|
f"{len(starting_realizations) - num_successful_realizations}"
|
|
820
830
|
)
|
|
821
|
-
|
|
831
|
+
|
|
832
|
+
# 0 means no violation
|
|
833
|
+
if self._max_parallelism_violation.amount > 0 and isinstance(
|
|
834
|
+
self._max_parallelism_violation.message, str
|
|
835
|
+
):
|
|
836
|
+
warnings.warn(
|
|
837
|
+
self._max_parallelism_violation.message,
|
|
838
|
+
PostSimulationWarning,
|
|
839
|
+
stacklevel=1,
|
|
840
|
+
)
|
|
841
|
+
|
|
822
842
|
self.run_workflows(
|
|
823
843
|
fixtures=PostSimulationFixtures(
|
|
824
844
|
storage=self._storage,
|
|
@@ -3,11 +3,17 @@ from __future__ import annotations
|
|
|
3
3
|
from pydantic import Field
|
|
4
4
|
|
|
5
5
|
from ert.run_models import EnsembleExperiment
|
|
6
|
+
from ert.run_models.ensemble_experiment import EnsembleExperimentConfig
|
|
6
7
|
|
|
7
8
|
SINGLE_TEST_RUN_GROUP = "Forward model evaluation"
|
|
8
9
|
|
|
9
10
|
|
|
10
|
-
class
|
|
11
|
+
class SingleTestRunConfig(EnsembleExperimentConfig):
|
|
12
|
+
active_realizations: list[bool] = Field(default_factory=lambda: [True])
|
|
13
|
+
minimum_required_realizations: int = 1
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class SingleTestRun(EnsembleExperiment, SingleTestRunConfig):
|
|
11
17
|
"""
|
|
12
18
|
Single test is equivalent to EnsembleExperiment, in that it
|
|
13
19
|
samples the prior and evaluates it.<br>There are two key differences:<br>
|
|
@@ -15,9 +21,6 @@ class SingleTestRun(EnsembleExperiment):
|
|
|
15
21
|
2) Only a <b>single realization</b> (realization-0) is run<br>
|
|
16
22
|
"""
|
|
17
23
|
|
|
18
|
-
active_realizations: list[bool] = Field(default_factory=lambda: [True])
|
|
19
|
-
minimum_required_realizations: int = 1
|
|
20
|
-
|
|
21
24
|
@classmethod
|
|
22
25
|
def name(cls) -> str:
|
|
23
26
|
return "Single realization test-run"
|
|
@@ -27,15 +27,17 @@ from ert.run_models.event import (
|
|
|
27
27
|
RunModelUpdateBeginEvent,
|
|
28
28
|
RunModelUpdateEndEvent,
|
|
29
29
|
)
|
|
30
|
-
from ert.run_models.run_model import ErtRunError, RunModel
|
|
30
|
+
from ert.run_models.run_model import ErtRunError, RunModel, RunModelConfig
|
|
31
31
|
from ert.storage import Ensemble, LocalExperiment
|
|
32
32
|
|
|
33
33
|
|
|
34
|
-
class
|
|
34
|
+
class UpdateRunModelConfig(RunModelConfig):
|
|
35
35
|
target_ensemble: str
|
|
36
36
|
analysis_settings: ESSettings
|
|
37
37
|
update_settings: ObservationSettings
|
|
38
38
|
|
|
39
|
+
|
|
40
|
+
class UpdateRunModel(RunModel, UpdateRunModelConfig):
|
|
39
41
|
@abstractmethod
|
|
40
42
|
def update_ensemble_parameters(
|
|
41
43
|
self, prior: Ensemble, posterior: Ensemble, weight: float
|
ert/runpaths.py
CHANGED
|
@@ -101,14 +101,13 @@ class Runpaths:
|
|
|
101
101
|
with open(self.runpath_list_filename, "w", encoding="utf-8") as filehandle:
|
|
102
102
|
for iteration in iteration_numbers:
|
|
103
103
|
for realization in realization_numbers:
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
realization,
|
|
107
|
-
iteration,
|
|
104
|
+
real_iter_substituter = self.substitutions.real_iter_substituter(
|
|
105
|
+
realization, iteration
|
|
108
106
|
)
|
|
109
|
-
|
|
110
|
-
self.
|
|
107
|
+
job_name_or_eclbase = real_iter_substituter.substitute(
|
|
108
|
+
self._eclbase or self._jobname_format,
|
|
111
109
|
)
|
|
110
|
+
runpath = real_iter_substituter.substitute(self._runpath_format)
|
|
112
111
|
|
|
113
112
|
filehandle.write(
|
|
114
113
|
f"{realization:03d} {runpath} "
|