ert 17.1.9__py3-none-any.whl → 18.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (165) hide show
  1. _ert/events.py +19 -2
  2. ert/__main__.py +8 -7
  3. ert/analysis/_update_commons.py +12 -3
  4. ert/cli/main.py +6 -3
  5. ert/cli/monitor.py +7 -0
  6. ert/config/__init__.py +13 -3
  7. ert/config/_create_observation_dataframes.py +60 -12
  8. ert/config/_observations.py +14 -1
  9. ert/config/_read_summary.py +8 -6
  10. ert/config/ensemble_config.py +6 -14
  11. ert/config/ert_config.py +19 -13
  12. ert/config/{everest_objective_config.py → everest_response.py} +23 -12
  13. ert/config/ext_param_config.py +133 -1
  14. ert/config/field.py +12 -8
  15. ert/config/forward_model_step.py +108 -6
  16. ert/config/gen_data_config.py +2 -6
  17. ert/config/gen_kw_config.py +0 -9
  18. ert/config/known_response_types.py +14 -0
  19. ert/config/parameter_config.py +0 -17
  20. ert/config/parsing/config_keywords.py +1 -0
  21. ert/config/parsing/config_schema.py +12 -0
  22. ert/config/parsing/config_schema_deprecations.py +11 -0
  23. ert/config/parsing/config_schema_item.py +1 -1
  24. ert/config/queue_config.py +4 -4
  25. ert/config/response_config.py +0 -7
  26. ert/config/rft_config.py +230 -0
  27. ert/config/summary_config.py +2 -6
  28. ert/config/violations.py +0 -0
  29. ert/config/workflow_fixtures.py +2 -1
  30. ert/dark_storage/client/__init__.py +2 -2
  31. ert/dark_storage/client/_session.py +4 -4
  32. ert/dark_storage/client/client.py +2 -2
  33. ert/dark_storage/compute/misfits.py +7 -6
  34. ert/dark_storage/endpoints/compute/misfits.py +2 -2
  35. ert/dark_storage/endpoints/observations.py +4 -4
  36. ert/dark_storage/endpoints/responses.py +15 -1
  37. ert/ensemble_evaluator/__init__.py +8 -1
  38. ert/ensemble_evaluator/evaluator.py +81 -29
  39. ert/ensemble_evaluator/event.py +6 -0
  40. ert/ensemble_evaluator/snapshot.py +3 -1
  41. ert/ensemble_evaluator/state.py +1 -0
  42. ert/field_utils/__init__.py +8 -0
  43. ert/field_utils/field_utils.py +211 -1
  44. ert/gui/ertwidgets/__init__.py +23 -16
  45. ert/gui/ertwidgets/analysismoduleedit.py +2 -2
  46. ert/gui/ertwidgets/checklist.py +1 -1
  47. ert/gui/ertwidgets/create_experiment_dialog.py +3 -1
  48. ert/gui/ertwidgets/ensembleselector.py +2 -2
  49. ert/gui/ertwidgets/models/__init__.py +2 -0
  50. ert/gui/ertwidgets/models/activerealizationsmodel.py +2 -1
  51. ert/gui/ertwidgets/models/path_model.py +1 -1
  52. ert/gui/ertwidgets/models/targetensemblemodel.py +2 -1
  53. ert/gui/ertwidgets/models/text_model.py +1 -1
  54. ert/gui/ertwidgets/searchbox.py +13 -4
  55. ert/gui/{suggestor → ertwidgets/suggestor}/_suggestor_message.py +13 -4
  56. ert/gui/main.py +11 -6
  57. ert/gui/main_window.py +1 -2
  58. ert/gui/simulation/ensemble_experiment_panel.py +1 -1
  59. ert/gui/simulation/ensemble_information_filter_panel.py +1 -1
  60. ert/gui/simulation/ensemble_smoother_panel.py +1 -1
  61. ert/gui/simulation/evaluate_ensemble_panel.py +1 -1
  62. ert/gui/simulation/experiment_panel.py +1 -1
  63. ert/gui/simulation/manual_update_panel.py +31 -8
  64. ert/gui/simulation/multiple_data_assimilation_panel.py +12 -8
  65. ert/gui/simulation/run_dialog.py +25 -4
  66. ert/gui/simulation/single_test_run_panel.py +2 -2
  67. ert/gui/summarypanel.py +1 -1
  68. ert/gui/tools/load_results/load_results_panel.py +1 -1
  69. ert/gui/tools/manage_experiments/storage_info_widget.py +7 -7
  70. ert/gui/tools/manage_experiments/storage_widget.py +1 -2
  71. ert/gui/tools/plot/plot_api.py +13 -10
  72. ert/gui/tools/plot/plot_window.py +12 -0
  73. ert/gui/tools/plot/plottery/plot_config.py +2 -0
  74. ert/gui/tools/plot/plottery/plot_context.py +14 -0
  75. ert/gui/tools/plot/plottery/plots/ensemble.py +9 -2
  76. ert/gui/tools/plot/plottery/plots/statistics.py +59 -19
  77. ert/mode_definitions.py +2 -0
  78. ert/plugins/__init__.py +0 -1
  79. ert/plugins/hook_implementations/workflows/gen_data_rft_export.py +10 -2
  80. ert/plugins/hook_specifications/__init__.py +0 -2
  81. ert/plugins/hook_specifications/jobs.py +0 -9
  82. ert/plugins/plugin_manager.py +2 -33
  83. ert/resources/shell_scripts/delete_directory.py +2 -2
  84. ert/run_models/__init__.py +18 -5
  85. ert/run_models/_create_run_path.py +33 -21
  86. ert/run_models/ensemble_experiment.py +10 -4
  87. ert/run_models/ensemble_information_filter.py +8 -1
  88. ert/run_models/ensemble_smoother.py +9 -3
  89. ert/run_models/evaluate_ensemble.py +8 -6
  90. ert/run_models/event.py +7 -3
  91. ert/run_models/everest_run_model.py +155 -44
  92. ert/run_models/initial_ensemble_run_model.py +23 -22
  93. ert/run_models/manual_update.py +4 -2
  94. ert/run_models/manual_update_enif.py +37 -0
  95. ert/run_models/model_factory.py +81 -22
  96. ert/run_models/multiple_data_assimilation.py +21 -10
  97. ert/run_models/run_model.py +54 -34
  98. ert/run_models/single_test_run.py +7 -4
  99. ert/run_models/update_run_model.py +4 -2
  100. ert/runpaths.py +5 -6
  101. ert/sample_prior.py +9 -4
  102. ert/scheduler/driver.py +37 -0
  103. ert/scheduler/event.py +3 -1
  104. ert/scheduler/job.py +23 -13
  105. ert/scheduler/lsf_driver.py +6 -2
  106. ert/scheduler/openpbs_driver.py +7 -1
  107. ert/scheduler/scheduler.py +5 -0
  108. ert/scheduler/slurm_driver.py +6 -2
  109. ert/services/__init__.py +2 -2
  110. ert/services/_base_service.py +31 -15
  111. ert/services/ert_server.py +317 -0
  112. ert/shared/_doc_utils/ert_jobs.py +1 -4
  113. ert/shared/storage/connection.py +3 -3
  114. ert/shared/version.py +3 -3
  115. ert/storage/local_ensemble.py +25 -5
  116. ert/storage/local_experiment.py +6 -14
  117. ert/storage/local_storage.py +35 -30
  118. ert/storage/migration/to18.py +12 -0
  119. ert/storage/migration/to8.py +4 -4
  120. ert/substitutions.py +12 -28
  121. ert/validation/active_range.py +7 -7
  122. ert/validation/rangestring.py +16 -16
  123. {ert-17.1.9.dist-info → ert-18.0.0.dist-info}/METADATA +8 -7
  124. {ert-17.1.9.dist-info → ert-18.0.0.dist-info}/RECORD +160 -159
  125. everest/api/everest_data_api.py +1 -14
  126. everest/bin/config_branch_script.py +3 -6
  127. everest/bin/everconfigdump_script.py +1 -9
  128. everest/bin/everest_script.py +21 -11
  129. everest/bin/kill_script.py +2 -2
  130. everest/bin/monitor_script.py +2 -2
  131. everest/bin/utils.py +6 -3
  132. everest/config/__init__.py +4 -1
  133. everest/config/control_config.py +61 -2
  134. everest/config/control_variable_config.py +2 -1
  135. everest/config/everest_config.py +38 -16
  136. everest/config/forward_model_config.py +5 -3
  137. everest/config/install_data_config.py +7 -5
  138. everest/config/install_job_config.py +7 -3
  139. everest/config/install_template_config.py +3 -3
  140. everest/config/optimization_config.py +19 -6
  141. everest/config/output_constraint_config.py +8 -2
  142. everest/config/server_config.py +6 -49
  143. everest/config/utils.py +25 -105
  144. everest/config/validation_utils.py +10 -10
  145. everest/config_file_loader.py +13 -2
  146. everest/detached/everserver.py +7 -8
  147. everest/everest_storage.py +6 -10
  148. everest/gui/everest_client.py +0 -1
  149. everest/gui/main_window.py +2 -2
  150. everest/optimizer/everest2ropt.py +59 -32
  151. everest/optimizer/opt_model_transforms.py +12 -13
  152. everest/optimizer/utils.py +0 -29
  153. everest/strings.py +0 -5
  154. ert/config/everest_constraints_config.py +0 -95
  155. ert/services/storage_service.py +0 -127
  156. everest/config/sampler_config.py +0 -103
  157. everest/simulator/__init__.py +0 -88
  158. everest/simulator/everest_to_ert.py +0 -51
  159. /ert/gui/{suggestor → ertwidgets/suggestor}/__init__.py +0 -0
  160. /ert/gui/{suggestor → ertwidgets/suggestor}/_colors.py +0 -0
  161. /ert/gui/{suggestor → ertwidgets/suggestor}/suggestor.py +0 -0
  162. {ert-17.1.9.dist-info → ert-18.0.0.dist-info}/WHEEL +0 -0
  163. {ert-17.1.9.dist-info → ert-18.0.0.dist-info}/entry_points.txt +0 -0
  164. {ert-17.1.9.dist-info → ert-18.0.0.dist-info}/licenses/COPYING +0 -0
  165. {ert-17.1.9.dist-info → ert-18.0.0.dist-info}/top_level.txt +0 -0
@@ -21,20 +21,28 @@ from ert.mode_definitions import (
21
21
  ENSEMBLE_SMOOTHER_MODE,
22
22
  ES_MDA_MODE,
23
23
  EVALUATE_ENSEMBLE_MODE,
24
+ MANUAL_ENIF_UPDATE_MODE,
24
25
  MANUAL_UPDATE_MODE,
25
26
  TEST_RUN_MODE,
26
27
  )
27
28
  from ert.validation import ActiveRange
28
29
 
29
- from .ensemble_experiment import EnsembleExperiment
30
- from .ensemble_information_filter import EnsembleInformationFilter
31
- from .ensemble_smoother import EnsembleSmoother
32
- from .evaluate_ensemble import EvaluateEnsemble
30
+ from .ensemble_experiment import EnsembleExperiment, EnsembleExperimentConfig
31
+ from .ensemble_information_filter import (
32
+ EnsembleInformationFilter,
33
+ EnsembleInformationFilterConfig,
34
+ )
35
+ from .ensemble_smoother import EnsembleSmoother, EnsembleSmootherConfig
36
+ from .evaluate_ensemble import EvaluateEnsemble, EvaluateEnsembleConfig
33
37
  from .initial_ensemble_run_model import DictEncodedDataFrame
34
- from .manual_update import ManualUpdate
35
- from .multiple_data_assimilation import MultipleDataAssimilation
38
+ from .manual_update import ManualUpdate, ManualUpdateConfig
39
+ from .manual_update_enif import ManualUpdateEnIF
40
+ from .multiple_data_assimilation import (
41
+ MultipleDataAssimilation,
42
+ MultipleDataAssimilationConfig,
43
+ )
36
44
  from .run_model import RunModel
37
- from .single_test_run import SingleTestRun
45
+ from .single_test_run import SingleTestRun, SingleTestRunConfig
38
46
 
39
47
  if TYPE_CHECKING:
40
48
  import numpy.typing as npt
@@ -78,6 +86,8 @@ def create_model(
78
86
  )
79
87
  if args.mode == MANUAL_UPDATE_MODE:
80
88
  return _setup_manual_update(config, args, update_settings, status_queue)
89
+ if args.mode == MANUAL_ENIF_UPDATE_MODE:
90
+ return _setup_manual_update_enif(config, args, update_settings, status_queue)
81
91
  raise NotImplementedError(f"Run type not supported {args.mode}")
82
92
 
83
93
 
@@ -122,7 +132,7 @@ def _setup_single_test_run(
122
132
  parameter_configs=config.ensemble_config.parameter_configuration,
123
133
  )
124
134
 
125
- return SingleTestRun(
135
+ runmodel_config = SingleTestRunConfig(
126
136
  random_seed=config.random_seed,
127
137
  runpath_file=config.runpath_file,
128
138
  active_realizations=[True],
@@ -144,6 +154,10 @@ def _setup_single_test_run(
144
154
  storage_path=config.ens_path,
145
155
  queue_config=config.queue_config.create_local_copy(),
146
156
  observations=config.observations,
157
+ )
158
+
159
+ return SingleTestRun(
160
+ **runmodel_config.model_dump(),
147
161
  status_queue=status_queue,
148
162
  )
149
163
 
@@ -177,7 +191,7 @@ def _setup_ensemble_experiment(
177
191
  parameter_configs=config.ensemble_config.parameter_configuration,
178
192
  )
179
193
 
180
- return EnsembleExperiment(
194
+ runmodel_config = EnsembleExperimentConfig(
181
195
  random_seed=config.random_seed,
182
196
  runpath_file=config.runpath_file,
183
197
  active_realizations=active_realizations,
@@ -199,6 +213,10 @@ def _setup_ensemble_experiment(
199
213
  storage_path=config.ens_path,
200
214
  queue_config=config.queue_config,
201
215
  observations=config.observations,
216
+ )
217
+
218
+ return EnsembleExperiment(
219
+ **runmodel_config.model_dump(),
202
220
  status_queue=status_queue,
203
221
  )
204
222
 
@@ -210,14 +228,13 @@ def _setup_evaluate_ensemble(
210
228
  ) -> EvaluateEnsemble:
211
229
  active_realizations = _get_and_validate_active_realizations_list(args, config)
212
230
  validate_minimum_realizations(config, active_realizations)
213
- return EvaluateEnsemble(
231
+ runmodel_config = EvaluateEnsembleConfig(
214
232
  random_seed=config.random_seed,
215
233
  active_realizations=active_realizations,
216
234
  ensemble_id=args.ensemble_id,
217
235
  minimum_required_realizations=config.analysis_config.minimum_required_realizations,
218
236
  storage_path=config.ens_path,
219
237
  queue_config=config.queue_config,
220
- status_queue=status_queue,
221
238
  runpath_file=config.runpath_file,
222
239
  user_config_file=Path(config.user_config_file),
223
240
  env_vars=config.env_vars,
@@ -228,6 +245,7 @@ def _setup_evaluate_ensemble(
228
245
  hooked_workflows=config.hooked_workflows,
229
246
  log_path=config.analysis_config.log_path,
230
247
  )
248
+ return EvaluateEnsemble(**runmodel_config.model_dump(), status_queue=status_queue)
231
249
 
232
250
 
233
251
  def _get_and_validate_active_realizations_list(
@@ -267,7 +285,7 @@ def _setup_manual_update(
267
285
  active_realizations = _realizations(args, config.runpath_config.num_realizations)
268
286
  validate_minimum_realizations(config, active_realizations.tolist())
269
287
 
270
- return ManualUpdate(
288
+ runmodel_config = ManualUpdateConfig(
271
289
  random_seed=config.random_seed,
272
290
  active_realizations=active_realizations.tolist(),
273
291
  ensemble_id=args.ensemble_id,
@@ -277,7 +295,6 @@ def _setup_manual_update(
277
295
  queue_config=config.queue_config,
278
296
  analysis_settings=config.analysis_config.es_settings,
279
297
  update_settings=update_settings,
280
- status_queue=status_queue,
281
298
  runpath_file=config.runpath_file,
282
299
  user_config_file=Path(config.user_config_file),
283
300
  env_vars=config.env_vars,
@@ -290,6 +307,44 @@ def _setup_manual_update(
290
307
  ert_templates=config.ert_templates,
291
308
  observations=config.observations,
292
309
  )
310
+ return ManualUpdate(**runmodel_config.model_dump(), status_queue=status_queue)
311
+
312
+
313
+ def _setup_manual_update_enif(
314
+ config: ErtConfig,
315
+ args: Namespace,
316
+ update_settings: ObservationSettings,
317
+ status_queue: SimpleQueue[StatusEvents],
318
+ ) -> ManualUpdate:
319
+ active_realizations = _realizations(args, config.runpath_config.num_realizations)
320
+
321
+ return ManualUpdateEnIF(
322
+ random_seed=config.random_seed,
323
+ active_realizations=active_realizations.tolist(),
324
+ ensemble_id=args.ensemble_id,
325
+ minimum_required_realizations=config.analysis_config.minimum_required_realizations,
326
+ target_ensemble=args.target_ensemble,
327
+ config=config,
328
+ storage_path=config.ens_path,
329
+ queue_config=config.queue_config,
330
+ analysis_settings=config.analysis_config.es_settings,
331
+ update_settings=update_settings,
332
+ status_queue=status_queue,
333
+ runpath_file=config.runpath_file,
334
+ design_matrix=config.analysis_config.design_matrix,
335
+ parameter_configuration=config.ensemble_config.parameter_configuration,
336
+ response_configuration=config.ensemble_config.response_configuration,
337
+ ert_templates=config.ert_templates,
338
+ user_config_file=Path(config.user_config_file),
339
+ env_vars=config.env_vars,
340
+ env_pr_fm_step=config.env_pr_fm_step,
341
+ runpath_config=config.runpath_config,
342
+ forward_model_steps=config.forward_model_steps,
343
+ substitutions=config.substitutions,
344
+ hooked_workflows=config.hooked_workflows,
345
+ log_path=config.analysis_config.log_path,
346
+ observations=config.observations,
347
+ )
293
348
 
294
349
 
295
350
  def _setup_ensemble_smoother(
@@ -300,7 +355,7 @@ def _setup_ensemble_smoother(
300
355
  ) -> EnsembleSmoother:
301
356
  active_realizations = _get_and_validate_active_realizations_list(args, config)
302
357
  validate_minimum_realizations(config, active_realizations)
303
- if len(active_realizations) < 2:
358
+ if sum(active_realizations) < 2:
304
359
  raise ConfigValidationError(
305
360
  "Number of active realizations must be at least 2 for an update step"
306
361
  )
@@ -311,7 +366,7 @@ def _setup_ensemble_smoother(
311
366
  require_updateable_param=True,
312
367
  )
313
368
 
314
- return EnsembleSmoother(
369
+ runmodel_config = EnsembleSmootherConfig(
315
370
  target_ensemble=args.target_ensemble,
316
371
  experiment_name=getattr(args, "experiment_name", ""),
317
372
  active_realizations=active_realizations,
@@ -321,7 +376,6 @@ def _setup_ensemble_smoother(
321
376
  queue_config=config.queue_config,
322
377
  analysis_settings=config.analysis_config.es_settings,
323
378
  update_settings=update_settings,
324
- status_queue=status_queue,
325
379
  runpath_file=config.runpath_file,
326
380
  design_matrix=design_matrix,
327
381
  parameter_configuration=parameter_configs,
@@ -337,6 +391,7 @@ def _setup_ensemble_smoother(
337
391
  log_path=config.analysis_config.log_path,
338
392
  observations=config.observations,
339
393
  )
394
+ return EnsembleSmoother(**runmodel_config.model_dump(), status_queue=status_queue)
340
395
 
341
396
 
342
397
  def _setup_ensemble_information_filter(
@@ -347,7 +402,7 @@ def _setup_ensemble_information_filter(
347
402
  ) -> EnsembleInformationFilter:
348
403
  active_realizations = _get_and_validate_active_realizations_list(args, config)
349
404
  validate_minimum_realizations(config, active_realizations)
350
- if len(active_realizations) < 2:
405
+ if sum(active_realizations) < 2:
351
406
  raise ConfigValidationError(
352
407
  "Number of active realizations must be at least 2 for an update step"
353
408
  )
@@ -357,7 +412,7 @@ def _setup_ensemble_information_filter(
357
412
  parameter_configs=config.ensemble_config.parameter_configuration,
358
413
  )
359
414
 
360
- return EnsembleInformationFilter(
415
+ runmodel_config = EnsembleInformationFilterConfig(
361
416
  target_ensemble=args.target_ensemble,
362
417
  experiment_name=getattr(args, "experiment_name", ""),
363
418
  active_realizations=active_realizations,
@@ -367,7 +422,6 @@ def _setup_ensemble_information_filter(
367
422
  queue_config=config.queue_config,
368
423
  analysis_settings=config.analysis_config.es_settings,
369
424
  update_settings=update_settings,
370
- status_queue=status_queue,
371
425
  runpath_file=config.runpath_file,
372
426
  design_matrix=design_matrix,
373
427
  parameter_configuration=parameter_configs,
@@ -383,6 +437,9 @@ def _setup_ensemble_information_filter(
383
437
  log_path=config.analysis_config.log_path,
384
438
  observations=config.observations,
385
439
  )
440
+ return EnsembleInformationFilter(
441
+ **runmodel_config.model_dump(), status_queue=status_queue
442
+ )
386
443
 
387
444
 
388
445
  def _determine_restart_info(args: Namespace) -> tuple[bool, str | None]:
@@ -413,7 +470,7 @@ def _setup_multiple_data_assimilation(
413
470
  restart_run, prior_ensemble = _determine_restart_info(args)
414
471
  active_realizations = _get_and_validate_active_realizations_list(args, config)
415
472
  validate_minimum_realizations(config, active_realizations)
416
- if len(active_realizations) < 2:
473
+ if sum(active_realizations) < 2:
417
474
  raise ConfigValidationError(
418
475
  "Number of active realizations must be at least 2 for an update step"
419
476
  )
@@ -424,7 +481,7 @@ def _setup_multiple_data_assimilation(
424
481
  require_updateable_param=True,
425
482
  )
426
483
 
427
- return MultipleDataAssimilation(
484
+ runmodel_config = MultipleDataAssimilationConfig(
428
485
  random_seed=config.random_seed,
429
486
  active_realizations=active_realizations,
430
487
  target_ensemble=_iterative_ensemble_format(args),
@@ -435,7 +492,6 @@ def _setup_multiple_data_assimilation(
435
492
  experiment_name=args.experiment_name,
436
493
  queue_config=config.queue_config,
437
494
  update_settings=update_settings,
438
- status_queue=status_queue,
439
495
  storage_path=config.ens_path,
440
496
  analysis_settings=config.analysis_config.es_settings,
441
497
  runpath_file=config.runpath_file,
@@ -453,6 +509,9 @@ def _setup_multiple_data_assimilation(
453
509
  log_path=config.analysis_config.log_path,
454
510
  observations=config.observations,
455
511
  )
512
+ return MultipleDataAssimilation(
513
+ **runmodel_config.model_dump(), status_queue=status_queue
514
+ )
456
515
 
457
516
 
458
517
  def _realizations(
@@ -14,8 +14,11 @@ from ert.config import (
14
14
  ResponseConfig,
15
15
  )
16
16
  from ert.ensemble_evaluator import EvaluatorServerConfig
17
- from ert.run_models.initial_ensemble_run_model import InitialEnsembleRunModel
18
- from ert.run_models.update_run_model import UpdateRunModel
17
+ from ert.run_models.initial_ensemble_run_model import (
18
+ InitialEnsembleRunModel,
19
+ InitialEnsembleRunModelConfig,
20
+ )
21
+ from ert.run_models.update_run_model import UpdateRunModel, UpdateRunModelConfig
19
22
  from ert.storage import Ensemble
20
23
  from ert.trace import tracer
21
24
 
@@ -28,18 +31,25 @@ logger = logging.getLogger(__name__)
28
31
  MULTIPLE_DATA_ASSIMILATION_GROUP = "Parameter update"
29
32
 
30
33
 
31
- class MultipleDataAssimilation(UpdateRunModel, InitialEnsembleRunModel):
32
- """
33
- Run multiple data assimilation (MDA) ensemble smoother with custom weights.
34
- """
35
-
34
+ class MultipleDataAssimilationConfig(
35
+ InitialEnsembleRunModelConfig, UpdateRunModelConfig
36
+ ):
36
37
  default_weights: ClassVar[str] = "4, 2, 1"
37
38
  restart_run: bool
38
39
  prior_ensemble_id: str | None
39
40
  weights: str
40
41
 
42
+
43
+ class MultipleDataAssimilation(
44
+ UpdateRunModel, InitialEnsembleRunModel, MultipleDataAssimilationConfig
45
+ ):
46
+ """
47
+ Run multiple data assimilation (MDA) ensemble smoother with custom weights.
48
+ """
49
+
41
50
  _parsed_weights: list[float] = PrivateAttr()
42
51
  _total_iterations: int = PrivateAttr(default=2)
52
+ _start_iteration: int = PrivateAttr(default=0)
43
53
 
44
54
  def model_post_init(self, ctx: Any) -> None:
45
55
  super().model_post_init(ctx)
@@ -56,7 +66,7 @@ class MultipleDataAssimilation(UpdateRunModel, InitialEnsembleRunModel):
56
66
  elif not self.experiment_name:
57
67
  raise ValueError("For non-restart run, experiment name must be set")
58
68
 
59
- self.start_iteration = start_iteration
69
+ self._start_iteration = start_iteration
60
70
  self._total_iterations = total_iterations
61
71
 
62
72
  @tracer.start_as_current_span(f"{__name__}.run_experiment")
@@ -80,10 +90,10 @@ class MultipleDataAssimilation(UpdateRunModel, InitialEnsembleRunModel):
80
90
  self.set_env_key("_ERT_EXPERIMENT_ID", str(experiment.id))
81
91
  self.set_env_key("_ERT_ENSEMBLE_ID", str(prior.id))
82
92
  assert isinstance(prior, Ensemble)
83
- if self.start_iteration != prior.iteration + 1:
93
+ if self._start_iteration != prior.iteration + 1:
84
94
  raise ValueError(
85
95
  "Experiment misconfigured, got starting "
86
- f"iteration: {self.start_iteration},"
96
+ f"iteration: {self._start_iteration},"
87
97
  f"restart iteration = {prior.iteration + 1}"
88
98
  )
89
99
  target_experiment = self._storage.create_experiment(
@@ -94,6 +104,7 @@ class MultipleDataAssimilation(UpdateRunModel, InitialEnsembleRunModel):
94
104
  name=f"Restart from {prior.name}",
95
105
  templates=self.ert_templates,
96
106
  )
107
+
97
108
  except (KeyError, ValueError) as err:
98
109
  raise ErtRunError(
99
110
  f"Prior ensemble with ID: {id_} does not exists"
@@ -3,20 +3,21 @@ from __future__ import annotations
3
3
  import asyncio
4
4
  import concurrent.futures
5
5
  import copy
6
- import dataclasses
7
6
  import functools
8
7
  import logging
9
8
  import os
10
9
  import queue
11
10
  import shutil
12
11
  import threading
13
- import time
14
12
  import traceback
15
13
  import uuid
14
+ import warnings
16
15
  from abc import ABC, abstractmethod
17
16
  from collections import defaultdict
18
17
  from collections.abc import Callable, Generator, MutableSequence
19
18
  from contextlib import contextmanager
19
+ from dataclasses import dataclass
20
+ from datetime import datetime
20
21
  from pathlib import Path
21
22
  from typing import TYPE_CHECKING, Any, ClassVar, Protocol, cast
22
23
 
@@ -24,8 +25,12 @@ import numpy as np
24
25
  from pydantic import PrivateAttr, field_validator
25
26
  from pydantic_core.core_schema import ValidationInfo
26
27
 
27
- from _ert.events import EEEvent, EESnapshot, EESnapshotUpdate
28
- from ert.base_model_context import BaseModelWithContextSupport
28
+ from _ert.events import (
29
+ EEEvent,
30
+ EESnapshot,
31
+ EESnapshotUpdate,
32
+ EnsembleEvaluationWarning,
33
+ )
29
34
  from ert.config import (
30
35
  ConfigValidationError,
31
36
  DesignMatrix,
@@ -37,7 +42,6 @@ from ert.config import (
37
42
  PostSimulationFixtures,
38
43
  PreSimulationFixtures,
39
44
  QueueConfig,
40
- QueueSystem,
41
45
  Workflow,
42
46
  create_workflow_fixtures_from_hooked,
43
47
  )
@@ -49,7 +53,7 @@ from ert.ensemble_evaluator import (
49
53
  Realization,
50
54
  WarningEvent,
51
55
  )
52
- from ert.ensemble_evaluator.evaluator import UserCancelled
56
+ from ert.ensemble_evaluator.evaluator import ParallelismViolation, UserCancelled
53
57
  from ert.ensemble_evaluator.snapshot import EnsembleSnapshot
54
58
  from ert.ensemble_evaluator.state import (
55
59
  REALIZATION_STATE_FAILED,
@@ -67,9 +71,16 @@ from ert.utils import log_duration
67
71
  from ert.warnings import PostSimulationWarning, capture_specific_warning
68
72
  from ert.workflow_runner import WorkflowRunner
69
73
 
74
+ from ..base_model_context import BaseModelWithContextSupport
70
75
  from ..run_arg import RunArg
71
76
  from ._create_run_path import create_run_path
72
- from .event import EndEvent, FullSnapshotEvent, SnapshotUpdateEvent, StatusEvents
77
+ from .event import (
78
+ EndEvent,
79
+ FullSnapshotEvent,
80
+ SnapshotUpdateEvent,
81
+ StartEvent,
82
+ StatusEvents,
83
+ )
73
84
 
74
85
  if TYPE_CHECKING:
75
86
  from ert.plugins import ErtRuntimePlugins
@@ -137,17 +148,16 @@ class StartSimulationsThreadFn(Protocol):
137
148
  ) -> None: ...
138
149
 
139
150
 
140
- @dataclasses.dataclass
151
+ @dataclass
141
152
  class RunModelAPI:
142
153
  experiment_name: str
143
154
  supports_rerunning_failed_realizations: bool
144
155
  start_simulations_thread: StartSimulationsThreadFn
145
156
  cancel: Callable[[], None]
146
- get_runtime: Callable[[], int]
147
157
  has_failed_realizations: Callable[[], bool]
148
158
 
149
159
 
150
- class RunModel(BaseModelWithContextSupport, ABC):
160
+ class RunModelConfig(BaseModelWithContextSupport):
151
161
  storage_path: str
152
162
  runpath_file: Path
153
163
  user_config_file: Path
@@ -165,9 +175,9 @@ class RunModel(BaseModelWithContextSupport, ABC):
165
175
  minimum_required_realizations: int = 0
166
176
  supports_rerunning_failed_realizations: ClassVar[bool] = False
167
177
 
178
+
179
+ class RunModel(RunModelConfig, ABC):
168
180
  # Private attributes initialized in model_post_init
169
- _start_time: int | None = PrivateAttr(None)
170
- _stop_time: int | None = PrivateAttr(None)
171
181
  _initial_realizations_mask: list[bool] = PrivateAttr()
172
182
  _completed_realizations_mask: list[bool] = PrivateAttr(default_factory=list)
173
183
  _storage: Storage = PrivateAttr()
@@ -179,6 +189,8 @@ class RunModel(BaseModelWithContextSupport, ABC):
179
189
  _is_rerunning_failed_realizations: bool = PrivateAttr(False)
180
190
  _run_paths: Runpaths = PrivateAttr()
181
191
  _total_iterations: int = PrivateAttr(default=1)
192
+ _start_iteration: int = PrivateAttr(default=0)
193
+ _max_parallelism_violation: ParallelismViolation = ParallelismViolation()
182
194
 
183
195
  def __init__(
184
196
  self,
@@ -198,21 +210,20 @@ class RunModel(BaseModelWithContextSupport, ABC):
198
210
  self._completed_realizations_mask = [False] * len(self.active_realizations)
199
211
  self._storage = open_storage(self.storage_path, mode="w")
200
212
  self._rng = np.random.default_rng(self.random_seed)
201
- self._model_config = self.runpath_config
213
+ self._start_iteration = self.start_iteration
202
214
 
203
215
  self._run_paths = Runpaths(
204
- jobname_format=self._model_config.jobname_format_string,
205
- runpath_format=self._model_config.runpath_format_string,
216
+ jobname_format=self.runpath_config.jobname_format_string,
217
+ runpath_format=self.runpath_config.runpath_format_string,
206
218
  filename=str(self.runpath_file),
207
219
  substitutions=self.substitutions,
208
- eclbase=self._model_config.eclbase_format_string,
220
+ eclbase=self.runpath_config.eclbase_format_string,
209
221
  )
210
222
 
211
223
  @property
212
224
  def api(self) -> RunModelAPI:
213
225
  return RunModelAPI(
214
226
  experiment_name=self.name(),
215
- get_runtime=self.get_runtime,
216
227
  start_simulations_thread=self.start_simulations_thread,
217
228
  has_failed_realizations=self.has_failed_realizations,
218
229
  supports_rerunning_failed_realizations=self.supports_rerunning_failed_realizations,
@@ -330,10 +341,6 @@ class RunModel(BaseModelWithContextSupport, ABC):
330
341
  def send_event(self, event: StatusEvents) -> None:
331
342
  self._status_queue.put(event)
332
343
 
333
- @property
334
- def queue_system(self) -> QueueSystem:
335
- return self.queue_config.queue_system
336
-
337
344
  @property
338
345
  def ensemble_size(self) -> int:
339
346
  return len(self._initial_realizations_mask)
@@ -396,9 +403,9 @@ class RunModel(BaseModelWithContextSupport, ABC):
396
403
  def handle_captured_event(message: Warning | str) -> None:
397
404
  self.send_event(WarningEvent(msg=str(message)))
398
405
 
406
+ start_timestamp = datetime.now()
399
407
  try:
400
- self._start_time = int(time.time())
401
- self._stop_time = None
408
+ self.send_event(StartEvent(timestamp=start_timestamp))
402
409
  with (
403
410
  capture_specific_warning(PostSimulationWarning, handle_captured_event),
404
411
  captured_logs(error_messages),
@@ -453,7 +460,6 @@ class RunModel(BaseModelWithContextSupport, ABC):
453
460
  finally:
454
461
  self._storage.close()
455
462
  self._clean_env_context()
456
- self._stop_time = int(time.time())
457
463
  self.send_event(
458
464
  EndEvent(
459
465
  failed=failed,
@@ -468,6 +474,10 @@ class RunModel(BaseModelWithContextSupport, ABC):
468
474
  ),
469
475
  )
470
476
  )
477
+ logger.info(
478
+ "Experiment run finished in: "
479
+ f"{(datetime.now() - start_timestamp).total_seconds():g}s"
480
+ )
471
481
 
472
482
  @abstractmethod
473
483
  def run_experiment(
@@ -489,13 +499,6 @@ class RunModel(BaseModelWithContextSupport, ABC):
489
499
  return f"{exception}\n{msg}"
490
500
  return f"{exception}\n{traceback}\n{msg}"
491
501
 
492
- def get_runtime(self) -> int:
493
- if self._start_time is None:
494
- return 0
495
- elif self._stop_time is None:
496
- return round(time.time() - self._start_time)
497
- return self._stop_time - self._start_time
498
-
499
502
  def get_current_status(self) -> dict[str, int]:
500
503
  status: dict[str, int] = defaultdict(int)
501
504
  if self._iter_snapshot.keys():
@@ -609,6 +612,8 @@ class RunModel(BaseModelWithContextSupport, ABC):
609
612
  snapshot=copy.deepcopy(snapshot),
610
613
  )
611
614
  )
615
+ elif type(event) is EnsembleEvaluationWarning:
616
+ self.send_event(event)
612
617
 
613
618
  async def run_ensemble_evaluator_async(
614
619
  self,
@@ -639,6 +644,10 @@ class RunModel(BaseModelWithContextSupport, ABC):
639
644
  finally:
640
645
  await evaluator_task
641
646
 
647
+ self._max_parallelism_violation = max(
648
+ self._max_parallelism_violation, evaluator.max_parallelism_violation
649
+ )
650
+
642
651
  logger.debug("tasks complete")
643
652
 
644
653
  if self._end_event.is_set():
@@ -704,7 +713,8 @@ class RunModel(BaseModelWithContextSupport, ABC):
704
713
  run_paths = []
705
714
  active_realizations = np.where(self.active_realizations)[0]
706
715
  for iteration in range(
707
- self.start_iteration, self._total_iterations + self.start_iteration
716
+ self._start_iteration,
717
+ self._total_iterations + self._start_iteration,
708
718
  ):
709
719
  run_paths.extend(self._run_paths.get_paths(active_realizations, iteration))
710
720
  return run_paths
@@ -772,7 +782,7 @@ class RunModel(BaseModelWithContextSupport, ABC):
772
782
  env_pr_fm_step=self.env_pr_fm_step,
773
783
  forward_model_steps=self.forward_model_steps,
774
784
  substitutions=self.substitutions,
775
- parameters_file=self._model_config.gen_kw_export_name,
785
+ parameters_file=self.runpath_config.gen_kw_export_name,
776
786
  runpaths=self._run_paths,
777
787
  context_env=self._context_env,
778
788
  )
@@ -818,7 +828,17 @@ class RunModel(BaseModelWithContextSupport, ABC):
818
828
  f"Experiment run ended with number of realizations failing: "
819
829
  f"{len(starting_realizations) - num_successful_realizations}"
820
830
  )
821
- logger.info(f"Experiment run finished in: {self.get_runtime()}s")
831
+
832
+ # 0 means no violation
833
+ if self._max_parallelism_violation.amount > 0 and isinstance(
834
+ self._max_parallelism_violation.message, str
835
+ ):
836
+ warnings.warn(
837
+ self._max_parallelism_violation.message,
838
+ PostSimulationWarning,
839
+ stacklevel=1,
840
+ )
841
+
822
842
  self.run_workflows(
823
843
  fixtures=PostSimulationFixtures(
824
844
  storage=self._storage,
@@ -3,11 +3,17 @@ from __future__ import annotations
3
3
  from pydantic import Field
4
4
 
5
5
  from ert.run_models import EnsembleExperiment
6
+ from ert.run_models.ensemble_experiment import EnsembleExperimentConfig
6
7
 
7
8
  SINGLE_TEST_RUN_GROUP = "Forward model evaluation"
8
9
 
9
10
 
10
- class SingleTestRun(EnsembleExperiment):
11
+ class SingleTestRunConfig(EnsembleExperimentConfig):
12
+ active_realizations: list[bool] = Field(default_factory=lambda: [True])
13
+ minimum_required_realizations: int = 1
14
+
15
+
16
+ class SingleTestRun(EnsembleExperiment, SingleTestRunConfig):
11
17
  """
12
18
  Single test is equivalent to EnsembleExperiment, in that it
13
19
  samples the prior and evaluates it.<br>There are two key differences:<br>
@@ -15,9 +21,6 @@ class SingleTestRun(EnsembleExperiment):
15
21
  2) Only a <b>single realization</b> (realization-0) is run<br>
16
22
  """
17
23
 
18
- active_realizations: list[bool] = Field(default_factory=lambda: [True])
19
- minimum_required_realizations: int = 1
20
-
21
24
  @classmethod
22
25
  def name(cls) -> str:
23
26
  return "Single realization test-run"
@@ -27,15 +27,17 @@ from ert.run_models.event import (
27
27
  RunModelUpdateBeginEvent,
28
28
  RunModelUpdateEndEvent,
29
29
  )
30
- from ert.run_models.run_model import ErtRunError, RunModel
30
+ from ert.run_models.run_model import ErtRunError, RunModel, RunModelConfig
31
31
  from ert.storage import Ensemble, LocalExperiment
32
32
 
33
33
 
34
- class UpdateRunModel(RunModel):
34
+ class UpdateRunModelConfig(RunModelConfig):
35
35
  target_ensemble: str
36
36
  analysis_settings: ESSettings
37
37
  update_settings: ObservationSettings
38
38
 
39
+
40
+ class UpdateRunModel(RunModel, UpdateRunModelConfig):
39
41
  @abstractmethod
40
42
  def update_ensemble_parameters(
41
43
  self, prior: Ensemble, posterior: Ensemble, weight: float
ert/runpaths.py CHANGED
@@ -101,14 +101,13 @@ class Runpaths:
101
101
  with open(self.runpath_list_filename, "w", encoding="utf-8") as filehandle:
102
102
  for iteration in iteration_numbers:
103
103
  for realization in realization_numbers:
104
- job_name_or_eclbase = self.substitutions.substitute_real_iter(
105
- self._eclbase or self._jobname_format,
106
- realization,
107
- iteration,
104
+ real_iter_substituter = self.substitutions.real_iter_substituter(
105
+ realization, iteration
108
106
  )
109
- runpath = self.substitutions.substitute_real_iter(
110
- self._runpath_format, realization, iteration
107
+ job_name_or_eclbase = real_iter_substituter.substitute(
108
+ self._eclbase or self._jobname_format,
111
109
  )
110
+ runpath = real_iter_substituter.substitute(self._runpath_format)
112
111
 
113
112
  filehandle.write(
114
113
  f"{realization:03d} {runpath} "