ert 17.0.0__py3-none-any.whl → 19.0.0rc2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (218) hide show
  1. _ert/events.py +19 -2
  2. _ert/forward_model_runner/client.py +6 -2
  3. ert/__main__.py +28 -13
  4. ert/analysis/_enif_update.py +8 -4
  5. ert/analysis/_es_update.py +19 -6
  6. ert/analysis/_update_commons.py +16 -6
  7. ert/cli/main.py +13 -6
  8. ert/cli/monitor.py +7 -0
  9. ert/config/__init__.py +15 -6
  10. ert/config/_create_observation_dataframes.py +117 -20
  11. ert/config/_get_num_cpu.py +1 -1
  12. ert/config/_observations.py +91 -2
  13. ert/config/_read_summary.py +8 -6
  14. ert/config/design_matrix.py +51 -24
  15. ert/config/distribution.py +1 -1
  16. ert/config/ensemble_config.py +9 -17
  17. ert/config/ert_config.py +103 -19
  18. ert/config/everest_control.py +234 -0
  19. ert/config/{everest_objective_config.py → everest_response.py} +24 -15
  20. ert/config/field.py +96 -84
  21. ert/config/forward_model_step.py +122 -17
  22. ert/config/gen_data_config.py +5 -10
  23. ert/config/gen_kw_config.py +5 -35
  24. ert/config/known_response_types.py +14 -0
  25. ert/config/parameter_config.py +1 -33
  26. ert/config/parsing/_option_dict.py +10 -2
  27. ert/config/parsing/config_keywords.py +2 -0
  28. ert/config/parsing/config_schema.py +23 -3
  29. ert/config/parsing/config_schema_deprecations.py +3 -14
  30. ert/config/parsing/config_schema_item.py +26 -11
  31. ert/config/parsing/context_values.py +3 -3
  32. ert/config/parsing/file_context_token.py +1 -1
  33. ert/config/parsing/observations_parser.py +6 -2
  34. ert/config/parsing/queue_system.py +9 -0
  35. ert/config/parsing/schema_item_type.py +1 -0
  36. ert/config/queue_config.py +4 -5
  37. ert/config/response_config.py +0 -8
  38. ert/config/rft_config.py +275 -0
  39. ert/config/summary_config.py +3 -8
  40. ert/config/surface_config.py +59 -16
  41. ert/config/workflow_fixtures.py +2 -1
  42. ert/dark_storage/client/__init__.py +2 -2
  43. ert/dark_storage/client/_session.py +4 -4
  44. ert/dark_storage/client/client.py +2 -2
  45. ert/dark_storage/common.py +1 -1
  46. ert/dark_storage/compute/misfits.py +11 -7
  47. ert/dark_storage/endpoints/compute/misfits.py +6 -4
  48. ert/dark_storage/endpoints/experiment_server.py +12 -9
  49. ert/dark_storage/endpoints/experiments.py +2 -2
  50. ert/dark_storage/endpoints/observations.py +8 -6
  51. ert/dark_storage/endpoints/parameters.py +2 -18
  52. ert/dark_storage/endpoints/responses.py +24 -5
  53. ert/dark_storage/json_schema/experiment.py +1 -1
  54. ert/data/_measured_data.py +6 -5
  55. ert/ensemble_evaluator/__init__.py +8 -1
  56. ert/ensemble_evaluator/config.py +2 -1
  57. ert/ensemble_evaluator/evaluator.py +81 -29
  58. ert/ensemble_evaluator/event.py +6 -0
  59. ert/ensemble_evaluator/snapshot.py +3 -1
  60. ert/ensemble_evaluator/state.py +1 -0
  61. ert/field_utils/__init__.py +8 -0
  62. ert/field_utils/field_utils.py +212 -3
  63. ert/field_utils/roff_io.py +1 -1
  64. ert/gui/__init__.py +5 -2
  65. ert/gui/ertnotifier.py +1 -1
  66. ert/gui/ertwidgets/__init__.py +23 -16
  67. ert/gui/ertwidgets/analysismoduleedit.py +2 -2
  68. ert/gui/ertwidgets/checklist.py +1 -1
  69. ert/gui/ertwidgets/create_experiment_dialog.py +3 -1
  70. ert/gui/ertwidgets/ensembleselector.py +2 -2
  71. ert/gui/ertwidgets/models/__init__.py +2 -0
  72. ert/gui/ertwidgets/models/activerealizationsmodel.py +2 -1
  73. ert/gui/ertwidgets/models/path_model.py +1 -1
  74. ert/gui/ertwidgets/models/targetensemblemodel.py +2 -1
  75. ert/gui/ertwidgets/models/text_model.py +1 -1
  76. ert/gui/ertwidgets/pathchooser.py +0 -3
  77. ert/gui/ertwidgets/searchbox.py +13 -4
  78. ert/gui/{suggestor → ertwidgets/suggestor}/_suggestor_message.py +13 -4
  79. ert/gui/{suggestor → ertwidgets/suggestor}/suggestor.py +63 -30
  80. ert/gui/main.py +37 -8
  81. ert/gui/main_window.py +1 -7
  82. ert/gui/simulation/ensemble_experiment_panel.py +1 -1
  83. ert/gui/simulation/ensemble_information_filter_panel.py +1 -1
  84. ert/gui/simulation/ensemble_smoother_panel.py +1 -1
  85. ert/gui/simulation/evaluate_ensemble_panel.py +1 -1
  86. ert/gui/simulation/experiment_panel.py +16 -3
  87. ert/gui/simulation/manual_update_panel.py +31 -8
  88. ert/gui/simulation/multiple_data_assimilation_panel.py +12 -8
  89. ert/gui/simulation/run_dialog.py +27 -20
  90. ert/gui/simulation/single_test_run_panel.py +2 -2
  91. ert/gui/summarypanel.py +20 -1
  92. ert/gui/tools/load_results/load_results_panel.py +1 -1
  93. ert/gui/tools/manage_experiments/export_dialog.py +136 -0
  94. ert/gui/tools/manage_experiments/storage_info_widget.py +121 -16
  95. ert/gui/tools/manage_experiments/storage_widget.py +1 -2
  96. ert/gui/tools/plot/plot_api.py +37 -25
  97. ert/gui/tools/plot/plot_widget.py +10 -2
  98. ert/gui/tools/plot/plot_window.py +38 -18
  99. ert/gui/tools/plot/plottery/plot_config.py +2 -0
  100. ert/gui/tools/plot/plottery/plot_context.py +14 -0
  101. ert/gui/tools/plot/plottery/plots/__init__.py +2 -0
  102. ert/gui/tools/plot/plottery/plots/cesp.py +3 -1
  103. ert/gui/tools/plot/plottery/plots/distribution.py +6 -1
  104. ert/gui/tools/plot/plottery/plots/ensemble.py +12 -3
  105. ert/gui/tools/plot/plottery/plots/gaussian_kde.py +12 -2
  106. ert/gui/tools/plot/plottery/plots/histogram.py +3 -1
  107. ert/gui/tools/plot/plottery/plots/misfits.py +436 -0
  108. ert/gui/tools/plot/plottery/plots/observations.py +18 -4
  109. ert/gui/tools/plot/plottery/plots/statistics.py +62 -20
  110. ert/gui/tools/plot/plottery/plots/std_dev.py +3 -1
  111. ert/mode_definitions.py +2 -0
  112. ert/plugins/__init__.py +0 -1
  113. ert/plugins/hook_implementations/workflows/csv_export.py +2 -3
  114. ert/plugins/hook_implementations/workflows/gen_data_rft_export.py +10 -2
  115. ert/plugins/hook_specifications/__init__.py +0 -2
  116. ert/plugins/hook_specifications/jobs.py +0 -9
  117. ert/plugins/plugin_manager.py +6 -33
  118. ert/resources/forward_models/run_reservoirsimulator.py +8 -3
  119. ert/resources/shell_scripts/delete_directory.py +2 -2
  120. ert/run_models/__init__.py +18 -5
  121. ert/run_models/_create_run_path.py +131 -37
  122. ert/run_models/ensemble_experiment.py +10 -4
  123. ert/run_models/ensemble_information_filter.py +8 -1
  124. ert/run_models/ensemble_smoother.py +9 -3
  125. ert/run_models/evaluate_ensemble.py +8 -6
  126. ert/run_models/event.py +7 -3
  127. ert/run_models/everest_run_model.py +159 -46
  128. ert/run_models/initial_ensemble_run_model.py +25 -24
  129. ert/run_models/manual_update.py +6 -3
  130. ert/run_models/manual_update_enif.py +37 -0
  131. ert/run_models/model_factory.py +81 -21
  132. ert/run_models/multiple_data_assimilation.py +22 -11
  133. ert/run_models/run_model.py +64 -55
  134. ert/run_models/single_test_run.py +7 -4
  135. ert/run_models/update_run_model.py +4 -2
  136. ert/runpaths.py +5 -6
  137. ert/sample_prior.py +9 -4
  138. ert/scheduler/driver.py +37 -0
  139. ert/scheduler/event.py +3 -1
  140. ert/scheduler/job.py +23 -13
  141. ert/scheduler/lsf_driver.py +6 -2
  142. ert/scheduler/openpbs_driver.py +7 -1
  143. ert/scheduler/scheduler.py +5 -0
  144. ert/scheduler/slurm_driver.py +6 -2
  145. ert/services/__init__.py +2 -2
  146. ert/services/_base_service.py +37 -20
  147. ert/services/ert_server.py +317 -0
  148. ert/shared/_doc_utils/__init__.py +4 -2
  149. ert/shared/_doc_utils/ert_jobs.py +1 -4
  150. ert/shared/net_utils.py +43 -18
  151. ert/shared/storage/connection.py +3 -3
  152. ert/shared/version.py +3 -3
  153. ert/storage/__init__.py +2 -0
  154. ert/storage/local_ensemble.py +38 -12
  155. ert/storage/local_experiment.py +8 -16
  156. ert/storage/local_storage.py +68 -42
  157. ert/storage/migration/to11.py +1 -1
  158. ert/storage/migration/to16.py +38 -0
  159. ert/storage/migration/to17.py +42 -0
  160. ert/storage/migration/to18.py +11 -0
  161. ert/storage/migration/to19.py +34 -0
  162. ert/storage/migration/to20.py +23 -0
  163. ert/storage/migration/to21.py +25 -0
  164. ert/storage/migration/to8.py +4 -4
  165. ert/substitutions.py +12 -28
  166. ert/validation/active_range.py +7 -7
  167. ert/validation/rangestring.py +16 -16
  168. ert/workflow_runner.py +2 -1
  169. {ert-17.0.0.dist-info → ert-19.0.0rc2.dist-info}/METADATA +9 -8
  170. {ert-17.0.0.dist-info → ert-19.0.0rc2.dist-info}/RECORD +208 -205
  171. {ert-17.0.0.dist-info → ert-19.0.0rc2.dist-info}/WHEEL +1 -1
  172. everest/api/everest_data_api.py +14 -1
  173. everest/bin/config_branch_script.py +3 -6
  174. everest/bin/everconfigdump_script.py +1 -9
  175. everest/bin/everest_script.py +21 -11
  176. everest/bin/everlint_script.py +0 -2
  177. everest/bin/kill_script.py +2 -2
  178. everest/bin/monitor_script.py +2 -2
  179. everest/bin/utils.py +8 -4
  180. everest/bin/visualization_script.py +6 -14
  181. everest/config/__init__.py +4 -1
  182. everest/config/control_config.py +81 -6
  183. everest/config/control_variable_config.py +4 -3
  184. everest/config/everest_config.py +75 -42
  185. everest/config/forward_model_config.py +5 -3
  186. everest/config/install_data_config.py +7 -5
  187. everest/config/install_job_config.py +7 -3
  188. everest/config/install_template_config.py +3 -3
  189. everest/config/optimization_config.py +19 -6
  190. everest/config/output_constraint_config.py +8 -2
  191. everest/config/server_config.py +6 -49
  192. everest/config/utils.py +25 -105
  193. everest/config/validation_utils.py +17 -11
  194. everest/config_file_loader.py +13 -4
  195. everest/detached/client.py +3 -3
  196. everest/detached/everserver.py +7 -8
  197. everest/everest_storage.py +6 -12
  198. everest/gui/everest_client.py +2 -3
  199. everest/gui/main_window.py +2 -2
  200. everest/optimizer/everest2ropt.py +59 -32
  201. everest/optimizer/opt_model_transforms.py +12 -13
  202. everest/optimizer/utils.py +0 -29
  203. everest/strings.py +0 -5
  204. ert/config/everest_constraints_config.py +0 -95
  205. ert/config/ext_param_config.py +0 -106
  206. ert/gui/tools/export/__init__.py +0 -3
  207. ert/gui/tools/export/export_panel.py +0 -83
  208. ert/gui/tools/export/export_tool.py +0 -69
  209. ert/gui/tools/export/exporter.py +0 -36
  210. ert/services/storage_service.py +0 -127
  211. everest/config/sampler_config.py +0 -103
  212. everest/simulator/__init__.py +0 -88
  213. everest/simulator/everest_to_ert.py +0 -51
  214. /ert/gui/{suggestor → ertwidgets/suggestor}/__init__.py +0 -0
  215. /ert/gui/{suggestor → ertwidgets/suggestor}/_colors.py +0 -0
  216. {ert-17.0.0.dist-info → ert-19.0.0rc2.dist-info}/entry_points.txt +0 -0
  217. {ert-17.0.0.dist-info → ert-19.0.0rc2.dist-info}/licenses/COPYING +0 -0
  218. {ert-17.0.0.dist-info → ert-19.0.0rc2.dist-info}/top_level.txt +0 -0
@@ -21,20 +21,28 @@ from ert.mode_definitions import (
21
21
  ENSEMBLE_SMOOTHER_MODE,
22
22
  ES_MDA_MODE,
23
23
  EVALUATE_ENSEMBLE_MODE,
24
+ MANUAL_ENIF_UPDATE_MODE,
24
25
  MANUAL_UPDATE_MODE,
25
26
  TEST_RUN_MODE,
26
27
  )
27
28
  from ert.validation import ActiveRange
28
29
 
29
- from .ensemble_experiment import EnsembleExperiment
30
- from .ensemble_information_filter import EnsembleInformationFilter
31
- from .ensemble_smoother import EnsembleSmoother
32
- from .evaluate_ensemble import EvaluateEnsemble
30
+ from .ensemble_experiment import EnsembleExperiment, EnsembleExperimentConfig
31
+ from .ensemble_information_filter import (
32
+ EnsembleInformationFilter,
33
+ EnsembleInformationFilterConfig,
34
+ )
35
+ from .ensemble_smoother import EnsembleSmoother, EnsembleSmootherConfig
36
+ from .evaluate_ensemble import EvaluateEnsemble, EvaluateEnsembleConfig
33
37
  from .initial_ensemble_run_model import DictEncodedDataFrame
34
- from .manual_update import ManualUpdate
35
- from .multiple_data_assimilation import MultipleDataAssimilation
38
+ from .manual_update import ManualUpdate, ManualUpdateConfig
39
+ from .manual_update_enif import ManualUpdateEnIF
40
+ from .multiple_data_assimilation import (
41
+ MultipleDataAssimilation,
42
+ MultipleDataAssimilationConfig,
43
+ )
36
44
  from .run_model import RunModel
37
- from .single_test_run import SingleTestRun
45
+ from .single_test_run import SingleTestRun, SingleTestRunConfig
38
46
 
39
47
  if TYPE_CHECKING:
40
48
  import numpy.typing as npt
@@ -78,6 +86,8 @@ def create_model(
78
86
  )
79
87
  if args.mode == MANUAL_UPDATE_MODE:
80
88
  return _setup_manual_update(config, args, update_settings, status_queue)
89
+ if args.mode == MANUAL_ENIF_UPDATE_MODE:
90
+ return _setup_manual_update_enif(config, args, update_settings, status_queue)
81
91
  raise NotImplementedError(f"Run type not supported {args.mode}")
82
92
 
83
93
 
@@ -122,7 +132,7 @@ def _setup_single_test_run(
122
132
  parameter_configs=config.ensemble_config.parameter_configuration,
123
133
  )
124
134
 
125
- return SingleTestRun(
135
+ runmodel_config = SingleTestRunConfig(
126
136
  random_seed=config.random_seed,
127
137
  runpath_file=config.runpath_file,
128
138
  active_realizations=[True],
@@ -144,6 +154,10 @@ def _setup_single_test_run(
144
154
  storage_path=config.ens_path,
145
155
  queue_config=config.queue_config.create_local_copy(),
146
156
  observations=config.observations,
157
+ )
158
+
159
+ return SingleTestRun(
160
+ **runmodel_config.model_dump(),
147
161
  status_queue=status_queue,
148
162
  )
149
163
 
@@ -177,7 +191,7 @@ def _setup_ensemble_experiment(
177
191
  parameter_configs=config.ensemble_config.parameter_configuration,
178
192
  )
179
193
 
180
- return EnsembleExperiment(
194
+ runmodel_config = EnsembleExperimentConfig(
181
195
  random_seed=config.random_seed,
182
196
  runpath_file=config.runpath_file,
183
197
  active_realizations=active_realizations,
@@ -199,6 +213,10 @@ def _setup_ensemble_experiment(
199
213
  storage_path=config.ens_path,
200
214
  queue_config=config.queue_config,
201
215
  observations=config.observations,
216
+ )
217
+
218
+ return EnsembleExperiment(
219
+ **runmodel_config.model_dump(),
202
220
  status_queue=status_queue,
203
221
  )
204
222
 
@@ -210,14 +228,13 @@ def _setup_evaluate_ensemble(
210
228
  ) -> EvaluateEnsemble:
211
229
  active_realizations = _get_and_validate_active_realizations_list(args, config)
212
230
  validate_minimum_realizations(config, active_realizations)
213
- return EvaluateEnsemble(
231
+ runmodel_config = EvaluateEnsembleConfig(
214
232
  random_seed=config.random_seed,
215
233
  active_realizations=active_realizations,
216
234
  ensemble_id=args.ensemble_id,
217
235
  minimum_required_realizations=config.analysis_config.minimum_required_realizations,
218
236
  storage_path=config.ens_path,
219
237
  queue_config=config.queue_config,
220
- status_queue=status_queue,
221
238
  runpath_file=config.runpath_file,
222
239
  user_config_file=Path(config.user_config_file),
223
240
  env_vars=config.env_vars,
@@ -228,6 +245,7 @@ def _setup_evaluate_ensemble(
228
245
  hooked_workflows=config.hooked_workflows,
229
246
  log_path=config.analysis_config.log_path,
230
247
  )
248
+ return EvaluateEnsemble(**runmodel_config.model_dump(), status_queue=status_queue)
231
249
 
232
250
 
233
251
  def _get_and_validate_active_realizations_list(
@@ -267,7 +285,7 @@ def _setup_manual_update(
267
285
  active_realizations = _realizations(args, config.runpath_config.num_realizations)
268
286
  validate_minimum_realizations(config, active_realizations.tolist())
269
287
 
270
- return ManualUpdate(
288
+ runmodel_config = ManualUpdateConfig(
271
289
  random_seed=config.random_seed,
272
290
  active_realizations=active_realizations.tolist(),
273
291
  ensemble_id=args.ensemble_id,
@@ -277,8 +295,46 @@ def _setup_manual_update(
277
295
  queue_config=config.queue_config,
278
296
  analysis_settings=config.analysis_config.es_settings,
279
297
  update_settings=update_settings,
298
+ runpath_file=config.runpath_file,
299
+ user_config_file=Path(config.user_config_file),
300
+ env_vars=config.env_vars,
301
+ env_pr_fm_step=config.env_pr_fm_step,
302
+ runpath_config=config.runpath_config,
303
+ forward_model_steps=config.forward_model_steps,
304
+ substitutions=config.substitutions,
305
+ hooked_workflows=config.hooked_workflows,
306
+ log_path=config.analysis_config.log_path,
307
+ ert_templates=config.ert_templates,
308
+ observations=config.observations,
309
+ )
310
+ return ManualUpdate(**runmodel_config.model_dump(), status_queue=status_queue)
311
+
312
+
313
+ def _setup_manual_update_enif(
314
+ config: ErtConfig,
315
+ args: Namespace,
316
+ update_settings: ObservationSettings,
317
+ status_queue: SimpleQueue[StatusEvents],
318
+ ) -> ManualUpdate:
319
+ active_realizations = _realizations(args, config.runpath_config.num_realizations)
320
+
321
+ return ManualUpdateEnIF(
322
+ random_seed=config.random_seed,
323
+ active_realizations=active_realizations.tolist(),
324
+ ensemble_id=args.ensemble_id,
325
+ minimum_required_realizations=config.analysis_config.minimum_required_realizations,
326
+ target_ensemble=args.target_ensemble,
327
+ config=config,
328
+ storage_path=config.ens_path,
329
+ queue_config=config.queue_config,
330
+ analysis_settings=config.analysis_config.es_settings,
331
+ update_settings=update_settings,
280
332
  status_queue=status_queue,
281
333
  runpath_file=config.runpath_file,
334
+ design_matrix=config.analysis_config.design_matrix,
335
+ parameter_configuration=config.ensemble_config.parameter_configuration,
336
+ response_configuration=config.ensemble_config.response_configuration,
337
+ ert_templates=config.ert_templates,
282
338
  user_config_file=Path(config.user_config_file),
283
339
  env_vars=config.env_vars,
284
340
  env_pr_fm_step=config.env_pr_fm_step,
@@ -299,7 +355,7 @@ def _setup_ensemble_smoother(
299
355
  ) -> EnsembleSmoother:
300
356
  active_realizations = _get_and_validate_active_realizations_list(args, config)
301
357
  validate_minimum_realizations(config, active_realizations)
302
- if len(active_realizations) < 2:
358
+ if sum(active_realizations) < 2:
303
359
  raise ConfigValidationError(
304
360
  "Number of active realizations must be at least 2 for an update step"
305
361
  )
@@ -310,7 +366,7 @@ def _setup_ensemble_smoother(
310
366
  require_updateable_param=True,
311
367
  )
312
368
 
313
- return EnsembleSmoother(
369
+ runmodel_config = EnsembleSmootherConfig(
314
370
  target_ensemble=args.target_ensemble,
315
371
  experiment_name=getattr(args, "experiment_name", ""),
316
372
  active_realizations=active_realizations,
@@ -320,7 +376,6 @@ def _setup_ensemble_smoother(
320
376
  queue_config=config.queue_config,
321
377
  analysis_settings=config.analysis_config.es_settings,
322
378
  update_settings=update_settings,
323
- status_queue=status_queue,
324
379
  runpath_file=config.runpath_file,
325
380
  design_matrix=design_matrix,
326
381
  parameter_configuration=parameter_configs,
@@ -336,6 +391,7 @@ def _setup_ensemble_smoother(
336
391
  log_path=config.analysis_config.log_path,
337
392
  observations=config.observations,
338
393
  )
394
+ return EnsembleSmoother(**runmodel_config.model_dump(), status_queue=status_queue)
339
395
 
340
396
 
341
397
  def _setup_ensemble_information_filter(
@@ -346,7 +402,7 @@ def _setup_ensemble_information_filter(
346
402
  ) -> EnsembleInformationFilter:
347
403
  active_realizations = _get_and_validate_active_realizations_list(args, config)
348
404
  validate_minimum_realizations(config, active_realizations)
349
- if len(active_realizations) < 2:
405
+ if sum(active_realizations) < 2:
350
406
  raise ConfigValidationError(
351
407
  "Number of active realizations must be at least 2 for an update step"
352
408
  )
@@ -356,7 +412,7 @@ def _setup_ensemble_information_filter(
356
412
  parameter_configs=config.ensemble_config.parameter_configuration,
357
413
  )
358
414
 
359
- return EnsembleInformationFilter(
415
+ runmodel_config = EnsembleInformationFilterConfig(
360
416
  target_ensemble=args.target_ensemble,
361
417
  experiment_name=getattr(args, "experiment_name", ""),
362
418
  active_realizations=active_realizations,
@@ -366,7 +422,6 @@ def _setup_ensemble_information_filter(
366
422
  queue_config=config.queue_config,
367
423
  analysis_settings=config.analysis_config.es_settings,
368
424
  update_settings=update_settings,
369
- status_queue=status_queue,
370
425
  runpath_file=config.runpath_file,
371
426
  design_matrix=design_matrix,
372
427
  parameter_configuration=parameter_configs,
@@ -382,6 +437,9 @@ def _setup_ensemble_information_filter(
382
437
  log_path=config.analysis_config.log_path,
383
438
  observations=config.observations,
384
439
  )
440
+ return EnsembleInformationFilter(
441
+ **runmodel_config.model_dump(), status_queue=status_queue
442
+ )
385
443
 
386
444
 
387
445
  def _determine_restart_info(args: Namespace) -> tuple[bool, str | None]:
@@ -412,7 +470,7 @@ def _setup_multiple_data_assimilation(
412
470
  restart_run, prior_ensemble = _determine_restart_info(args)
413
471
  active_realizations = _get_and_validate_active_realizations_list(args, config)
414
472
  validate_minimum_realizations(config, active_realizations)
415
- if len(active_realizations) < 2:
473
+ if sum(active_realizations) < 2:
416
474
  raise ConfigValidationError(
417
475
  "Number of active realizations must be at least 2 for an update step"
418
476
  )
@@ -423,7 +481,7 @@ def _setup_multiple_data_assimilation(
423
481
  require_updateable_param=True,
424
482
  )
425
483
 
426
- return MultipleDataAssimilation(
484
+ runmodel_config = MultipleDataAssimilationConfig(
427
485
  random_seed=config.random_seed,
428
486
  active_realizations=active_realizations,
429
487
  target_ensemble=_iterative_ensemble_format(args),
@@ -434,7 +492,6 @@ def _setup_multiple_data_assimilation(
434
492
  experiment_name=args.experiment_name,
435
493
  queue_config=config.queue_config,
436
494
  update_settings=update_settings,
437
- status_queue=status_queue,
438
495
  storage_path=config.ens_path,
439
496
  analysis_settings=config.analysis_config.es_settings,
440
497
  runpath_file=config.runpath_file,
@@ -452,6 +509,9 @@ def _setup_multiple_data_assimilation(
452
509
  log_path=config.analysis_config.log_path,
453
510
  observations=config.observations,
454
511
  )
512
+ return MultipleDataAssimilation(
513
+ **runmodel_config.model_dump(), status_queue=status_queue
514
+ )
455
515
 
456
516
 
457
517
  def _realizations(
@@ -14,8 +14,11 @@ from ert.config import (
14
14
  ResponseConfig,
15
15
  )
16
16
  from ert.ensemble_evaluator import EvaluatorServerConfig
17
- from ert.run_models.initial_ensemble_run_model import InitialEnsembleRunModel
18
- from ert.run_models.update_run_model import UpdateRunModel
17
+ from ert.run_models.initial_ensemble_run_model import (
18
+ InitialEnsembleRunModel,
19
+ InitialEnsembleRunModelConfig,
20
+ )
21
+ from ert.run_models.update_run_model import UpdateRunModel, UpdateRunModelConfig
19
22
  from ert.storage import Ensemble
20
23
  from ert.trace import tracer
21
24
 
@@ -28,18 +31,25 @@ logger = logging.getLogger(__name__)
28
31
  MULTIPLE_DATA_ASSIMILATION_GROUP = "Parameter update"
29
32
 
30
33
 
31
- class MultipleDataAssimilation(UpdateRunModel, InitialEnsembleRunModel):
32
- """
33
- Run multiple data assimilation (MDA) ensemble smoother with custom weights.
34
- """
35
-
34
+ class MultipleDataAssimilationConfig(
35
+ InitialEnsembleRunModelConfig, UpdateRunModelConfig
36
+ ):
36
37
  default_weights: ClassVar[str] = "4, 2, 1"
37
38
  restart_run: bool
38
39
  prior_ensemble_id: str | None
39
40
  weights: str
40
41
 
42
+
43
+ class MultipleDataAssimilation(
44
+ UpdateRunModel, InitialEnsembleRunModel, MultipleDataAssimilationConfig
45
+ ):
46
+ """
47
+ Run multiple data assimilation (MDA) ensemble smoother with custom weights.
48
+ """
49
+
41
50
  _parsed_weights: list[float] = PrivateAttr()
42
51
  _total_iterations: int = PrivateAttr(default=2)
52
+ _start_iteration: int = PrivateAttr(default=0)
43
53
 
44
54
  def model_post_init(self, ctx: Any) -> None:
45
55
  super().model_post_init(ctx)
@@ -56,7 +66,7 @@ class MultipleDataAssimilation(UpdateRunModel, InitialEnsembleRunModel):
56
66
  elif not self.experiment_name:
57
67
  raise ValueError("For non-restart run, experiment name must be set")
58
68
 
59
- self.start_iteration = start_iteration
69
+ self._start_iteration = start_iteration
60
70
  self._total_iterations = total_iterations
61
71
 
62
72
  @tracer.start_as_current_span(f"{__name__}.run_experiment")
@@ -80,10 +90,10 @@ class MultipleDataAssimilation(UpdateRunModel, InitialEnsembleRunModel):
80
90
  self.set_env_key("_ERT_EXPERIMENT_ID", str(experiment.id))
81
91
  self.set_env_key("_ERT_ENSEMBLE_ID", str(prior.id))
82
92
  assert isinstance(prior, Ensemble)
83
- if self.start_iteration != prior.iteration + 1:
93
+ if self._start_iteration != prior.iteration + 1:
84
94
  raise ValueError(
85
95
  "Experiment misconfigured, got starting "
86
- f"iteration: {self.start_iteration},"
96
+ f"iteration: {self._start_iteration},"
87
97
  f"restart iteration = {prior.iteration + 1}"
88
98
  )
89
99
  target_experiment = self._storage.create_experiment(
@@ -92,8 +102,9 @@ class MultipleDataAssimilation(UpdateRunModel, InitialEnsembleRunModel):
92
102
  observations=prior.experiment.observations,
93
103
  simulation_arguments=prior.experiment.metadata,
94
104
  name=f"Restart from {prior.name}",
95
- templates=prior.experiment.templates_configuration,
105
+ templates=self.ert_templates,
96
106
  )
107
+
97
108
  except (KeyError, ValueError) as err:
98
109
  raise ErtRunError(
99
110
  f"Prior ensemble with ID: {id_} does not exists"
@@ -3,20 +3,21 @@ from __future__ import annotations
3
3
  import asyncio
4
4
  import concurrent.futures
5
5
  import copy
6
- import dataclasses
7
6
  import functools
8
7
  import logging
9
8
  import os
10
9
  import queue
11
10
  import shutil
12
11
  import threading
13
- import time
14
12
  import traceback
15
13
  import uuid
14
+ import warnings
16
15
  from abc import ABC, abstractmethod
17
16
  from collections import defaultdict
18
17
  from collections.abc import Callable, Generator, MutableSequence
19
18
  from contextlib import contextmanager
19
+ from dataclasses import dataclass
20
+ from datetime import datetime
20
21
  from pathlib import Path
21
22
  from typing import TYPE_CHECKING, Any, ClassVar, Protocol, cast
22
23
 
@@ -24,8 +25,12 @@ import numpy as np
24
25
  from pydantic import PrivateAttr, field_validator
25
26
  from pydantic_core.core_schema import ValidationInfo
26
27
 
27
- from _ert.events import EEEvent, EESnapshot, EESnapshotUpdate
28
- from ert.base_model_context import BaseModelWithContextSupport
28
+ from _ert.events import (
29
+ EEEvent,
30
+ EESnapshot,
31
+ EESnapshotUpdate,
32
+ EnsembleEvaluationWarning,
33
+ )
29
34
  from ert.config import (
30
35
  ConfigValidationError,
31
36
  DesignMatrix,
@@ -37,7 +42,6 @@ from ert.config import (
37
42
  PostSimulationFixtures,
38
43
  PreSimulationFixtures,
39
44
  QueueConfig,
40
- QueueSystem,
41
45
  Workflow,
42
46
  create_workflow_fixtures_from_hooked,
43
47
  )
@@ -49,7 +53,7 @@ from ert.ensemble_evaluator import (
49
53
  Realization,
50
54
  WarningEvent,
51
55
  )
52
- from ert.ensemble_evaluator.evaluator import UserCancelled
56
+ from ert.ensemble_evaluator.evaluator import ParallelismViolation, UserCancelled
53
57
  from ert.ensemble_evaluator.snapshot import EnsembleSnapshot
54
58
  from ert.ensemble_evaluator.state import (
55
59
  REALIZATION_STATE_FAILED,
@@ -59,6 +63,7 @@ from ert.mode_definitions import MODULE_MODE
59
63
  from ert.runpaths import Runpaths
60
64
  from ert.storage import (
61
65
  Ensemble,
66
+ LocalStorage,
62
67
  Storage,
63
68
  open_storage,
64
69
  )
@@ -67,9 +72,16 @@ from ert.utils import log_duration
67
72
  from ert.warnings import PostSimulationWarning, capture_specific_warning
68
73
  from ert.workflow_runner import WorkflowRunner
69
74
 
75
+ from ..base_model_context import BaseModelWithContextSupport
70
76
  from ..run_arg import RunArg
71
77
  from ._create_run_path import create_run_path
72
- from .event import EndEvent, FullSnapshotEvent, SnapshotUpdateEvent, StatusEvents
78
+ from .event import (
79
+ EndEvent,
80
+ FullSnapshotEvent,
81
+ SnapshotUpdateEvent,
82
+ StartEvent,
83
+ StatusEvents,
84
+ )
73
85
 
74
86
  if TYPE_CHECKING:
75
87
  from ert.plugins import ErtRuntimePlugins
@@ -137,17 +149,16 @@ class StartSimulationsThreadFn(Protocol):
137
149
  ) -> None: ...
138
150
 
139
151
 
140
- @dataclasses.dataclass
152
+ @dataclass
141
153
  class RunModelAPI:
142
154
  experiment_name: str
143
155
  supports_rerunning_failed_realizations: bool
144
156
  start_simulations_thread: StartSimulationsThreadFn
145
157
  cancel: Callable[[], None]
146
- get_runtime: Callable[[], int]
147
158
  has_failed_realizations: Callable[[], bool]
148
159
 
149
160
 
150
- class RunModel(BaseModelWithContextSupport, ABC):
161
+ class RunModelConfig(BaseModelWithContextSupport):
151
162
  storage_path: str
152
163
  runpath_file: Path
153
164
  user_config_file: Path
@@ -165,9 +176,9 @@ class RunModel(BaseModelWithContextSupport, ABC):
165
176
  minimum_required_realizations: int = 0
166
177
  supports_rerunning_failed_realizations: ClassVar[bool] = False
167
178
 
179
+
180
+ class RunModel(RunModelConfig, ABC):
168
181
  # Private attributes initialized in model_post_init
169
- _start_time: int | None = PrivateAttr(None)
170
- _stop_time: int | None = PrivateAttr(None)
171
182
  _initial_realizations_mask: list[bool] = PrivateAttr()
172
183
  _completed_realizations_mask: list[bool] = PrivateAttr(default_factory=list)
173
184
  _storage: Storage = PrivateAttr()
@@ -179,6 +190,8 @@ class RunModel(BaseModelWithContextSupport, ABC):
179
190
  _is_rerunning_failed_realizations: bool = PrivateAttr(False)
180
191
  _run_paths: Runpaths = PrivateAttr()
181
192
  _total_iterations: int = PrivateAttr(default=1)
193
+ _start_iteration: int = PrivateAttr(default=0)
194
+ _max_parallelism_violation: ParallelismViolation = ParallelismViolation()
182
195
 
183
196
  def __init__(
184
197
  self,
@@ -196,23 +209,26 @@ class RunModel(BaseModelWithContextSupport, ABC):
196
209
  def model_post_init(self, ctx: Any) -> None:
197
210
  self._initial_realizations_mask = self.active_realizations.copy()
198
211
  self._completed_realizations_mask = [False] * len(self.active_realizations)
212
+
213
+ if LocalStorage.check_migration_needed(Path(self.storage_path)):
214
+ LocalStorage.perform_migration(Path(self.storage_path))
215
+
199
216
  self._storage = open_storage(self.storage_path, mode="w")
200
217
  self._rng = np.random.default_rng(self.random_seed)
201
- self._model_config = self.runpath_config
218
+ self._start_iteration = self.start_iteration
202
219
 
203
220
  self._run_paths = Runpaths(
204
- jobname_format=self._model_config.jobname_format_string,
205
- runpath_format=self._model_config.runpath_format_string,
221
+ jobname_format=self.runpath_config.jobname_format_string,
222
+ runpath_format=self.runpath_config.runpath_format_string,
206
223
  filename=str(self.runpath_file),
207
224
  substitutions=self.substitutions,
208
- eclbase=self._model_config.eclbase_format_string,
225
+ eclbase=self.runpath_config.eclbase_format_string,
209
226
  )
210
227
 
211
228
  @property
212
229
  def api(self) -> RunModelAPI:
213
230
  return RunModelAPI(
214
231
  experiment_name=self.name(),
215
- get_runtime=self.get_runtime,
216
232
  start_simulations_thread=self.start_simulations_thread,
217
233
  has_failed_realizations=self.has_failed_realizations,
218
234
  supports_rerunning_failed_realizations=self.supports_rerunning_failed_realizations,
@@ -237,28 +253,8 @@ class RunModel(BaseModelWithContextSupport, ABC):
237
253
  for key, value in self.__dict__.items()
238
254
  if key not in keys_to_drop
239
255
  }
240
- settings_summary = {
241
- "run_model": self.name(),
242
- "num_realizations": self.runpath_config.num_realizations,
243
- "num_active_realizations": self.active_realizations.count(True),
244
- "num_parameters": (
245
- sum(
246
- len(param_config.parameter_keys)
247
- for param_config in self.parameter_configuration
248
- )
249
- if hasattr(self, "parameter_configuration")
250
- else "NA"
251
- ),
252
- "localization": getattr(
253
- settings_dict.get("analysis_settings", {}), "localization", "NA"
254
- ),
255
- }
256
256
 
257
- logger.info(
258
- f"Running '{self.name()}'\n\n"
259
- f"Settings summary: {settings_summary}\n\n"
260
- f"Settings: {settings_dict}"
261
- )
257
+ logger.info(f"Running '{self.name()}'\n\nSettings: {settings_dict}")
262
258
 
263
259
  @field_validator("env_vars", mode="after")
264
260
  @classmethod
@@ -330,10 +326,6 @@ class RunModel(BaseModelWithContextSupport, ABC):
330
326
  def send_event(self, event: StatusEvents) -> None:
331
327
  self._status_queue.put(event)
332
328
 
333
- @property
334
- def queue_system(self) -> QueueSystem:
335
- return self.queue_config.queue_system
336
-
337
329
  @property
338
330
  def ensemble_size(self) -> int:
339
331
  return len(self._initial_realizations_mask)
@@ -396,9 +388,9 @@ class RunModel(BaseModelWithContextSupport, ABC):
396
388
  def handle_captured_event(message: Warning | str) -> None:
397
389
  self.send_event(WarningEvent(msg=str(message)))
398
390
 
391
+ start_timestamp = datetime.now()
399
392
  try:
400
- self._start_time = int(time.time())
401
- self._stop_time = None
393
+ self.send_event(StartEvent(timestamp=start_timestamp))
402
394
  with (
403
395
  capture_specific_warning(PostSimulationWarning, handle_captured_event),
404
396
  captured_logs(error_messages),
@@ -453,7 +445,6 @@ class RunModel(BaseModelWithContextSupport, ABC):
453
445
  finally:
454
446
  self._storage.close()
455
447
  self._clean_env_context()
456
- self._stop_time = int(time.time())
457
448
  self.send_event(
458
449
  EndEvent(
459
450
  failed=failed,
@@ -468,6 +459,10 @@ class RunModel(BaseModelWithContextSupport, ABC):
468
459
  ),
469
460
  )
470
461
  )
462
+ logger.info(
463
+ "Experiment run finished in: "
464
+ f"{(datetime.now() - start_timestamp).total_seconds():g}s"
465
+ )
471
466
 
472
467
  @abstractmethod
473
468
  def run_experiment(
@@ -489,13 +484,6 @@ class RunModel(BaseModelWithContextSupport, ABC):
489
484
  return f"{exception}\n{msg}"
490
485
  return f"{exception}\n{traceback}\n{msg}"
491
486
 
492
- def get_runtime(self) -> int:
493
- if self._start_time is None:
494
- return 0
495
- elif self._stop_time is None:
496
- return round(time.time() - self._start_time)
497
- return self._stop_time - self._start_time
498
-
499
487
  def get_current_status(self) -> dict[str, int]:
500
488
  status: dict[str, int] = defaultdict(int)
501
489
  if self._iter_snapshot.keys():
@@ -609,6 +597,8 @@ class RunModel(BaseModelWithContextSupport, ABC):
609
597
  snapshot=copy.deepcopy(snapshot),
610
598
  )
611
599
  )
600
+ elif type(event) is EnsembleEvaluationWarning:
601
+ self.send_event(event)
612
602
 
613
603
  async def run_ensemble_evaluator_async(
614
604
  self,
@@ -639,6 +629,10 @@ class RunModel(BaseModelWithContextSupport, ABC):
639
629
  finally:
640
630
  await evaluator_task
641
631
 
632
+ self._max_parallelism_violation = max(
633
+ self._max_parallelism_violation, evaluator.max_parallelism_violation
634
+ )
635
+
642
636
  logger.debug("tasks complete")
643
637
 
644
638
  if self._end_event.is_set():
@@ -704,7 +698,8 @@ class RunModel(BaseModelWithContextSupport, ABC):
704
698
  run_paths = []
705
699
  active_realizations = np.where(self.active_realizations)[0]
706
700
  for iteration in range(
707
- self.start_iteration, self._total_iterations + self.start_iteration
701
+ self._start_iteration,
702
+ self._total_iterations + self._start_iteration,
708
703
  ):
709
704
  run_paths.extend(self._run_paths.get_paths(active_realizations, iteration))
710
705
  return run_paths
@@ -772,7 +767,7 @@ class RunModel(BaseModelWithContextSupport, ABC):
772
767
  env_pr_fm_step=self.env_pr_fm_step,
773
768
  forward_model_steps=self.forward_model_steps,
774
769
  substitutions=self.substitutions,
775
- parameters_file=self._model_config.gen_kw_export_name,
770
+ parameters_file=self.runpath_config.gen_kw_export_name,
776
771
  runpaths=self._run_paths,
777
772
  context_env=self._context_env,
778
773
  )
@@ -818,7 +813,21 @@ class RunModel(BaseModelWithContextSupport, ABC):
818
813
  f"Experiment run ended with number of realizations failing: "
819
814
  f"{len(starting_realizations) - num_successful_realizations}"
820
815
  )
821
- logger.info(f"Experiment run finished in: {self.get_runtime()}s")
816
+
817
+ # 0 means no violation
818
+ if self._max_parallelism_violation.amount > 0 and isinstance(
819
+ self._max_parallelism_violation.message, str
820
+ ):
821
+ logger.info(
822
+ "Warning displayed to user for NUM_CPU misconfiguration:\n"
823
+ f"{self._max_parallelism_violation.message}"
824
+ )
825
+ warnings.warn(
826
+ self._max_parallelism_violation.message,
827
+ PostSimulationWarning,
828
+ stacklevel=1,
829
+ )
830
+
822
831
  self.run_workflows(
823
832
  fixtures=PostSimulationFixtures(
824
833
  storage=self._storage,
@@ -3,11 +3,17 @@ from __future__ import annotations
3
3
  from pydantic import Field
4
4
 
5
5
  from ert.run_models import EnsembleExperiment
6
+ from ert.run_models.ensemble_experiment import EnsembleExperimentConfig
6
7
 
7
8
  SINGLE_TEST_RUN_GROUP = "Forward model evaluation"
8
9
 
9
10
 
10
- class SingleTestRun(EnsembleExperiment):
11
+ class SingleTestRunConfig(EnsembleExperimentConfig):
12
+ active_realizations: list[bool] = Field(default_factory=lambda: [True])
13
+ minimum_required_realizations: int = 1
14
+
15
+
16
+ class SingleTestRun(EnsembleExperiment, SingleTestRunConfig):
11
17
  """
12
18
  Single test is equivalent to EnsembleExperiment, in that it
13
19
  samples the prior and evaluates it.<br>There are two key differences:<br>
@@ -15,9 +21,6 @@ class SingleTestRun(EnsembleExperiment):
15
21
  2) Only a <b>single realization</b> (realization-0) is run<br>
16
22
  """
17
23
 
18
- active_realizations: list[bool] = Field(default_factory=lambda: [True])
19
- minimum_required_realizations: int = 1
20
-
21
24
  @classmethod
22
25
  def name(cls) -> str:
23
26
  return "Single realization test-run"
@@ -27,15 +27,17 @@ from ert.run_models.event import (
27
27
  RunModelUpdateBeginEvent,
28
28
  RunModelUpdateEndEvent,
29
29
  )
30
- from ert.run_models.run_model import ErtRunError, RunModel
30
+ from ert.run_models.run_model import ErtRunError, RunModel, RunModelConfig
31
31
  from ert.storage import Ensemble, LocalExperiment
32
32
 
33
33
 
34
- class UpdateRunModel(RunModel):
34
+ class UpdateRunModelConfig(RunModelConfig):
35
35
  target_ensemble: str
36
36
  analysis_settings: ESSettings
37
37
  update_settings: ObservationSettings
38
38
 
39
+
40
+ class UpdateRunModel(RunModel, UpdateRunModelConfig):
39
41
  @abstractmethod
40
42
  def update_ensemble_parameters(
41
43
  self, prior: Ensemble, posterior: Ensemble, weight: float