ert 16.0.9__py3-none-any.whl → 19.0.0rc2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (286) hide show
  1. _ert/events.py +19 -2
  2. _ert/forward_model_runner/client.py +6 -2
  3. _ert/forward_model_runner/fm_dispatch.py +9 -6
  4. _ert/forward_model_runner/reporting/event.py +1 -0
  5. _ert/forward_model_runner/runner.py +1 -2
  6. _ert/utils.py +12 -0
  7. ert/__main__.py +58 -38
  8. ert/analysis/_enif_update.py +8 -4
  9. ert/analysis/_es_update.py +19 -6
  10. ert/analysis/_update_commons.py +16 -6
  11. ert/base_model_context.py +1 -1
  12. ert/cli/main.py +17 -12
  13. ert/cli/monitor.py +7 -0
  14. ert/config/__init__.py +17 -6
  15. ert/config/_create_observation_dataframes.py +118 -21
  16. ert/config/_get_num_cpu.py +1 -1
  17. ert/config/_observations.py +91 -2
  18. ert/config/_read_summary.py +74 -328
  19. ert/config/design_matrix.py +62 -23
  20. ert/config/distribution.py +1 -1
  21. ert/config/ensemble_config.py +9 -17
  22. ert/config/ert_config.py +155 -58
  23. ert/config/everest_control.py +234 -0
  24. ert/config/{everest_constraints_config.py → everest_response.py} +27 -15
  25. ert/config/field.py +99 -90
  26. ert/config/forward_model_step.py +122 -17
  27. ert/config/gen_data_config.py +5 -10
  28. ert/config/gen_kw_config.py +11 -41
  29. ert/config/known_response_types.py +14 -0
  30. ert/config/parameter_config.py +1 -33
  31. ert/config/parsing/_option_dict.py +10 -2
  32. ert/config/parsing/config_errors.py +1 -1
  33. ert/config/parsing/config_keywords.py +2 -1
  34. ert/config/parsing/config_schema.py +23 -11
  35. ert/config/parsing/config_schema_deprecations.py +3 -3
  36. ert/config/parsing/config_schema_item.py +26 -11
  37. ert/config/parsing/context_values.py +3 -3
  38. ert/config/parsing/file_context_token.py +1 -1
  39. ert/config/parsing/observations_parser.py +6 -2
  40. ert/config/parsing/queue_system.py +9 -0
  41. ert/config/parsing/schema_item_type.py +1 -0
  42. ert/config/queue_config.py +42 -50
  43. ert/config/response_config.py +0 -8
  44. ert/config/rft_config.py +275 -0
  45. ert/config/summary_config.py +3 -8
  46. ert/config/surface_config.py +73 -26
  47. ert/config/workflow_fixtures.py +2 -1
  48. ert/config/workflow_job.py +135 -54
  49. ert/dark_storage/client/__init__.py +2 -2
  50. ert/dark_storage/client/_session.py +4 -4
  51. ert/dark_storage/client/client.py +2 -2
  52. ert/dark_storage/common.py +12 -3
  53. ert/dark_storage/compute/misfits.py +11 -7
  54. ert/dark_storage/endpoints/compute/misfits.py +6 -4
  55. ert/dark_storage/endpoints/ensembles.py +4 -0
  56. ert/dark_storage/endpoints/experiment_server.py +30 -24
  57. ert/dark_storage/endpoints/experiments.py +2 -2
  58. ert/dark_storage/endpoints/observations.py +8 -6
  59. ert/dark_storage/endpoints/parameters.py +4 -12
  60. ert/dark_storage/endpoints/responses.py +24 -5
  61. ert/dark_storage/json_schema/ensemble.py +3 -0
  62. ert/dark_storage/json_schema/experiment.py +1 -1
  63. ert/data/_measured_data.py +6 -5
  64. ert/ensemble_evaluator/__init__.py +8 -1
  65. ert/ensemble_evaluator/config.py +2 -1
  66. ert/ensemble_evaluator/evaluator.py +81 -29
  67. ert/ensemble_evaluator/event.py +6 -0
  68. ert/ensemble_evaluator/snapshot.py +3 -1
  69. ert/ensemble_evaluator/state.py +1 -0
  70. ert/field_utils/__init__.py +8 -0
  71. ert/field_utils/field_utils.py +228 -15
  72. ert/field_utils/grdecl_io.py +1 -1
  73. ert/field_utils/roff_io.py +1 -1
  74. ert/gui/__init__.py +5 -2
  75. ert/gui/ertnotifier.py +1 -1
  76. ert/gui/ertwidgets/__init__.py +23 -16
  77. ert/gui/ertwidgets/analysismoduleedit.py +2 -2
  78. ert/gui/ertwidgets/checklist.py +1 -1
  79. ert/gui/ertwidgets/closabledialog.py +2 -0
  80. ert/gui/ertwidgets/copyablelabel.py +2 -0
  81. ert/gui/ertwidgets/create_experiment_dialog.py +3 -1
  82. ert/gui/ertwidgets/ensembleselector.py +2 -2
  83. ert/gui/ertwidgets/listeditbox.py +2 -0
  84. ert/gui/ertwidgets/models/__init__.py +2 -0
  85. ert/gui/ertwidgets/models/activerealizationsmodel.py +5 -1
  86. ert/gui/ertwidgets/models/path_model.py +1 -1
  87. ert/gui/ertwidgets/models/targetensemblemodel.py +5 -1
  88. ert/gui/ertwidgets/models/text_model.py +4 -1
  89. ert/gui/ertwidgets/pathchooser.py +0 -3
  90. ert/gui/ertwidgets/searchbox.py +17 -4
  91. ert/gui/ertwidgets/stringbox.py +2 -0
  92. ert/gui/{suggestor → ertwidgets/suggestor}/_suggestor_message.py +13 -4
  93. ert/gui/{suggestor → ertwidgets/suggestor}/suggestor.py +63 -30
  94. ert/gui/main.py +41 -13
  95. ert/gui/main_window.py +3 -7
  96. ert/gui/model/fm_step_list.py +3 -0
  97. ert/gui/model/real_list.py +1 -0
  98. ert/gui/model/snapshot.py +1 -0
  99. ert/gui/simulation/combobox_with_description.py +3 -0
  100. ert/gui/simulation/ensemble_experiment_panel.py +8 -2
  101. ert/gui/simulation/ensemble_information_filter_panel.py +7 -2
  102. ert/gui/simulation/ensemble_smoother_panel.py +8 -2
  103. ert/gui/simulation/evaluate_ensemble_panel.py +17 -7
  104. ert/gui/simulation/experiment_panel.py +18 -6
  105. ert/gui/simulation/manual_update_panel.py +35 -10
  106. ert/gui/simulation/multiple_data_assimilation_panel.py +13 -9
  107. ert/gui/simulation/run_dialog.py +47 -20
  108. ert/gui/simulation/single_test_run_panel.py +6 -3
  109. ert/gui/simulation/view/progress_widget.py +2 -0
  110. ert/gui/simulation/view/realization.py +5 -1
  111. ert/gui/simulation/view/update.py +2 -0
  112. ert/gui/summarypanel.py +20 -1
  113. ert/gui/tools/event_viewer/panel.py +3 -4
  114. ert/gui/tools/event_viewer/tool.py +2 -0
  115. ert/gui/tools/load_results/load_results_panel.py +1 -1
  116. ert/gui/tools/load_results/load_results_tool.py +2 -0
  117. ert/gui/tools/manage_experiments/export_dialog.py +136 -0
  118. ert/gui/tools/manage_experiments/manage_experiments_panel.py +2 -0
  119. ert/gui/tools/manage_experiments/storage_info_widget.py +121 -16
  120. ert/gui/tools/manage_experiments/storage_widget.py +4 -3
  121. ert/gui/tools/plot/customize/color_chooser.py +5 -2
  122. ert/gui/tools/plot/customize/customize_plot_dialog.py +2 -0
  123. ert/gui/tools/plot/customize/default_customization_view.py +4 -0
  124. ert/gui/tools/plot/customize/limits_customization_view.py +3 -0
  125. ert/gui/tools/plot/customize/statistics_customization_view.py +3 -0
  126. ert/gui/tools/plot/customize/style_chooser.py +2 -0
  127. ert/gui/tools/plot/customize/style_customization_view.py +3 -0
  128. ert/gui/tools/plot/data_type_keys_widget.py +2 -0
  129. ert/gui/tools/plot/data_type_proxy_model.py +3 -0
  130. ert/gui/tools/plot/plot_api.py +50 -28
  131. ert/gui/tools/plot/plot_ensemble_selection_widget.py +17 -10
  132. ert/gui/tools/plot/plot_widget.py +15 -2
  133. ert/gui/tools/plot/plot_window.py +41 -19
  134. ert/gui/tools/plot/plottery/plot_config.py +2 -0
  135. ert/gui/tools/plot/plottery/plot_context.py +14 -0
  136. ert/gui/tools/plot/plottery/plots/__init__.py +2 -0
  137. ert/gui/tools/plot/plottery/plots/cesp.py +3 -1
  138. ert/gui/tools/plot/plottery/plots/distribution.py +6 -1
  139. ert/gui/tools/plot/plottery/plots/ensemble.py +13 -5
  140. ert/gui/tools/plot/plottery/plots/gaussian_kde.py +12 -2
  141. ert/gui/tools/plot/plottery/plots/histogram.py +3 -1
  142. ert/gui/tools/plot/plottery/plots/misfits.py +436 -0
  143. ert/gui/tools/plot/plottery/plots/observations.py +18 -4
  144. ert/gui/tools/plot/plottery/plots/statistics.py +62 -20
  145. ert/gui/tools/plot/plottery/plots/std_dev.py +3 -1
  146. ert/gui/tools/plot/widgets/clearable_line_edit.py +9 -0
  147. ert/gui/tools/plot/widgets/filter_popup.py +2 -0
  148. ert/gui/tools/plot/widgets/filterable_kw_list_model.py +3 -0
  149. ert/gui/tools/plugins/plugin.py +1 -1
  150. ert/gui/tools/plugins/plugins_tool.py +2 -0
  151. ert/gui/tools/plugins/process_job_dialog.py +3 -0
  152. ert/gui/tools/workflows/workflow_dialog.py +2 -0
  153. ert/gui/tools/workflows/workflows_tool.py +2 -0
  154. ert/libres_facade.py +5 -7
  155. ert/logging/__init__.py +4 -1
  156. ert/mode_definitions.py +2 -0
  157. ert/plugins/__init__.py +4 -6
  158. ert/plugins/hook_implementations/workflows/csv_export.py +2 -3
  159. ert/plugins/hook_implementations/workflows/gen_data_rft_export.py +10 -2
  160. ert/plugins/hook_specifications/__init__.py +0 -10
  161. ert/plugins/hook_specifications/jobs.py +0 -9
  162. ert/plugins/plugin_manager.py +53 -124
  163. ert/resources/forward_models/run_reservoirsimulator.py +8 -4
  164. ert/resources/forward_models/template_render.py +10 -10
  165. ert/resources/shell_scripts/delete_directory.py +2 -2
  166. ert/run_models/__init__.py +24 -6
  167. ert/run_models/_create_run_path.py +133 -38
  168. ert/run_models/ensemble_experiment.py +10 -4
  169. ert/run_models/ensemble_information_filter.py +8 -1
  170. ert/run_models/ensemble_smoother.py +9 -3
  171. ert/run_models/evaluate_ensemble.py +8 -6
  172. ert/run_models/event.py +7 -3
  173. ert/run_models/everest_run_model.py +337 -113
  174. ert/run_models/initial_ensemble_run_model.py +25 -24
  175. ert/run_models/manual_update.py +6 -3
  176. ert/run_models/manual_update_enif.py +37 -0
  177. ert/run_models/model_factory.py +78 -18
  178. ert/run_models/multiple_data_assimilation.py +22 -11
  179. ert/run_models/run_model.py +72 -73
  180. ert/run_models/single_test_run.py +7 -4
  181. ert/run_models/update_run_model.py +4 -2
  182. ert/runpaths.py +5 -6
  183. ert/sample_prior.py +9 -4
  184. ert/scheduler/__init__.py +10 -5
  185. ert/scheduler/driver.py +40 -0
  186. ert/scheduler/event.py +3 -1
  187. ert/scheduler/job.py +23 -13
  188. ert/scheduler/lsf_driver.py +15 -5
  189. ert/scheduler/openpbs_driver.py +10 -4
  190. ert/scheduler/scheduler.py +5 -0
  191. ert/scheduler/slurm_driver.py +20 -5
  192. ert/services/__init__.py +2 -2
  193. ert/services/_base_service.py +37 -20
  194. ert/services/_storage_main.py +20 -18
  195. ert/services/ert_server.py +317 -0
  196. ert/shared/_doc_utils/__init__.py +4 -2
  197. ert/shared/_doc_utils/ert_jobs.py +1 -4
  198. ert/shared/net_utils.py +43 -18
  199. ert/shared/storage/connection.py +3 -3
  200. ert/shared/version.py +3 -3
  201. ert/storage/__init__.py +14 -1
  202. ert/storage/local_ensemble.py +44 -13
  203. ert/storage/local_experiment.py +54 -34
  204. ert/storage/local_storage.py +90 -58
  205. ert/storage/migration/to10.py +3 -2
  206. ert/storage/migration/to11.py +9 -10
  207. ert/storage/migration/to12.py +19 -20
  208. ert/storage/migration/to13.py +28 -27
  209. ert/storage/migration/to14.py +3 -3
  210. ert/storage/migration/to15.py +25 -0
  211. ert/storage/migration/to16.py +38 -0
  212. ert/storage/migration/to17.py +42 -0
  213. ert/storage/migration/to18.py +11 -0
  214. ert/storage/migration/to19.py +34 -0
  215. ert/storage/migration/to20.py +23 -0
  216. ert/storage/migration/to21.py +25 -0
  217. ert/storage/migration/to6.py +3 -2
  218. ert/storage/migration/to7.py +12 -13
  219. ert/storage/migration/to8.py +9 -11
  220. ert/storage/migration/to9.py +5 -4
  221. ert/storage/realization_storage_state.py +7 -7
  222. ert/substitutions.py +12 -28
  223. ert/validation/active_range.py +7 -7
  224. ert/validation/ensemble_realizations_argument.py +4 -2
  225. ert/validation/rangestring.py +16 -16
  226. ert/workflow_runner.py +6 -3
  227. {ert-16.0.9.dist-info → ert-19.0.0rc2.dist-info}/METADATA +21 -15
  228. ert-19.0.0rc2.dist-info/RECORD +524 -0
  229. {ert-16.0.9.dist-info → ert-19.0.0rc2.dist-info}/WHEEL +1 -1
  230. everest/api/everest_data_api.py +14 -1
  231. everest/assets/everest_logo.svg +406 -0
  232. everest/bin/config_branch_script.py +30 -14
  233. everest/bin/everconfigdump_script.py +2 -10
  234. everest/bin/everest_script.py +53 -33
  235. everest/bin/everlint_script.py +3 -5
  236. everest/bin/kill_script.py +7 -5
  237. everest/bin/main.py +11 -24
  238. everest/bin/monitor_script.py +64 -35
  239. everest/bin/utils.py +58 -43
  240. everest/bin/visualization_script.py +23 -13
  241. everest/config/__init__.py +4 -1
  242. everest/config/control_config.py +81 -6
  243. everest/config/control_variable_config.py +4 -3
  244. everest/config/everest_config.py +102 -79
  245. everest/config/forward_model_config.py +5 -3
  246. everest/config/install_data_config.py +7 -5
  247. everest/config/install_job_config.py +45 -3
  248. everest/config/install_template_config.py +3 -3
  249. everest/config/optimization_config.py +19 -6
  250. everest/config/output_constraint_config.py +8 -2
  251. everest/config/server_config.py +6 -55
  252. everest/config/simulator_config.py +62 -17
  253. everest/config/utils.py +25 -105
  254. everest/config/validation_utils.py +34 -15
  255. everest/config_file_loader.py +30 -21
  256. everest/detached/__init__.py +0 -6
  257. everest/detached/client.py +7 -52
  258. everest/detached/everserver.py +19 -45
  259. everest/everest_storage.py +24 -40
  260. everest/gui/everest_client.py +2 -3
  261. everest/gui/main_window.py +2 -2
  262. everest/optimizer/everest2ropt.py +68 -42
  263. everest/optimizer/opt_model_transforms.py +15 -20
  264. everest/optimizer/utils.py +0 -29
  265. everest/plugins/hook_specs.py +0 -24
  266. everest/strings.py +1 -6
  267. everest/util/__init__.py +3 -1
  268. ert/config/everest_objective_config.py +0 -95
  269. ert/config/ext_param_config.py +0 -107
  270. ert/gui/tools/export/__init__.py +0 -3
  271. ert/gui/tools/export/export_panel.py +0 -83
  272. ert/gui/tools/export/export_tool.py +0 -67
  273. ert/gui/tools/export/exporter.py +0 -36
  274. ert/plugins/hook_specifications/ecl_config.py +0 -29
  275. ert/services/storage_service.py +0 -127
  276. ert/summary_key_type.py +0 -234
  277. ert-16.0.9.dist-info/RECORD +0 -521
  278. everest/bin/everexport_script.py +0 -53
  279. everest/config/sampler_config.py +0 -103
  280. everest/simulator/__init__.py +0 -88
  281. everest/simulator/everest_to_ert.py +0 -252
  282. /ert/gui/{suggestor → ertwidgets/suggestor}/__init__.py +0 -0
  283. /ert/gui/{suggestor → ertwidgets/suggestor}/_colors.py +0 -0
  284. {ert-16.0.9.dist-info → ert-19.0.0rc2.dist-info}/entry_points.txt +0 -0
  285. {ert-16.0.9.dist-info → ert-19.0.0rc2.dist-info}/licenses/COPYING +0 -0
  286. {ert-16.0.9.dist-info → ert-19.0.0rc2.dist-info}/top_level.txt +0 -0
@@ -10,43 +10,45 @@ import os
10
10
  import queue
11
11
  import shutil
12
12
  import traceback
13
+ import warnings
13
14
  from collections import defaultdict
14
15
  from collections.abc import Callable, Iterator, MutableSequence
15
16
  from enum import IntEnum, auto
16
17
  from functools import cached_property
17
18
  from pathlib import Path
18
19
  from types import TracebackType
19
- from typing import TYPE_CHECKING, Any, Protocol
20
+ from typing import TYPE_CHECKING, Any, Protocol, cast
20
21
 
21
22
  import numpy as np
22
23
  from numpy.typing import NDArray
23
24
  from pydantic import PrivateAttr, ValidationError
24
25
  from ropt.enums import ExitCode as RoptExitCode
25
26
  from ropt.evaluator import EvaluatorContext, EvaluatorResult
26
- from ropt.plan import BasicOptimizer
27
27
  from ropt.results import FunctionResults, Results
28
28
  from ropt.transforms import OptModelTransforms
29
+ from ropt.workflow import BasicOptimizer
29
30
  from typing_extensions import TypedDict
30
31
 
31
32
  from ert.config import (
32
33
  EverestConstraintsConfig,
34
+ EverestControl,
33
35
  EverestObjectivesConfig,
34
36
  GenDataConfig,
37
+ HookRuntime,
38
+ KnownQueueOptionsAdapter,
35
39
  ParameterConfig,
36
40
  QueueConfig,
37
41
  ResponseConfig,
38
42
  SummaryConfig,
43
+ WorkflowJob,
39
44
  )
40
45
  from ert.config.ert_config import (
41
46
  create_and_hook_workflows,
42
- read_templates,
43
47
  uppercase_subkeys_and_stringify_subvalues,
44
- workflow_jobs_from_dict,
45
- )
46
- from ert.config.model_config import (
47
- DEFAULT_ECLBASE_FORMAT,
48
48
  )
49
+ from ert.config.model_config import DEFAULT_ECLBASE_FORMAT
49
50
  from ert.config.model_config import ModelConfig as ErtModelConfig
51
+ from ert.config.parsing import ConfigWarning
50
52
  from ert.ensemble_evaluator import EndEvent, EvaluatorServerConfig
51
53
  from ert.plugins import ErtRuntimePlugins
52
54
  from ert.runpaths import Runpaths
@@ -55,31 +57,23 @@ from everest.config import (
55
57
  EverestConfig,
56
58
  InputConstraintConfig,
57
59
  ModelConfig,
58
- ObjectiveFunctionConfig,
59
60
  OptimizationConfig,
60
- OutputConstraintConfig,
61
61
  )
62
- from everest.config.forward_model_config import SummaryResults
62
+ from everest.config.forward_model_config import ForwardModelStepConfig, SummaryResults
63
63
  from everest.everest_storage import EverestStorage
64
64
  from everest.optimizer.everest2ropt import everest2ropt
65
65
  from everest.optimizer.opt_model_transforms import (
66
66
  EverestOptModelTransforms,
67
67
  get_optimization_domain_transforms,
68
68
  )
69
- from everest.simulator.everest_to_ert import (
70
- _get_workflow_files,
71
- everest_to_ert_config_dict,
72
- extract_summary_keys,
73
- get_substitutions,
74
- get_workflow_jobs,
75
- )
76
- from everest.strings import EVEREST, STORAGE_DIR
69
+ from everest.strings import EVEREST
77
70
 
78
71
  from ..run_arg import RunArg, create_run_arguments
72
+ from ..storage import ExperimentState, ExperimentStatus
79
73
  from ..storage.local_ensemble import EverestRealizationInfo
80
74
  from ..substitutions import Substitutions
81
75
  from .event import EverestBatchResultEvent, EverestStatusEvent
82
- from .run_model import RunModel, StatusEvents
76
+ from .run_model import RunModel, RunModelConfig, StatusEvents
83
77
 
84
78
  if TYPE_CHECKING:
85
79
  from ert.storage import Ensemble, Experiment
@@ -173,6 +167,17 @@ def _get_install_data_files(ever_config: EverestConfig) -> Iterator[tuple[Path,
173
167
  yield (data_storage / Path(target).name, data)
174
168
 
175
169
 
170
+ def _get_workflow_files(ever_config: EverestConfig) -> dict[str, tuple[Path, str]]:
171
+ data_storage = (Path(ever_config.output_dir) / ".internal_data").resolve()
172
+ return {
173
+ trigger: (
174
+ data_storage / f"{trigger}.workflow",
175
+ "\n".join(getattr(ever_config.workflows, trigger, [])),
176
+ )
177
+ for trigger in ("pre_simulation", "post_simulation")
178
+ }
179
+
180
+
176
181
  def _get_internal_files(ever_config: EverestConfig) -> dict[Path, str]:
177
182
  return dict(
178
183
  [
@@ -187,30 +192,52 @@ def _get_internal_files(ever_config: EverestConfig) -> dict[Path, str]:
187
192
  )
188
193
 
189
194
 
190
- class EverestRunModel(RunModel):
195
+ def _get_workflow_jobs(ever_config: EverestConfig) -> dict[str, WorkflowJob]:
196
+ workflow_jobs: dict[str, WorkflowJob] = {}
197
+ for job in ever_config.install_workflow_jobs or []:
198
+ workflow = job.to_ert_executable_workflow(ever_config.config_directory)
199
+ if workflow.name in workflow_jobs:
200
+ ConfigWarning.warn(
201
+ f"Duplicate workflow job with name {job.name!r}, "
202
+ f"overriding it with {job.executable!r}.",
203
+ job.name,
204
+ )
205
+ workflow_jobs[workflow.name] = workflow
206
+ return workflow_jobs
207
+
208
+
209
+ def _get_workflows(
210
+ ever_config: EverestConfig,
211
+ ) -> tuple[list[tuple[str, HookRuntime]], list[tuple[str, str]]]:
212
+ trigger2res: dict[str, HookRuntime] = {
213
+ "pre_simulation": HookRuntime.PRE_SIMULATION,
214
+ "post_simulation": HookRuntime.POST_SIMULATION,
215
+ }
216
+ res_hooks: list[tuple[str, HookRuntime]] = []
217
+ res_workflows: list[tuple[str, str]] = []
218
+ for ever_trigger, (workflow_file, jobs) in _get_workflow_files(ever_config).items():
219
+ if jobs:
220
+ res_hooks.append((ever_trigger, trigger2res[ever_trigger]))
221
+ res_workflows.append((str(workflow_file), ever_trigger))
222
+ return res_hooks, res_workflows
223
+
224
+
225
+ class EverestRunModelConfig(RunModelConfig):
191
226
  optimization_output_dir: str
192
227
  simulation_dir: str
193
228
 
194
229
  parameter_configuration: list[ParameterConfig]
195
230
  response_configuration: list[ResponseConfig]
196
- ert_templates: list[tuple[str, str]]
197
-
198
- controls: list[ControlConfig]
199
-
200
- objective_functions: list[ObjectiveFunctionConfig]
201
- objective_names: list[str]
202
231
 
203
232
  input_constraints: list[InputConstraintConfig]
204
-
205
- output_constraints: list[OutputConstraintConfig]
206
- constraint_names: list[str]
207
-
208
233
  optimization: OptimizationConfig
209
234
  model: ModelConfig
210
235
  keep_run_path: bool
211
236
  experiment_name: str
212
237
  target_ensemble: str
213
238
 
239
+
240
+ class EverestRunModel(RunModel, EverestRunModelConfig):
214
241
  _exit_code: EverestExitCode | None = PrivateAttr(default=None)
215
242
  _experiment: Experiment | None = PrivateAttr(default=None)
216
243
  _eval_server_cfg: EvaluatorServerConfig | None = PrivateAttr(default=None)
@@ -238,13 +265,9 @@ class EverestRunModel(RunModel):
238
265
  everest_config.environment.random_seed,
239
266
  )
240
267
 
241
- storage_dir = os.path.join(everest_config.output_dir, STORAGE_DIR)
242
-
243
268
  if status_queue is None:
244
269
  status_queue = queue.SimpleQueue()
245
270
 
246
- config_dict = everest_to_ert_config_dict(everest_config)
247
-
248
271
  runpath_file: Path = Path(
249
272
  os.path.join(everest_config.output_dir, ".res_runpath_list")
250
273
  )
@@ -267,20 +290,11 @@ class EverestRunModel(RunModel):
267
290
 
268
291
  response_configs: list[ResponseConfig] = []
269
292
 
270
- objective_names = [c.name for c in everest_config.objective_functions]
271
- response_configs.append(
272
- EverestObjectivesConfig(keys=objective_names, input_files=objective_names)
273
- )
293
+ response_configs.append(everest_config.create_ert_objectives_config())
274
294
 
275
- constraint_names = [c.name for c in everest_config.output_constraints]
276
-
277
- if constraint_names:
278
- response_configs.append(
279
- EverestConstraintsConfig(
280
- keys=constraint_names,
281
- input_files=constraint_names,
282
- )
283
- )
295
+ constraints_config = everest_config.create_ert_output_constraints_config()
296
+ if constraints_config is not None:
297
+ response_configs.append(constraints_config)
284
298
 
285
299
  gen_data_keys = [
286
300
  fm.results.file_name
@@ -302,7 +316,7 @@ class EverestRunModel(RunModel):
302
316
  eclbase = summary_fm.results.file_name
303
317
  response_configs.append(
304
318
  SummaryConfig(
305
- keys=extract_summary_keys(everest_config), input_files=[eclbase]
319
+ keys=_extract_summary_keys(everest_config), input_files=[eclbase]
306
320
  )
307
321
  )
308
322
  else:
@@ -315,7 +329,7 @@ class EverestRunModel(RunModel):
315
329
  runpath_format_string=str(
316
330
  Path(everest_config.simulation_dir)
317
331
  / "batch_<ITER>"
318
- / "realization_<GEO_ID>"
332
+ / "realization_<REALIZATION_ID>"
319
333
  / "<SIM_DIR>"
320
334
  ),
321
335
  eclbase_format_string=eclbase
@@ -323,34 +337,91 @@ class EverestRunModel(RunModel):
323
337
  else DEFAULT_ECLBASE_FORMAT,
324
338
  )
325
339
 
326
- queue_config = QueueConfig.from_dict(
327
- config_dict,
328
- site_queue_options=runtime_plugins.queue_options
329
- if runtime_plugins
330
- else None,
340
+ simulator_config = everest_config.simulator
341
+ queue_options_from_everconfig: dict[str, Any] = (
342
+ {"name": "local"}
343
+ if simulator_config.queue_system is None
344
+ else (
345
+ simulator_config.queue_system.model_dump(exclude_unset=True)
346
+ | {
347
+ "name": simulator_config.queue_system.name,
348
+ }
349
+ )
350
+ )
351
+
352
+ if simulator_config.max_memory is not None:
353
+ queue_options_from_everconfig["realization_memory"] = (
354
+ simulator_config.max_memory or 0
355
+ )
356
+
357
+ # Number of cores reserved on queue nodes (NUM_CPU)
358
+ if (num_fm_cpu := simulator_config.cores_per_node) is not None:
359
+ if (
360
+ simulator_config.queue_system is not None
361
+ and "num_cpu" not in simulator_config.queue_system.model_fields_set
362
+ ):
363
+ queue_options_from_everconfig["num_cpu"] = num_fm_cpu
364
+ else:
365
+ warnings.warn(
366
+ "Ignoring cores_per_node as num_cpu was set",
367
+ UserWarning,
368
+ stacklevel=2,
369
+ )
370
+
371
+ # Only take into account site queue options
372
+ # if and only if they exist and are of same type as user
373
+ # specified queue system
374
+ site_queue_options_to_apply = (
375
+ runtime_plugins.queue_options.model_dump(exclude_unset=True)
376
+ if (
377
+ runtime_plugins is not None
378
+ and runtime_plugins.queue_options is not None
379
+ and runtime_plugins.queue_options.name
380
+ == queue_options_from_everconfig["name"]
381
+ )
382
+ else {}
331
383
  )
332
- assert everest_config.simulator is not None
333
- assert everest_config.simulator.queue_system is not None
334
- queue_config.queue_options = everest_config.simulator.queue_system
335
- queue_config.queue_system = everest_config.simulator.queue_system.name
336
-
337
- substitutions = get_substitutions(
338
- config_dict,
339
- runpath_config,
340
- runpath_file,
341
- queue_config.queue_options.num_cpu,
384
+
385
+ queue_options_dict = site_queue_options_to_apply | queue_options_from_everconfig
386
+
387
+ queue_options = KnownQueueOptionsAdapter.validate_python(queue_options_dict)
388
+
389
+ if queue_options.project_code is None:
390
+ tags = {
391
+ fm_name.lower()
392
+ for fm_name in everest_config.forward_model_step_commands
393
+ if fm_name.split(" ")[0].upper()
394
+ in {"RMS", "FLOW", "ECLIPSE100", "ECLIPSE300"}
395
+ }
396
+ if tags:
397
+ queue_options.project_code = "+".join(tags)
398
+
399
+ queue_config = QueueConfig(
400
+ max_submit=simulator_config.resubmit_limit + 1,
401
+ queue_system=queue_options.name,
402
+ queue_options=queue_options,
403
+ stop_long_running=False,
404
+ max_runtime=simulator_config.max_runtime,
342
405
  )
343
- ert_templates = read_templates(config_dict)
406
+
407
+ substitutions = {
408
+ "<RUNPATH_FILE>": str(runpath_file),
409
+ "<RUNPATH>": runpath_config.runpath_format_string,
410
+ "<ECL_BASE>": runpath_config.eclbase_format_string,
411
+ "<ECLBASE>": runpath_config.eclbase_format_string,
412
+ "<NUM_CPU>": str(queue_config.queue_options.num_cpu),
413
+ "<CONFIG_PATH>": everest_config.config_directory,
414
+ "<CONFIG_FILE>": Path(everest_config.config_file).stem,
415
+ }
344
416
 
345
417
  for datafile, data in _get_internal_files(everest_config).items():
346
418
  datafile.parent.mkdir(exist_ok=True, parents=True)
347
419
  datafile.write_text(data, encoding="utf-8")
348
420
 
349
- workflow_jobs = get_workflow_jobs(everest_config)
350
- if deprecated_workflow_jobs := workflow_jobs_from_dict(config_dict):
351
- workflow_jobs.update(deprecated_workflow_jobs)
421
+ workflow_jobs = _get_workflow_jobs(everest_config)
422
+ hooks, workflows = _get_workflows(everest_config)
352
423
  _, hooked_workflows = create_and_hook_workflows(
353
- config_dict, workflow_jobs, substitutions
424
+ hooks, workflows, workflow_jobs, substitutions
354
425
  )
355
426
 
356
427
  install_job_fm_steps = {
@@ -425,21 +496,6 @@ class EverestRunModel(RunModel):
425
496
  for key, val in plugin_env_vars.items():
426
497
  env_vars[key] = substituter.substitute(val)
427
498
 
428
- for key, val in config_dict.get("SETENV", []):
429
- if key in env_vars:
430
- message = (
431
- f"Overriding environment variable: {key}, "
432
- f"old value: {env_vars[key]}, new value: {val}"
433
- )
434
-
435
- if key in plugin_env_vars:
436
- site_value = plugin_env_vars
437
- message += f", site configuration value: {site_value}"
438
-
439
- logger.warning(message)
440
-
441
- env_vars[key] = substituter.substitute(val)
442
-
443
499
  delete_run_path: bool = (
444
500
  everest_config.simulator is not None
445
501
  and everest_config.simulator.delete_run_path
@@ -452,10 +508,8 @@ class EverestRunModel(RunModel):
452
508
  simulation_dir=everest_config.simulation_dir,
453
509
  keep_run_path=not delete_run_path,
454
510
  objective_names=everest_config.objective_names,
455
- constraint_names=everest_config.constraint_names,
456
511
  objective_functions=everest_config.objective_functions,
457
512
  input_constraints=everest_config.input_constraints,
458
- output_constraints=everest_config.output_constraints,
459
513
  optimization=everest_config.optimization,
460
514
  model=everest_config.model,
461
515
  optimization_output_dir=everest_config.optimization_output_dir,
@@ -467,7 +521,6 @@ class EverestRunModel(RunModel):
467
521
  active_realizations=[],
468
522
  parameter_configuration=parameter_configs,
469
523
  response_configuration=response_configs,
470
- ert_templates=ert_templates,
471
524
  user_config_file=config_file,
472
525
  env_vars=env_vars,
473
526
  env_pr_fm_step=env_pr_fm_step,
@@ -475,19 +528,29 @@ class EverestRunModel(RunModel):
475
528
  forward_model_steps=forward_model_steps,
476
529
  substitutions=substitutions,
477
530
  hooked_workflows=hooked_workflows,
478
- storage_path=storage_dir,
531
+ storage_path=everest_config.storage_dir,
479
532
  queue_config=queue_config,
480
533
  status_queue=status_queue,
481
534
  optimization_callback=optimization_callback,
482
535
  )
483
536
 
537
+ @property
538
+ def _everest_control_configs(self) -> list[EverestControl]:
539
+ controls = [
540
+ c for c in self.parameter_configuration if c.type == "everest_parameters"
541
+ ]
542
+
543
+ # There will and must always be one EverestControl config for an
544
+ # Everest optimization.
545
+ return cast(list[EverestControl], controls)
546
+
484
547
  @cached_property
485
548
  def _transforms(self) -> EverestOptModelTransforms:
486
549
  return get_optimization_domain_transforms(
487
- self.controls,
488
- self.objective_functions,
550
+ self._everest_control_configs,
551
+ self.objectives_config,
489
552
  self.input_constraints,
490
- self.output_constraints,
553
+ self.output_constraints_config,
491
554
  self.model,
492
555
  self.optimization.auto_scale,
493
556
  )
@@ -504,6 +567,14 @@ class EverestRunModel(RunModel):
504
567
  def exit_code(self) -> EverestExitCode | None:
505
568
  return self._exit_code
506
569
 
570
+ def cancel(self) -> None:
571
+ if self._experiment is not None:
572
+ self._experiment.status = ExperimentStatus(
573
+ message="Optimization aborted",
574
+ status=ExperimentState.stopped,
575
+ )
576
+ super().cancel()
577
+
507
578
  def __repr__(self) -> str:
508
579
  return f"EverestRunModel(config={self.user_config_file})"
509
580
 
@@ -542,7 +613,7 @@ class EverestRunModel(RunModel):
542
613
  traceback=traceback_str,
543
614
  )
544
615
  if failed
545
- else "Experiment completed."
616
+ else "Experiment completed"
546
617
  ),
547
618
  )
548
619
  )
@@ -573,6 +644,29 @@ class EverestRunModel(RunModel):
573
644
  )
574
645
  )
575
646
 
647
+ @property
648
+ def output_constraints_config(self) -> EverestConstraintsConfig | None:
649
+ constraints_config = next(
650
+ (c for c in self.response_configuration if c.type == "everest_constraints"),
651
+ None,
652
+ )
653
+
654
+ if constraints_config is None:
655
+ return None
656
+
657
+ assert isinstance(constraints_config, EverestConstraintsConfig)
658
+ return constraints_config
659
+
660
+ @property
661
+ def objectives_config(self) -> EverestObjectivesConfig:
662
+ obj_config = next(
663
+ c for c in self.response_configuration if c.type == "everest_objectives"
664
+ )
665
+ # There will and must always be one objectives config for an
666
+ # Everest optimization.
667
+ assert isinstance(obj_config, EverestObjectivesConfig)
668
+ return obj_config
669
+
576
670
  def run_experiment(
577
671
  self,
578
672
  evaluator_server_config: EvaluatorServerConfig,
@@ -587,6 +681,10 @@ class EverestRunModel(RunModel):
587
681
  responses=self.response_configuration,
588
682
  )
589
683
 
684
+ self._experiment.status = ExperimentStatus(
685
+ message="Experiment started", status=ExperimentState.running
686
+ )
687
+
590
688
  # Initialize the ropt optimizer:
591
689
  optimizer, initial_guesses = self._create_optimizer()
592
690
 
@@ -595,18 +693,20 @@ class EverestRunModel(RunModel):
595
693
  )
596
694
 
597
695
  formatted_control_names = [
598
- name for config in self.controls for name in config.formatted_control_names
696
+ name
697
+ for config in self._everest_control_configs
698
+ for name in config.input_keys
599
699
  ]
600
700
  self._ever_storage.init(
601
701
  formatted_control_names=formatted_control_names,
602
- objective_functions=self.objective_functions,
603
- output_constraints=self.output_constraints,
702
+ objective_functions=self.objectives_config,
703
+ output_constraints=self.output_constraints_config,
604
704
  realizations=self.model.realizations,
605
705
  )
606
706
  optimizer.set_results_callback(self._handle_optimizer_results)
607
707
 
608
708
  # Run the optimization:
609
- optimizer_exit_code = optimizer.run(initial_guesses).exit_code
709
+ optimizer_exit_code = optimizer.run(initial_guesses)
610
710
 
611
711
  # Store some final results.
612
712
  self._ever_storage.on_optimization_finished()
@@ -617,22 +717,46 @@ class EverestRunModel(RunModel):
617
717
  ):
618
718
  self._ever_storage.export_everest_opt_results_to_csv()
619
719
 
720
+ experiment_status = None
620
721
  if self._exit_code is None:
621
722
  match optimizer_exit_code:
622
723
  case RoptExitCode.MAX_FUNCTIONS_REACHED:
623
724
  self._exit_code = EverestExitCode.MAX_FUNCTIONS_REACHED
725
+ experiment_status = ExperimentStatus(
726
+ message="Maximum number of function evaluations reached",
727
+ status=ExperimentState.completed,
728
+ )
624
729
  case RoptExitCode.MAX_BATCHES_REACHED:
625
730
  self._exit_code = EverestExitCode.MAX_BATCH_NUM_REACHED
731
+ experiment_status = ExperimentStatus(
732
+ message="Maximum number of batches reached",
733
+ status=ExperimentState.completed,
734
+ )
626
735
  case RoptExitCode.USER_ABORT:
627
736
  self._exit_code = EverestExitCode.USER_ABORT
737
+ experiment_status = ExperimentStatus(
738
+ message="Optimization aborted", status=ExperimentState.stopped
739
+ )
628
740
  case RoptExitCode.TOO_FEW_REALIZATIONS:
629
741
  self._exit_code = (
630
742
  EverestExitCode.TOO_FEW_REALIZATIONS
631
743
  if self.get_number_of_successful_realizations() > 0
632
744
  else EverestExitCode.ALL_REALIZATIONS_FAILED
633
745
  )
746
+ experiment_status = ExperimentStatus(
747
+ message="Too few realizations are evaluated successfully"
748
+ if self.get_number_of_successful_realizations() > 0
749
+ else "All realizations failed",
750
+ status=ExperimentState.failed,
751
+ )
634
752
  case _:
635
753
  self._exit_code = EverestExitCode.COMPLETED
754
+ experiment_status = ExperimentStatus(
755
+ message="Experiment completed", status=ExperimentState.completed
756
+ )
757
+
758
+ if experiment_status is not None:
759
+ self._experiment.status = experiment_status
636
760
 
637
761
  logger.debug(
638
762
  f"Everest experiment finished with exit code {self._exit_code.name}"
@@ -640,10 +764,10 @@ class EverestRunModel(RunModel):
640
764
 
641
765
  def _create_optimizer(self) -> tuple[BasicOptimizer, list[float]]:
642
766
  enopt_config, initial_guesses = everest2ropt(
643
- self.controls,
644
- self.objective_functions,
767
+ cast(list[EverestControl], self.parameter_configuration),
768
+ self.objectives_config,
645
769
  self.input_constraints,
646
- self.output_constraints,
770
+ self.output_constraints_config,
647
771
  self.optimization,
648
772
  self.model,
649
773
  self.random_seed,
@@ -710,18 +834,13 @@ class EverestRunModel(RunModel):
710
834
  for sim_id in range(sim_to_control_vector.shape[0]):
711
835
  sim_controls = sim_to_control_vector[sim_id]
712
836
  offset = 0
713
- for control_config in self.controls:
714
- ext_param_config = next(
715
- c
716
- for c in self.parameter_configuration
717
- if c.name == control_config.name
718
- )
719
- n_param_keys = len(ext_param_config.parameter_keys)
837
+ for control_config in self._everest_control_configs:
838
+ n_param_keys = len(control_config.parameter_keys)
720
839
 
721
840
  # Save controls to ensemble
722
841
  ensemble.save_parameters_numpy(
723
842
  sim_controls[offset : (offset + n_param_keys)].reshape(-1, 1),
724
- ext_param_config.name,
843
+ control_config.name,
725
844
  np.array([sim_id]),
726
845
  )
727
846
  offset += n_param_keys
@@ -888,15 +1007,15 @@ class EverestRunModel(RunModel):
888
1007
  # Nothing to do, there may only have been inactive control vectors:
889
1008
  num_all_simulations = control_values.shape[0]
890
1009
  objectives = np.zeros(
891
- (num_all_simulations, len(self.objective_names)),
1010
+ (num_all_simulations, len(self.objectives_config.keys)),
892
1011
  dtype=np.float64,
893
1012
  )
894
1013
  constraints = (
895
1014
  np.zeros(
896
- (num_all_simulations, len(self.constraint_names)),
1015
+ (num_all_simulations, len(self.output_constraints_config.keys)),
897
1016
  dtype=np.float64,
898
1017
  )
899
- if self.output_constraints
1018
+ if self.output_constraints_config
900
1019
  else None
901
1020
  )
902
1021
  sim_ids = np.array([-1] * num_all_simulations, dtype=np.int32)
@@ -959,7 +1078,7 @@ class EverestRunModel(RunModel):
959
1078
  for sim_id, (model_realization, perturbation) in enumerate(
960
1079
  zip(sim_to_model_realization, sim_to_perturbation, strict=True)
961
1080
  ):
962
- substitutions[f"<GEO_ID_{sim_id}_{ensemble.iteration}>"] = str(
1081
+ substitutions[f"<REALIZATION_ID_{sim_id}_{ensemble.iteration}>"] = str(
963
1082
  int(model_realization)
964
1083
  )
965
1084
  if perturbation >= 0:
@@ -1006,10 +1125,14 @@ class EverestRunModel(RunModel):
1006
1125
  def _gather_simulation_results(
1007
1126
  self, ensemble: Ensemble
1008
1127
  ) -> tuple[NDArray[np.float64], NDArray[np.float64] | None]:
1009
- objective_names = self.objective_names
1128
+ objective_names = self.objectives_config.keys
1010
1129
  objectives = np.zeros((ensemble.ensemble_size, len(objective_names)))
1011
1130
 
1012
- constraint_names = self.constraint_names
1131
+ constraint_names = (
1132
+ self.output_constraints_config.keys
1133
+ if self.output_constraints_config is not None
1134
+ else []
1135
+ )
1013
1136
  constraints = np.zeros((ensemble.ensemble_size, len(constraint_names)))
1014
1137
 
1015
1138
  if not any(self.active_realizations):
@@ -1046,3 +1169,104 @@ class EverestRunModel(RunModel):
1046
1169
  return os.path.exists(self.simulation_dir) and any(
1047
1170
  os.listdir(self.simulation_dir)
1048
1171
  )
1172
+
1173
+
1174
+ def _extract_summary_keys(ever_config: EverestConfig) -> list[str]:
1175
+ DEFAULT_DATA_SUMMARY_KEYS = ["YEAR", "YEARS", "TCPU", "TCPUDAY", "MONTH", "DAY"]
1176
+
1177
+ DEFAULT_FIELD_SUMMARY_KEYS = [
1178
+ "FOPR",
1179
+ "FOPT",
1180
+ "FOIR",
1181
+ "FOIT",
1182
+ "FWPR",
1183
+ "FWPT",
1184
+ "FWIR",
1185
+ "FWIT",
1186
+ "FGPR",
1187
+ "FGPT",
1188
+ "FGIR",
1189
+ "FGIT",
1190
+ "FVPR",
1191
+ "FVPT",
1192
+ "FVIR",
1193
+ "FVIT",
1194
+ "FWCT",
1195
+ "FGOR",
1196
+ "FOIP",
1197
+ "FOIPL",
1198
+ "FOIPG",
1199
+ "FWIP",
1200
+ "FGIP",
1201
+ "FGIPL",
1202
+ "FGIPG",
1203
+ "FPR",
1204
+ "FAQR",
1205
+ "FAQRG",
1206
+ "FAQT",
1207
+ "FAQTG",
1208
+ "FWGR",
1209
+ ]
1210
+
1211
+ DEFAULT_WELL_SUMMARY_KEYS = [
1212
+ "WOPR",
1213
+ "WOPT",
1214
+ "WOIR",
1215
+ "WOIT",
1216
+ "WWPR",
1217
+ "WWPT",
1218
+ "WWIR",
1219
+ "WWIT",
1220
+ "WGPR",
1221
+ "WGPT",
1222
+ "WGIR",
1223
+ "WGIT",
1224
+ "WVPR",
1225
+ "WVPT",
1226
+ "WVIR",
1227
+ "WVIT",
1228
+ "WWCT",
1229
+ "WGOR",
1230
+ "WWGR",
1231
+ "WBHP",
1232
+ "WTHP",
1233
+ "WPI",
1234
+ ]
1235
+
1236
+ DEFAULT_WELL_TARGET_SUMMARY_KEYS = [
1237
+ well_key + "T"
1238
+ for well_key in DEFAULT_WELL_SUMMARY_KEYS
1239
+ if well_key.endswith("R") and well_key != "WGOR"
1240
+ ]
1241
+
1242
+ summary_fms: list[ForwardModelStepConfig] = [
1243
+ fm
1244
+ for fm in ever_config.forward_model
1245
+ if fm.results is not None and fm.results.type == "summary"
1246
+ ]
1247
+
1248
+ if not summary_fms:
1249
+ return []
1250
+
1251
+ smry_results = summary_fms[0].results
1252
+ assert isinstance(smry_results, SummaryResults)
1253
+
1254
+ requested_keys: list[str] = ["*"] if smry_results.keys == "*" else smry_results.keys
1255
+
1256
+ well_keys = [
1257
+ f"{sum_key}:*"
1258
+ for sum_key in DEFAULT_WELL_SUMMARY_KEYS + DEFAULT_WELL_TARGET_SUMMARY_KEYS
1259
+ ]
1260
+ deprecated_user_specified_keys = (
1261
+ [] if ever_config.export is None else ever_config.export.keywords
1262
+ )
1263
+
1264
+ return list(
1265
+ set(
1266
+ requested_keys
1267
+ + DEFAULT_DATA_SUMMARY_KEYS
1268
+ + DEFAULT_FIELD_SUMMARY_KEYS
1269
+ + well_keys
1270
+ + deprecated_user_specified_keys
1271
+ )
1272
+ )