ert 17.0.0__py3-none-any.whl → 19.0.0rc2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (218) hide show
  1. _ert/events.py +19 -2
  2. _ert/forward_model_runner/client.py +6 -2
  3. ert/__main__.py +28 -13
  4. ert/analysis/_enif_update.py +8 -4
  5. ert/analysis/_es_update.py +19 -6
  6. ert/analysis/_update_commons.py +16 -6
  7. ert/cli/main.py +13 -6
  8. ert/cli/monitor.py +7 -0
  9. ert/config/__init__.py +15 -6
  10. ert/config/_create_observation_dataframes.py +117 -20
  11. ert/config/_get_num_cpu.py +1 -1
  12. ert/config/_observations.py +91 -2
  13. ert/config/_read_summary.py +8 -6
  14. ert/config/design_matrix.py +51 -24
  15. ert/config/distribution.py +1 -1
  16. ert/config/ensemble_config.py +9 -17
  17. ert/config/ert_config.py +103 -19
  18. ert/config/everest_control.py +234 -0
  19. ert/config/{everest_objective_config.py → everest_response.py} +24 -15
  20. ert/config/field.py +96 -84
  21. ert/config/forward_model_step.py +122 -17
  22. ert/config/gen_data_config.py +5 -10
  23. ert/config/gen_kw_config.py +5 -35
  24. ert/config/known_response_types.py +14 -0
  25. ert/config/parameter_config.py +1 -33
  26. ert/config/parsing/_option_dict.py +10 -2
  27. ert/config/parsing/config_keywords.py +2 -0
  28. ert/config/parsing/config_schema.py +23 -3
  29. ert/config/parsing/config_schema_deprecations.py +3 -14
  30. ert/config/parsing/config_schema_item.py +26 -11
  31. ert/config/parsing/context_values.py +3 -3
  32. ert/config/parsing/file_context_token.py +1 -1
  33. ert/config/parsing/observations_parser.py +6 -2
  34. ert/config/parsing/queue_system.py +9 -0
  35. ert/config/parsing/schema_item_type.py +1 -0
  36. ert/config/queue_config.py +4 -5
  37. ert/config/response_config.py +0 -8
  38. ert/config/rft_config.py +275 -0
  39. ert/config/summary_config.py +3 -8
  40. ert/config/surface_config.py +59 -16
  41. ert/config/workflow_fixtures.py +2 -1
  42. ert/dark_storage/client/__init__.py +2 -2
  43. ert/dark_storage/client/_session.py +4 -4
  44. ert/dark_storage/client/client.py +2 -2
  45. ert/dark_storage/common.py +1 -1
  46. ert/dark_storage/compute/misfits.py +11 -7
  47. ert/dark_storage/endpoints/compute/misfits.py +6 -4
  48. ert/dark_storage/endpoints/experiment_server.py +12 -9
  49. ert/dark_storage/endpoints/experiments.py +2 -2
  50. ert/dark_storage/endpoints/observations.py +8 -6
  51. ert/dark_storage/endpoints/parameters.py +2 -18
  52. ert/dark_storage/endpoints/responses.py +24 -5
  53. ert/dark_storage/json_schema/experiment.py +1 -1
  54. ert/data/_measured_data.py +6 -5
  55. ert/ensemble_evaluator/__init__.py +8 -1
  56. ert/ensemble_evaluator/config.py +2 -1
  57. ert/ensemble_evaluator/evaluator.py +81 -29
  58. ert/ensemble_evaluator/event.py +6 -0
  59. ert/ensemble_evaluator/snapshot.py +3 -1
  60. ert/ensemble_evaluator/state.py +1 -0
  61. ert/field_utils/__init__.py +8 -0
  62. ert/field_utils/field_utils.py +212 -3
  63. ert/field_utils/roff_io.py +1 -1
  64. ert/gui/__init__.py +5 -2
  65. ert/gui/ertnotifier.py +1 -1
  66. ert/gui/ertwidgets/__init__.py +23 -16
  67. ert/gui/ertwidgets/analysismoduleedit.py +2 -2
  68. ert/gui/ertwidgets/checklist.py +1 -1
  69. ert/gui/ertwidgets/create_experiment_dialog.py +3 -1
  70. ert/gui/ertwidgets/ensembleselector.py +2 -2
  71. ert/gui/ertwidgets/models/__init__.py +2 -0
  72. ert/gui/ertwidgets/models/activerealizationsmodel.py +2 -1
  73. ert/gui/ertwidgets/models/path_model.py +1 -1
  74. ert/gui/ertwidgets/models/targetensemblemodel.py +2 -1
  75. ert/gui/ertwidgets/models/text_model.py +1 -1
  76. ert/gui/ertwidgets/pathchooser.py +0 -3
  77. ert/gui/ertwidgets/searchbox.py +13 -4
  78. ert/gui/{suggestor → ertwidgets/suggestor}/_suggestor_message.py +13 -4
  79. ert/gui/{suggestor → ertwidgets/suggestor}/suggestor.py +63 -30
  80. ert/gui/main.py +37 -8
  81. ert/gui/main_window.py +1 -7
  82. ert/gui/simulation/ensemble_experiment_panel.py +1 -1
  83. ert/gui/simulation/ensemble_information_filter_panel.py +1 -1
  84. ert/gui/simulation/ensemble_smoother_panel.py +1 -1
  85. ert/gui/simulation/evaluate_ensemble_panel.py +1 -1
  86. ert/gui/simulation/experiment_panel.py +16 -3
  87. ert/gui/simulation/manual_update_panel.py +31 -8
  88. ert/gui/simulation/multiple_data_assimilation_panel.py +12 -8
  89. ert/gui/simulation/run_dialog.py +27 -20
  90. ert/gui/simulation/single_test_run_panel.py +2 -2
  91. ert/gui/summarypanel.py +20 -1
  92. ert/gui/tools/load_results/load_results_panel.py +1 -1
  93. ert/gui/tools/manage_experiments/export_dialog.py +136 -0
  94. ert/gui/tools/manage_experiments/storage_info_widget.py +121 -16
  95. ert/gui/tools/manage_experiments/storage_widget.py +1 -2
  96. ert/gui/tools/plot/plot_api.py +37 -25
  97. ert/gui/tools/plot/plot_widget.py +10 -2
  98. ert/gui/tools/plot/plot_window.py +38 -18
  99. ert/gui/tools/plot/plottery/plot_config.py +2 -0
  100. ert/gui/tools/plot/plottery/plot_context.py +14 -0
  101. ert/gui/tools/plot/plottery/plots/__init__.py +2 -0
  102. ert/gui/tools/plot/plottery/plots/cesp.py +3 -1
  103. ert/gui/tools/plot/plottery/plots/distribution.py +6 -1
  104. ert/gui/tools/plot/plottery/plots/ensemble.py +12 -3
  105. ert/gui/tools/plot/plottery/plots/gaussian_kde.py +12 -2
  106. ert/gui/tools/plot/plottery/plots/histogram.py +3 -1
  107. ert/gui/tools/plot/plottery/plots/misfits.py +436 -0
  108. ert/gui/tools/plot/plottery/plots/observations.py +18 -4
  109. ert/gui/tools/plot/plottery/plots/statistics.py +62 -20
  110. ert/gui/tools/plot/plottery/plots/std_dev.py +3 -1
  111. ert/mode_definitions.py +2 -0
  112. ert/plugins/__init__.py +0 -1
  113. ert/plugins/hook_implementations/workflows/csv_export.py +2 -3
  114. ert/plugins/hook_implementations/workflows/gen_data_rft_export.py +10 -2
  115. ert/plugins/hook_specifications/__init__.py +0 -2
  116. ert/plugins/hook_specifications/jobs.py +0 -9
  117. ert/plugins/plugin_manager.py +6 -33
  118. ert/resources/forward_models/run_reservoirsimulator.py +8 -3
  119. ert/resources/shell_scripts/delete_directory.py +2 -2
  120. ert/run_models/__init__.py +18 -5
  121. ert/run_models/_create_run_path.py +131 -37
  122. ert/run_models/ensemble_experiment.py +10 -4
  123. ert/run_models/ensemble_information_filter.py +8 -1
  124. ert/run_models/ensemble_smoother.py +9 -3
  125. ert/run_models/evaluate_ensemble.py +8 -6
  126. ert/run_models/event.py +7 -3
  127. ert/run_models/everest_run_model.py +159 -46
  128. ert/run_models/initial_ensemble_run_model.py +25 -24
  129. ert/run_models/manual_update.py +6 -3
  130. ert/run_models/manual_update_enif.py +37 -0
  131. ert/run_models/model_factory.py +81 -21
  132. ert/run_models/multiple_data_assimilation.py +22 -11
  133. ert/run_models/run_model.py +64 -55
  134. ert/run_models/single_test_run.py +7 -4
  135. ert/run_models/update_run_model.py +4 -2
  136. ert/runpaths.py +5 -6
  137. ert/sample_prior.py +9 -4
  138. ert/scheduler/driver.py +37 -0
  139. ert/scheduler/event.py +3 -1
  140. ert/scheduler/job.py +23 -13
  141. ert/scheduler/lsf_driver.py +6 -2
  142. ert/scheduler/openpbs_driver.py +7 -1
  143. ert/scheduler/scheduler.py +5 -0
  144. ert/scheduler/slurm_driver.py +6 -2
  145. ert/services/__init__.py +2 -2
  146. ert/services/_base_service.py +37 -20
  147. ert/services/ert_server.py +317 -0
  148. ert/shared/_doc_utils/__init__.py +4 -2
  149. ert/shared/_doc_utils/ert_jobs.py +1 -4
  150. ert/shared/net_utils.py +43 -18
  151. ert/shared/storage/connection.py +3 -3
  152. ert/shared/version.py +3 -3
  153. ert/storage/__init__.py +2 -0
  154. ert/storage/local_ensemble.py +38 -12
  155. ert/storage/local_experiment.py +8 -16
  156. ert/storage/local_storage.py +68 -42
  157. ert/storage/migration/to11.py +1 -1
  158. ert/storage/migration/to16.py +38 -0
  159. ert/storage/migration/to17.py +42 -0
  160. ert/storage/migration/to18.py +11 -0
  161. ert/storage/migration/to19.py +34 -0
  162. ert/storage/migration/to20.py +23 -0
  163. ert/storage/migration/to21.py +25 -0
  164. ert/storage/migration/to8.py +4 -4
  165. ert/substitutions.py +12 -28
  166. ert/validation/active_range.py +7 -7
  167. ert/validation/rangestring.py +16 -16
  168. ert/workflow_runner.py +2 -1
  169. {ert-17.0.0.dist-info → ert-19.0.0rc2.dist-info}/METADATA +9 -8
  170. {ert-17.0.0.dist-info → ert-19.0.0rc2.dist-info}/RECORD +208 -205
  171. {ert-17.0.0.dist-info → ert-19.0.0rc2.dist-info}/WHEEL +1 -1
  172. everest/api/everest_data_api.py +14 -1
  173. everest/bin/config_branch_script.py +3 -6
  174. everest/bin/everconfigdump_script.py +1 -9
  175. everest/bin/everest_script.py +21 -11
  176. everest/bin/everlint_script.py +0 -2
  177. everest/bin/kill_script.py +2 -2
  178. everest/bin/monitor_script.py +2 -2
  179. everest/bin/utils.py +8 -4
  180. everest/bin/visualization_script.py +6 -14
  181. everest/config/__init__.py +4 -1
  182. everest/config/control_config.py +81 -6
  183. everest/config/control_variable_config.py +4 -3
  184. everest/config/everest_config.py +75 -42
  185. everest/config/forward_model_config.py +5 -3
  186. everest/config/install_data_config.py +7 -5
  187. everest/config/install_job_config.py +7 -3
  188. everest/config/install_template_config.py +3 -3
  189. everest/config/optimization_config.py +19 -6
  190. everest/config/output_constraint_config.py +8 -2
  191. everest/config/server_config.py +6 -49
  192. everest/config/utils.py +25 -105
  193. everest/config/validation_utils.py +17 -11
  194. everest/config_file_loader.py +13 -4
  195. everest/detached/client.py +3 -3
  196. everest/detached/everserver.py +7 -8
  197. everest/everest_storage.py +6 -12
  198. everest/gui/everest_client.py +2 -3
  199. everest/gui/main_window.py +2 -2
  200. everest/optimizer/everest2ropt.py +59 -32
  201. everest/optimizer/opt_model_transforms.py +12 -13
  202. everest/optimizer/utils.py +0 -29
  203. everest/strings.py +0 -5
  204. ert/config/everest_constraints_config.py +0 -95
  205. ert/config/ext_param_config.py +0 -106
  206. ert/gui/tools/export/__init__.py +0 -3
  207. ert/gui/tools/export/export_panel.py +0 -83
  208. ert/gui/tools/export/export_tool.py +0 -69
  209. ert/gui/tools/export/exporter.py +0 -36
  210. ert/services/storage_service.py +0 -127
  211. everest/config/sampler_config.py +0 -103
  212. everest/simulator/__init__.py +0 -88
  213. everest/simulator/everest_to_ert.py +0 -51
  214. /ert/gui/{suggestor → ertwidgets/suggestor}/__init__.py +0 -0
  215. /ert/gui/{suggestor → ertwidgets/suggestor}/_colors.py +0 -0
  216. {ert-17.0.0.dist-info → ert-19.0.0rc2.dist-info}/entry_points.txt +0 -0
  217. {ert-17.0.0.dist-info → ert-19.0.0rc2.dist-info}/licenses/COPYING +0 -0
  218. {ert-17.0.0.dist-info → ert-19.0.0rc2.dist-info}/top_level.txt +0 -0
@@ -111,7 +111,7 @@ class CSVExportJob(ErtScript):
111
111
  summary_data = pl.DataFrame({})
112
112
 
113
113
  if not summary_data.is_empty():
114
- pivoted_summary = summary_data.pivot(
114
+ pivoted_summary = summary_data.pivot( # noqa: PD010
115
115
  index=["realization", "time"], on="response_key", values="values"
116
116
  ).to_pandas()
117
117
 
@@ -123,8 +123,7 @@ class CSVExportJob(ErtScript):
123
123
  # Reset index to make 'Realization' a regular column
124
124
  ensemble_data = ensemble_data.reset_index()
125
125
 
126
- ensemble_data = pd.merge(
127
- ensemble_data,
126
+ ensemble_data = ensemble_data.merge(
128
127
  pivoted_summary,
129
128
  on="Realization",
130
129
  how="left",
@@ -1,19 +1,23 @@
1
+ from __future__ import annotations
2
+
1
3
  import contextlib
2
4
  import json
3
5
  import logging
4
6
  import os
5
7
  from collections.abc import Sequence
6
- from typing import Any
8
+ from typing import TYPE_CHECKING, Any
7
9
 
8
10
  import numpy as np
9
11
  import pandas as pd
10
12
  import polars as pl
11
- from PyQt6.QtWidgets import QCheckBox, QWidget
12
13
 
13
14
  from ert.config import ErtPlugin
14
15
  from ert.plugins import CancelPluginException
15
16
  from ert.storage import Storage
16
17
 
18
+ if TYPE_CHECKING:
19
+ from PyQt6.QtWidgets import QWidget
20
+
17
21
  logger = logging.getLogger(__name__)
18
22
 
19
23
 
@@ -240,6 +244,10 @@ class GenDataRFTCSVExportJob(ErtPlugin):
240
244
  list_edit = ListEditBox(ensemble_with_data_dict)
241
245
  list_edit.setObjectName("list_of_ensembles")
242
246
 
247
+ # Lazy load qt outside of gui to allow
248
+ # running of ert in cli without wm
249
+ from PyQt6.QtWidgets import QCheckBox # noqa: PLC0415
250
+
243
251
  drop_const_columns_check = QCheckBox()
244
252
  drop_const_columns_check.setChecked(False)
245
253
  drop_const_columns_check.setObjectName("drop_const_columns_check")
@@ -5,7 +5,6 @@ from .forward_model_steps import (
5
5
  from .help_resources import help_links
6
6
  from .jobs import (
7
7
  ertscript_workflow,
8
- installable_jobs,
9
8
  installable_workflow_jobs,
10
9
  job_documentation,
11
10
  legacy_ertscript_workflow,
@@ -20,7 +19,6 @@ __all__ = [
20
19
  "forward_model_configuration",
21
20
  "help_links",
22
21
  "installable_forward_model_steps",
23
- "installable_jobs",
24
22
  "installable_workflow_jobs",
25
23
  "job_documentation",
26
24
  "legacy_ertscript_workflow",
@@ -9,15 +9,6 @@ if TYPE_CHECKING:
9
9
  from ert.plugins.plugin_response import PluginResponse
10
10
 
11
11
 
12
- @no_type_check
13
- @hook_specification
14
- def installable_jobs() -> PluginResponse[dict[str, str]]:
15
- """
16
- :return: dict with job names as keys and path to config as value
17
- :rtype: PluginResponse with data as dict[str,str]
18
- """
19
-
20
-
21
12
  @no_type_check
22
13
  @hook_specification(firstresult=True)
23
14
  def job_documentation(job_name: str) -> PluginResponse[dict[str, str] | None]:
@@ -5,7 +5,6 @@ import logging
5
5
  import warnings
6
6
  from argparse import ArgumentParser
7
7
  from collections.abc import Callable, Mapping, Sequence
8
- from pathlib import Path
9
8
  from typing import TYPE_CHECKING, Any, Literal, TypeVar, overload
10
9
 
11
10
  import pluggy
@@ -13,15 +12,14 @@ from pydantic import BaseModel, Field
13
12
  from typing_extensions import TypedDict
14
13
 
15
14
  from ert.config import (
16
- ForwardModelStep,
17
15
  ForwardModelStepDocumentation,
18
16
  ForwardModelStepPlugin,
19
17
  KnownQueueOptions,
20
18
  LegacyWorkflowConfigs,
21
19
  LocalQueueOptions,
20
+ SiteInstalledForwardModelStep,
22
21
  WorkflowConfigs,
23
22
  WorkflowJob,
24
- forward_model_step_from_config_contents,
25
23
  workflow_job_from_file,
26
24
  )
27
25
  from ert.trace import add_span_processor
@@ -251,32 +249,9 @@ class ErtPluginManager(pluggy.PluginManager):
251
249
  return merged_dict
252
250
  return {k: v[0] for k, v in merged_dict.items()}
253
251
 
254
- def get_installable_jobs(self) -> Mapping[str, str]:
255
- return ErtPluginManager._merge_dicts(self.hook.installable_jobs())
256
-
257
252
  def _get_config_workflow_jobs(self) -> dict[str, str]:
258
253
  return ErtPluginManager._merge_dicts(self.hook.installable_workflow_jobs())
259
254
 
260
- def get_documentation_for_jobs(self) -> dict[str, Any]:
261
- job_docs = {
262
- k: {
263
- "config_file": v[0],
264
- "source_package": v[1].plugin_name,
265
- "source_function_name": v[1].function_name,
266
- }
267
- for k, v in ErtPluginManager._merge_dicts(
268
- self.hook.installable_jobs(), include_plugin_data=True
269
- ).items()
270
- }
271
- for key, value in job_docs.items():
272
- value.update(
273
- ErtPluginManager._evaluate_job_doc_hook(
274
- self.hook.job_documentation,
275
- key,
276
- )
277
- )
278
- return job_docs
279
-
280
255
  def get_documentation_for_forward_model_steps(
281
256
  self,
282
257
  ) -> dict[str, ForwardModelStepDocumentation]:
@@ -353,7 +328,7 @@ class ErtPluginManager(pluggy.PluginManager):
353
328
 
354
329
 
355
330
  class ErtRuntimePlugins(BaseModel):
356
- installed_forward_model_steps: Mapping[str, ForwardModelStep] = Field(
331
+ installed_forward_model_steps: Mapping[str, SiteInstalledForwardModelStep] = Field(
357
332
  default_factory=dict
358
333
  )
359
334
  installed_workflow_jobs: Mapping[str, WorkflowJob] = Field(default_factory=dict)
@@ -363,6 +338,7 @@ class ErtRuntimePlugins(BaseModel):
363
338
  environment_variables: Mapping[str, str] = Field(default_factory=dict)
364
339
  env_pr_fm_step: Mapping[str, Mapping[str, Any]] = Field(default_factory=dict)
365
340
  help_links: dict[str, str] = Field(default_factory=dict)
341
+ prioritize_private_ip_address: bool = False
366
342
 
367
343
 
368
344
  def get_site_plugins(
@@ -380,12 +356,6 @@ def get_site_plugins(
380
356
  else {}
381
357
  )
382
358
 
383
- for job_name, job_path in plugin_manager.get_installable_jobs().items():
384
- fm_step = forward_model_step_from_config_contents(
385
- Path(job_path).read_text(encoding="utf-8"), job_path, job_name
386
- )
387
- all_forward_model_steps[job_name] = fm_step
388
-
389
359
  all_workflow_jobs: dict[str, WorkflowJob] = dict[str, WorkflowJob](
390
360
  plugin_manager.get_ertscript_workflows().get_workflows()
391
361
  ) | dict[str, WorkflowJob](
@@ -417,6 +387,9 @@ def get_site_plugins(
417
387
  ),
418
388
  env_pr_fm_step=plugin_manager.get_forward_model_configuration(),
419
389
  help_links=plugin_manager.get_help_links(),
390
+ prioritize_private_ip_address=site_configurations.prioritize_private_ip_address
391
+ if site_configurations
392
+ else False,
420
393
  )
421
394
 
422
395
  return runtime_plugins
@@ -8,10 +8,9 @@ import subprocess
8
8
  import sys
9
9
  import time
10
10
  from argparse import ArgumentParser
11
- from collections import namedtuple
12
11
  from pathlib import Path
13
12
  from random import random
14
- from typing import Literal, get_args
13
+ from typing import Literal, NamedTuple, get_args
15
14
 
16
15
  import resfo
17
16
 
@@ -43,7 +42,13 @@ class EclError(RuntimeError):
43
42
 
44
43
 
45
44
  Simulators = Literal["flow", "eclipse", "e300"]
46
- EclipseResult = namedtuple("EclipseResult", "errors bugs")
45
+
46
+
47
+ class EclipseResult(NamedTuple):
48
+ errors: int
49
+ bugs: int
50
+
51
+
47
52
  body_sub_pattern = r"(\s^\s@.+$)*"
48
53
  date_sub_pattern = r"\s+AT TIME\s+(?P<Days>\d+\.\d+)\s+DAYS\s+\((?P<Date>(.+)):\s*$"
49
54
  error_pattern_e100 = (
@@ -48,8 +48,8 @@ def delete_directory(path: str) -> None:
48
48
  for file in files:
49
49
  delete_file(os.path.join(root, file))
50
50
 
51
- for _dir in dirs:
52
- delete_empty_directory(os.path.join(root, _dir))
51
+ for dir_ in dirs:
52
+ delete_empty_directory(os.path.join(root, dir_))
53
53
 
54
54
  else:
55
55
  raise OSError(f"Entry:'{path}' is not a directory")
@@ -1,6 +1,10 @@
1
- from .ensemble_experiment import EnsembleExperiment
2
- from .ensemble_information_filter import EnsembleInformationFilter
3
- from .ensemble_smoother import EnsembleSmoother
1
+ from .ensemble_experiment import EnsembleExperiment, EnsembleExperimentConfig
2
+ from .ensemble_information_filter import (
3
+ EnsembleInformationFilter,
4
+ EnsembleInformationFilterConfig,
5
+ )
6
+ from .ensemble_smoother import EnsembleSmoother, EnsembleSmootherConfig
7
+ from .evaluate_ensemble import EvaluateEnsembleConfig
4
8
  from .event import (
5
9
  RunModelEvent,
6
10
  RunModelStatusEvent,
@@ -9,21 +13,29 @@ from .event import (
9
13
  RunModelUpdateEndEvent,
10
14
  )
11
15
  from .model_factory import create_model
12
- from .multiple_data_assimilation import MultipleDataAssimilation
16
+ from .multiple_data_assimilation import (
17
+ MultipleDataAssimilation,
18
+ MultipleDataAssimilationConfig,
19
+ )
13
20
  from .run_model import (
14
21
  ErtRunError,
15
22
  RunModel,
16
23
  RunModelAPI,
17
24
  StatusEvents,
18
25
  )
19
- from .single_test_run import SingleTestRun
26
+ from .single_test_run import SingleTestRun, SingleTestRunConfig
20
27
 
21
28
  __all__ = [
22
29
  "EnsembleExperiment",
30
+ "EnsembleExperimentConfig",
23
31
  "EnsembleInformationFilter",
32
+ "EnsembleInformationFilterConfig",
24
33
  "EnsembleSmoother",
34
+ "EnsembleSmootherConfig",
25
35
  "ErtRunError",
36
+ "EvaluateEnsembleConfig",
26
37
  "MultipleDataAssimilation",
38
+ "MultipleDataAssimilationConfig",
27
39
  "RunModel",
28
40
  "RunModelAPI",
29
41
  "RunModelEvent",
@@ -32,6 +44,7 @@ __all__ = [
32
44
  "RunModelUpdateBeginEvent",
33
45
  "RunModelUpdateEndEvent",
34
46
  "SingleTestRun",
47
+ "SingleTestRunConfig",
35
48
  "StatusEvents",
36
49
  "create_model",
37
50
  ]
@@ -2,8 +2,12 @@ from __future__ import annotations
2
2
 
3
3
  import json
4
4
  import logging
5
+ import math
5
6
  import os
7
+ import time
8
+ from collections import defaultdict
6
9
  from collections.abc import Iterable, Mapping
10
+ from copy import deepcopy
7
11
  from datetime import UTC, datetime
8
12
  from pathlib import Path
9
13
  from typing import TYPE_CHECKING, Any
@@ -12,13 +16,16 @@ import orjson
12
16
 
13
17
  from _ert.utils import file_safe_timestamp
14
18
  from ert.config import (
15
- ExtParamConfig,
19
+ EverestControl,
16
20
  Field,
17
21
  ForwardModelStep,
18
22
  GenKwConfig,
23
+ ParameterCardinality,
19
24
  ParameterConfig,
20
25
  SurfaceConfig,
21
26
  )
27
+ from ert.config.design_matrix import DESIGN_MATRIX_GROUP
28
+ from ert.config.distribution import LogNormalSettings, LogUnifSettings
22
29
  from ert.config.ert_config import create_forward_model_json
23
30
  from ert.substitutions import Substitutions, substitute_runpath_name
24
31
  from ert.utils import log_duration
@@ -53,10 +60,17 @@ def _value_export_txt(
53
60
  with path.open("w") as f:
54
61
  for key, param_map in values.items():
55
62
  for param, value in param_map.items():
56
- if isinstance(value, (int | float)):
57
- print(f"{key}:{param} {value:g}", file=f)
63
+ if isinstance(value, int):
64
+ value_str = str(value)
65
+ elif isinstance(value, float):
66
+ value_str = f"{value:g}"
58
67
  else:
59
- print(f"{key}:{param} {value}", file=f)
68
+ value_str = str(value)
69
+
70
+ if key == DESIGN_MATRIX_GROUP:
71
+ print(f"{param} {value_str}", file=f)
72
+ else:
73
+ print(f"{key}:{param} {value_str}", file=f)
60
74
 
61
75
 
62
76
  def _value_export_json(
@@ -70,18 +84,18 @@ def _value_export_json(
70
84
  if len(values) == 0:
71
85
  return
72
86
 
73
- # Hierarchical
74
- json_out: dict[str, float | dict[str, float | str]] = {
75
- key: dict(param_map.items()) for key, param_map in values.items()
76
- }
87
+ # parameter file is {param: {"value": value}}
88
+ json_out: dict[str, dict[str, float | str]] = {}
89
+ for param_map in values.values():
90
+ for param, value in param_map.items():
91
+ json_out[param] = {"value": value}
77
92
 
78
93
  # Disallow NaN from being written: ERT produces the parameters and the only
79
94
  # way for the output to be NaN is if the input is invalid or if the sampling
80
95
  # function is buggy. Either way, that would be a bug and we can report it by
81
96
  # having json throw an error.
82
- json.dump(
83
- json_out, path.open("w"), allow_nan=False, indent=0, separators=(", ", " : ")
84
- )
97
+ with path.open("w") as f:
98
+ json.dump(json_out, f, allow_nan=False, indent=0, separators=(", ", " : "))
85
99
 
86
100
 
87
101
  def _generate_parameter_files(
@@ -91,7 +105,7 @@ def _generate_parameter_files(
91
105
  iens: int,
92
106
  fs: Ensemble,
93
107
  iteration: int,
94
- ) -> Mapping[str, Mapping[str, float | str]]:
108
+ ) -> tuple[Mapping[str, Mapping[str, float | str]], Mapping[str, float]]:
95
109
  """
96
110
  Generate parameter files that are placed in each runtime directory for
97
111
  forward-model jobs to consume.
@@ -106,10 +120,25 @@ def _generate_parameter_files(
106
120
  fs: Ensemble from which to load parameter data
107
121
 
108
122
  Returns:
109
- Returns the union of parameters returned by write_to_runpath for each
110
- parameter_config.
123
+ Returns a tuple containing: the union of parameters returned by
124
+ write_to_runpath for each parameter_config, and a dict with
125
+ timings/durations for each parameter type.
111
126
  """
127
+ # preload scalar parameters for this realization
128
+ keys = [
129
+ p.name
130
+ for p in parameter_configs
131
+ if p.cardinality == ParameterCardinality.multiple_configs_per_ensemble_dataset
132
+ ]
133
+ export_timings: defaultdict[str, float] = defaultdict(float)
134
+ scalar_data: dict[str, float | str] = {}
135
+ if keys:
136
+ start_time = time.perf_counter()
137
+ df = fs.load_scalar_keys(keys=keys, realizations=iens, transformed=True)
138
+ scalar_data = df.to_dicts()[0]
139
+ export_timings["load_scalar_keys"] = time.perf_counter() - start_time
112
140
  exports: dict[str, dict[str, float | str]] = {}
141
+ log_exports: dict[str, dict[str, float | str]] = {}
113
142
 
114
143
  for param in parameter_configs:
115
144
  # For the first iteration we do not write the parameter
@@ -117,15 +146,42 @@ def _generate_parameter_files(
117
146
  # model has completed.
118
147
  if param.forward_init and iteration == 0:
119
148
  continue
120
- export_values = param.write_to_runpath(Path(run_path), iens, fs)
149
+ start_time = time.perf_counter()
150
+ export_values: dict[str, dict[str, float | str]] | None = None
151
+ log_export_values: dict[str, dict[str, float | str]] | None = {}
152
+ if param.name in scalar_data:
153
+ scalar_value = scalar_data[param.name]
154
+ export_values = {param.group_name: {param.name: scalar_value}}
155
+ if isinstance(param, GenKwConfig) and isinstance(
156
+ param.distribution, (LogNormalSettings, LogUnifSettings)
157
+ ):
158
+ if isinstance(scalar_value, float) and scalar_value > 0:
159
+ log_value = math.log10(scalar_value)
160
+ log_export_values = {
161
+ f"LOG10_{param.group_name}": {param.name: log_value}
162
+ }
163
+ else:
164
+ logger.warning(
165
+ "Could not export the log10 value of "
166
+ f"{scalar_value} as it is invalid"
167
+ )
168
+ else:
169
+ export_values = param.write_to_runpath(Path(run_path), iens, fs)
121
170
  if export_values:
122
171
  for group, vals in export_values.items():
123
172
  exports.setdefault(group, {}).update(vals)
173
+ if log_export_values:
174
+ for group, vals in log_export_values.items():
175
+ log_exports.setdefault(group, {}).update(vals)
176
+ export_timings[param.type] += time.perf_counter() - start_time
124
177
  continue
125
-
126
- _value_export_txt(run_path, export_base_name, exports)
178
+ start_time = time.perf_counter()
179
+ _value_export_txt(run_path, export_base_name, exports | log_exports)
180
+ export_timings["value_export_txt"] = time.perf_counter() - start_time
181
+ start_time = time.perf_counter()
127
182
  _value_export_json(run_path, export_base_name, exports)
128
- return exports
183
+ export_timings["value_export_json"] = time.perf_counter() - start_time
184
+ return (exports, dict(export_timings))
129
185
 
130
186
 
131
187
  def _manifest_to_json(ensemble: Ensemble, iens: int, iter_: int) -> dict[str, Any]:
@@ -134,7 +190,7 @@ def _manifest_to_json(ensemble: Ensemble, iens: int, iter_: int) -> dict[str, An
134
190
  for param_config in ensemble.experiment.parameter_configuration.values():
135
191
  assert isinstance(
136
192
  param_config,
137
- ExtParamConfig | GenKwConfig | Field | SurfaceConfig,
193
+ EverestControl | GenKwConfig | Field | SurfaceConfig,
138
194
  )
139
195
  if param_config.forward_init and ensemble.iteration == 0:
140
196
  assert not isinstance(param_config, GenKwConfig)
@@ -146,13 +202,32 @@ def _manifest_to_json(ensemble: Ensemble, iens: int, iter_: int) -> dict[str, An
146
202
  # Add expected response files to manifest
147
203
  for response_config in ensemble.experiment.response_configuration.values():
148
204
  for input_file in response_config.expected_input_files:
149
- manifest[f"{response_config.response_type}_{input_file}"] = (
150
- substitute_runpath_name(input_file, iens, iter_)
205
+ manifest[f"{response_config.type}_{input_file}"] = substitute_runpath_name(
206
+ input_file, iens, iter_
151
207
  )
152
208
 
153
209
  return manifest
154
210
 
155
211
 
212
+ def _make_param_substituter(
213
+ substituter: Substitutions,
214
+ param_data: Mapping[str, Mapping[str, str | float]],
215
+ ) -> Substitutions:
216
+ param_substituter = deepcopy(substituter)
217
+ for values in param_data.values():
218
+ for param_name, value in values.items():
219
+ if isinstance(value, int):
220
+ formatted_value = str(value)
221
+ elif isinstance(value, float):
222
+ formatted_value = f"{value:g}"
223
+ else:
224
+ formatted_value = str(value)
225
+
226
+ param_substituter[f"<{param_name}>"] = formatted_value
227
+
228
+ return param_substituter
229
+
230
+
156
231
  @log_duration(logger, logging.INFO)
157
232
  def create_run_path(
158
233
  run_args: list[RunArg],
@@ -169,13 +244,19 @@ def create_run_path(
169
244
  if context_env is None:
170
245
  context_env = {}
171
246
  runpaths.set_ert_ensemble(ensemble.name)
172
-
247
+ timings = {
248
+ "generate_parameter_files": 0.0,
249
+ "substitute_parameters": 0.0,
250
+ "substitute_real_iter": 0.0,
251
+ "result_file_to_target": 0.0,
252
+ }
173
253
  substituter = Substitutions(substitutions)
174
254
  for run_arg in run_args:
175
255
  run_path = Path(run_arg.runpath)
176
256
  if run_arg.active:
177
257
  run_path.mkdir(parents=True, exist_ok=True)
178
- param_data = _generate_parameter_files(
258
+ start_time = time.perf_counter()
259
+ (param_data, detailed_parameter_timings) = _generate_parameter_files(
179
260
  ensemble.experiment.parameter_configuration.values(),
180
261
  parameters_file,
181
262
  run_path,
@@ -183,22 +264,29 @@ def create_run_path(
183
264
  ensemble,
184
265
  ensemble.iteration,
185
266
  )
267
+ for parameter_type, duration in detailed_parameter_timings.items():
268
+ if parameter_type not in timings:
269
+ timings[parameter_type] = 0.0
270
+ timings[parameter_type] += duration
271
+
272
+ timings["generate_parameter_files"] += time.perf_counter() - start_time
273
+ real_iter_substituter = substituter.real_iter_substituter(
274
+ run_arg.iens, ensemble.iteration
275
+ )
276
+ param_substituter = _make_param_substituter(
277
+ real_iter_substituter, param_data
278
+ )
186
279
  for (
187
280
  source_file_content,
188
281
  target_file,
189
282
  ) in ensemble.experiment.templates_configuration:
190
- target_file = substituter.substitute_real_iter(
191
- target_file, run_arg.iens, ensemble.iteration
192
- )
193
- result = substituter.substitute_real_iter(
194
- source_file_content,
195
- run_arg.iens,
196
- ensemble.iteration,
197
- )
198
- result = substituter.substitute_parameters(
199
- result,
200
- param_data,
201
- )
283
+ start_time = time.perf_counter()
284
+ target_file = real_iter_substituter.substitute(target_file)
285
+ timings["substitute_real_iter"] += time.perf_counter() - start_time
286
+ start_time = time.perf_counter()
287
+ result = param_substituter.substitute(source_file_content)
288
+ timings["substitute_parameters"] += time.perf_counter() - start_time
289
+ start_time = time.perf_counter()
202
290
  target = run_path / target_file
203
291
  if not target.parent.exists():
204
292
  os.makedirs(
@@ -206,10 +294,13 @@ def create_run_path(
206
294
  exist_ok=True,
207
295
  )
208
296
  target.write_text(result)
297
+ timings["result_file_to_target"] += time.perf_counter() - start_time
209
298
 
210
299
  path = run_path / "jobs.json"
300
+ start_time = time.perf_counter()
211
301
  _backup_if_existing(path)
212
-
302
+ timings["backup_if_existing"] = time.perf_counter() - start_time
303
+ start_time = time.perf_counter()
213
304
  forward_model_output = create_forward_model_json(
214
305
  context=substitutions,
215
306
  forward_model_steps=forward_model_steps,
@@ -226,12 +317,15 @@ def create_run_path(
226
317
  option=orjson.OPT_NON_STR_KEYS | orjson.OPT_INDENT_2,
227
318
  )
228
319
  )
320
+ timings["jobs_to_json"] = time.perf_counter() - start_time
229
321
  # Write MANIFEST file to runpath use to avoid NFS sync issues
322
+ start_time = time.perf_counter()
230
323
  data = _manifest_to_json(ensemble, run_arg.iens, run_arg.itr)
231
324
  Path(run_path / "manifest.json").write_bytes(
232
325
  orjson.dumps(data, option=orjson.OPT_NON_STR_KEYS | orjson.OPT_INDENT_2)
233
326
  )
234
-
327
+ timings["manifest_to_json"] = time.perf_counter() - start_time
328
+ logger.info(f"_create_run_path durations: {timings}")
235
329
  runpaths.write_runpath_list(
236
330
  [ensemble.iteration], [real.iens for real in run_args if real.active]
237
331
  )
@@ -13,14 +13,22 @@ from ert.config import (
13
13
  ResponseConfig,
14
14
  )
15
15
  from ert.ensemble_evaluator import EvaluatorServerConfig
16
- from ert.run_models.initial_ensemble_run_model import InitialEnsembleRunModel
16
+ from ert.run_models.initial_ensemble_run_model import (
17
+ InitialEnsembleRunModel,
18
+ InitialEnsembleRunModelConfig,
19
+ )
17
20
  from ert.storage import Ensemble
18
21
  from ert.trace import tracer
19
22
 
20
23
  logger = logging.getLogger(__name__)
21
24
 
22
25
 
23
- class EnsembleExperiment(InitialEnsembleRunModel):
26
+ class EnsembleExperimentConfig(InitialEnsembleRunModelConfig):
27
+ target_ensemble: str
28
+ supports_rerunning_failed_realizations: ClassVar[bool] = True
29
+
30
+
31
+ class EnsembleExperiment(InitialEnsembleRunModel, EnsembleExperimentConfig):
24
32
  """
25
33
  This workflow will create a new experiment and a new ensemble from
26
34
  the user configuration.<br>It will never overwrite existing ensembles, and
@@ -28,8 +36,6 @@ class EnsembleExperiment(InitialEnsembleRunModel):
28
36
  """
29
37
 
30
38
  _ensemble_id: UUID | None = PrivateAttr(None)
31
- supports_rerunning_failed_realizations: ClassVar[bool] = True
32
- target_ensemble: str
33
39
 
34
40
  @property
35
41
  def _ensemble(self) -> Ensemble:
@@ -7,11 +7,18 @@ from ert.run_models.ensemble_smoother import EnsembleSmoother
7
7
  from ert.storage import Ensemble
8
8
 
9
9
  from ..analysis import enif_update
10
+ from .initial_ensemble_run_model import InitialEnsembleRunModelConfig
11
+ from .update_run_model import UpdateRunModelConfig
10
12
 
11
13
  logger = logging.getLogger(__name__)
12
14
 
13
15
 
14
- class EnsembleInformationFilter(EnsembleSmoother):
16
+ class EnsembleInformationFilterConfig(
17
+ InitialEnsembleRunModelConfig, UpdateRunModelConfig
18
+ ): ...
19
+
20
+
21
+ class EnsembleInformationFilter(EnsembleSmoother, EnsembleInformationFilterConfig):
15
22
  def update_ensemble_parameters(
16
23
  self, prior: Ensemble, posterior: Ensemble, weight: float
17
24
  ) -> None:
@@ -14,8 +14,11 @@ from ert.config import (
14
14
  ResponseConfig,
15
15
  )
16
16
  from ert.ensemble_evaluator import EvaluatorServerConfig
17
- from ert.run_models.initial_ensemble_run_model import InitialEnsembleRunModel
18
- from ert.run_models.update_run_model import UpdateRunModel
17
+ from ert.run_models.initial_ensemble_run_model import (
18
+ InitialEnsembleRunModel,
19
+ InitialEnsembleRunModelConfig,
20
+ )
21
+ from ert.run_models.update_run_model import UpdateRunModel, UpdateRunModelConfig
19
22
  from ert.storage import Ensemble
20
23
  from ert.trace import tracer
21
24
 
@@ -26,7 +29,10 @@ from .run_model import ErtRunError
26
29
  logger = logging.getLogger(__name__)
27
30
 
28
31
 
29
- class EnsembleSmoother(UpdateRunModel, InitialEnsembleRunModel):
32
+ class EnsembleSmootherConfig(InitialEnsembleRunModelConfig, UpdateRunModelConfig): ...
33
+
34
+
35
+ class EnsembleSmoother(InitialEnsembleRunModel, UpdateRunModel, EnsembleSmootherConfig):
30
36
  _total_iterations: int = PrivateAttr(default=2)
31
37
 
32
38
  @tracer.start_as_current_span(f"{__name__}.run_experiment")