ert 17.1.9__py3-none-any.whl → 18.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (165) hide show
  1. _ert/events.py +19 -2
  2. ert/__main__.py +8 -7
  3. ert/analysis/_update_commons.py +12 -3
  4. ert/cli/main.py +6 -3
  5. ert/cli/monitor.py +7 -0
  6. ert/config/__init__.py +13 -3
  7. ert/config/_create_observation_dataframes.py +60 -12
  8. ert/config/_observations.py +14 -1
  9. ert/config/_read_summary.py +8 -6
  10. ert/config/ensemble_config.py +6 -14
  11. ert/config/ert_config.py +19 -13
  12. ert/config/{everest_objective_config.py → everest_response.py} +23 -12
  13. ert/config/ext_param_config.py +133 -1
  14. ert/config/field.py +12 -8
  15. ert/config/forward_model_step.py +108 -6
  16. ert/config/gen_data_config.py +2 -6
  17. ert/config/gen_kw_config.py +0 -9
  18. ert/config/known_response_types.py +14 -0
  19. ert/config/parameter_config.py +0 -17
  20. ert/config/parsing/config_keywords.py +1 -0
  21. ert/config/parsing/config_schema.py +12 -0
  22. ert/config/parsing/config_schema_deprecations.py +11 -0
  23. ert/config/parsing/config_schema_item.py +1 -1
  24. ert/config/queue_config.py +4 -4
  25. ert/config/response_config.py +0 -7
  26. ert/config/rft_config.py +230 -0
  27. ert/config/summary_config.py +2 -6
  28. ert/config/violations.py +0 -0
  29. ert/config/workflow_fixtures.py +2 -1
  30. ert/dark_storage/client/__init__.py +2 -2
  31. ert/dark_storage/client/_session.py +4 -4
  32. ert/dark_storage/client/client.py +2 -2
  33. ert/dark_storage/compute/misfits.py +7 -6
  34. ert/dark_storage/endpoints/compute/misfits.py +2 -2
  35. ert/dark_storage/endpoints/observations.py +4 -4
  36. ert/dark_storage/endpoints/responses.py +15 -1
  37. ert/ensemble_evaluator/__init__.py +8 -1
  38. ert/ensemble_evaluator/evaluator.py +81 -29
  39. ert/ensemble_evaluator/event.py +6 -0
  40. ert/ensemble_evaluator/snapshot.py +3 -1
  41. ert/ensemble_evaluator/state.py +1 -0
  42. ert/field_utils/__init__.py +8 -0
  43. ert/field_utils/field_utils.py +211 -1
  44. ert/gui/ertwidgets/__init__.py +23 -16
  45. ert/gui/ertwidgets/analysismoduleedit.py +2 -2
  46. ert/gui/ertwidgets/checklist.py +1 -1
  47. ert/gui/ertwidgets/create_experiment_dialog.py +3 -1
  48. ert/gui/ertwidgets/ensembleselector.py +2 -2
  49. ert/gui/ertwidgets/models/__init__.py +2 -0
  50. ert/gui/ertwidgets/models/activerealizationsmodel.py +2 -1
  51. ert/gui/ertwidgets/models/path_model.py +1 -1
  52. ert/gui/ertwidgets/models/targetensemblemodel.py +2 -1
  53. ert/gui/ertwidgets/models/text_model.py +1 -1
  54. ert/gui/ertwidgets/searchbox.py +13 -4
  55. ert/gui/{suggestor → ertwidgets/suggestor}/_suggestor_message.py +13 -4
  56. ert/gui/main.py +11 -6
  57. ert/gui/main_window.py +1 -2
  58. ert/gui/simulation/ensemble_experiment_panel.py +1 -1
  59. ert/gui/simulation/ensemble_information_filter_panel.py +1 -1
  60. ert/gui/simulation/ensemble_smoother_panel.py +1 -1
  61. ert/gui/simulation/evaluate_ensemble_panel.py +1 -1
  62. ert/gui/simulation/experiment_panel.py +1 -1
  63. ert/gui/simulation/manual_update_panel.py +31 -8
  64. ert/gui/simulation/multiple_data_assimilation_panel.py +12 -8
  65. ert/gui/simulation/run_dialog.py +25 -4
  66. ert/gui/simulation/single_test_run_panel.py +2 -2
  67. ert/gui/summarypanel.py +1 -1
  68. ert/gui/tools/load_results/load_results_panel.py +1 -1
  69. ert/gui/tools/manage_experiments/storage_info_widget.py +7 -7
  70. ert/gui/tools/manage_experiments/storage_widget.py +1 -2
  71. ert/gui/tools/plot/plot_api.py +13 -10
  72. ert/gui/tools/plot/plot_window.py +12 -0
  73. ert/gui/tools/plot/plottery/plot_config.py +2 -0
  74. ert/gui/tools/plot/plottery/plot_context.py +14 -0
  75. ert/gui/tools/plot/plottery/plots/ensemble.py +9 -2
  76. ert/gui/tools/plot/plottery/plots/statistics.py +59 -19
  77. ert/mode_definitions.py +2 -0
  78. ert/plugins/__init__.py +0 -1
  79. ert/plugins/hook_implementations/workflows/gen_data_rft_export.py +10 -2
  80. ert/plugins/hook_specifications/__init__.py +0 -2
  81. ert/plugins/hook_specifications/jobs.py +0 -9
  82. ert/plugins/plugin_manager.py +2 -33
  83. ert/resources/shell_scripts/delete_directory.py +2 -2
  84. ert/run_models/__init__.py +18 -5
  85. ert/run_models/_create_run_path.py +33 -21
  86. ert/run_models/ensemble_experiment.py +10 -4
  87. ert/run_models/ensemble_information_filter.py +8 -1
  88. ert/run_models/ensemble_smoother.py +9 -3
  89. ert/run_models/evaluate_ensemble.py +8 -6
  90. ert/run_models/event.py +7 -3
  91. ert/run_models/everest_run_model.py +155 -44
  92. ert/run_models/initial_ensemble_run_model.py +23 -22
  93. ert/run_models/manual_update.py +4 -2
  94. ert/run_models/manual_update_enif.py +37 -0
  95. ert/run_models/model_factory.py +81 -22
  96. ert/run_models/multiple_data_assimilation.py +21 -10
  97. ert/run_models/run_model.py +54 -34
  98. ert/run_models/single_test_run.py +7 -4
  99. ert/run_models/update_run_model.py +4 -2
  100. ert/runpaths.py +5 -6
  101. ert/sample_prior.py +9 -4
  102. ert/scheduler/driver.py +37 -0
  103. ert/scheduler/event.py +3 -1
  104. ert/scheduler/job.py +23 -13
  105. ert/scheduler/lsf_driver.py +6 -2
  106. ert/scheduler/openpbs_driver.py +7 -1
  107. ert/scheduler/scheduler.py +5 -0
  108. ert/scheduler/slurm_driver.py +6 -2
  109. ert/services/__init__.py +2 -2
  110. ert/services/_base_service.py +31 -15
  111. ert/services/ert_server.py +317 -0
  112. ert/shared/_doc_utils/ert_jobs.py +1 -4
  113. ert/shared/storage/connection.py +3 -3
  114. ert/shared/version.py +3 -3
  115. ert/storage/local_ensemble.py +25 -5
  116. ert/storage/local_experiment.py +6 -14
  117. ert/storage/local_storage.py +35 -30
  118. ert/storage/migration/to18.py +12 -0
  119. ert/storage/migration/to8.py +4 -4
  120. ert/substitutions.py +12 -28
  121. ert/validation/active_range.py +7 -7
  122. ert/validation/rangestring.py +16 -16
  123. {ert-17.1.9.dist-info → ert-18.0.0.dist-info}/METADATA +8 -7
  124. {ert-17.1.9.dist-info → ert-18.0.0.dist-info}/RECORD +160 -159
  125. everest/api/everest_data_api.py +1 -14
  126. everest/bin/config_branch_script.py +3 -6
  127. everest/bin/everconfigdump_script.py +1 -9
  128. everest/bin/everest_script.py +21 -11
  129. everest/bin/kill_script.py +2 -2
  130. everest/bin/monitor_script.py +2 -2
  131. everest/bin/utils.py +6 -3
  132. everest/config/__init__.py +4 -1
  133. everest/config/control_config.py +61 -2
  134. everest/config/control_variable_config.py +2 -1
  135. everest/config/everest_config.py +38 -16
  136. everest/config/forward_model_config.py +5 -3
  137. everest/config/install_data_config.py +7 -5
  138. everest/config/install_job_config.py +7 -3
  139. everest/config/install_template_config.py +3 -3
  140. everest/config/optimization_config.py +19 -6
  141. everest/config/output_constraint_config.py +8 -2
  142. everest/config/server_config.py +6 -49
  143. everest/config/utils.py +25 -105
  144. everest/config/validation_utils.py +10 -10
  145. everest/config_file_loader.py +13 -2
  146. everest/detached/everserver.py +7 -8
  147. everest/everest_storage.py +6 -10
  148. everest/gui/everest_client.py +0 -1
  149. everest/gui/main_window.py +2 -2
  150. everest/optimizer/everest2ropt.py +59 -32
  151. everest/optimizer/opt_model_transforms.py +12 -13
  152. everest/optimizer/utils.py +0 -29
  153. everest/strings.py +0 -5
  154. ert/config/everest_constraints_config.py +0 -95
  155. ert/services/storage_service.py +0 -127
  156. everest/config/sampler_config.py +0 -103
  157. everest/simulator/__init__.py +0 -88
  158. everest/simulator/everest_to_ert.py +0 -51
  159. /ert/gui/{suggestor → ertwidgets/suggestor}/__init__.py +0 -0
  160. /ert/gui/{suggestor → ertwidgets/suggestor}/_colors.py +0 -0
  161. /ert/gui/{suggestor → ertwidgets/suggestor}/suggestor.py +0 -0
  162. {ert-17.1.9.dist-info → ert-18.0.0.dist-info}/WHEEL +0 -0
  163. {ert-17.1.9.dist-info → ert-18.0.0.dist-info}/entry_points.txt +0 -0
  164. {ert-17.1.9.dist-info → ert-18.0.0.dist-info}/licenses/COPYING +0 -0
  165. {ert-17.1.9.dist-info → ert-18.0.0.dist-info}/top_level.txt +0 -0
_ert/events.py CHANGED
@@ -41,10 +41,12 @@ class Id:
41
41
  ENSEMBLE_SUCCEEDED_TYPE = Literal["ensemble.succeeded"]
42
42
  ENSEMBLE_CANCELLED_TYPE = Literal["ensemble.cancelled"]
43
43
  ENSEMBLE_FAILED_TYPE = Literal["ensemble.failed"]
44
+ ENSEMBLE_WARNING_TYPE = Literal["ensemble.warning"]
44
45
  ENSEMBLE_STARTED: Final = "ensemble.started"
45
46
  ENSEMBLE_SUCCEEDED: Final = "ensemble.succeeded"
46
47
  ENSEMBLE_CANCELLED: Final = "ensemble.cancelled"
47
48
  ENSEMBLE_FAILED: Final = "ensemble.failed"
49
+ ENSEMBLE_WARNING: Final = "ensemble.warning"
48
50
  ENSEMBLE_TYPES = (
49
51
  ENSEMBLE_STARTED_TYPE
50
52
  | ENSEMBLE_FAILED_TYPE
@@ -170,6 +172,15 @@ class EnsembleCancelled(EnsembleBaseEvent):
170
172
  event_type: Id.ENSEMBLE_CANCELLED_TYPE = Id.ENSEMBLE_CANCELLED
171
173
 
172
174
 
175
+ class EnsembleEvaluationWarning(EnsembleBaseEvent):
176
+ """This event is to indicate that something unexpected happened while
177
+ running the ensemble, and that it might be stuck in an unresponsive state.
178
+ """
179
+
180
+ event_type: Id.ENSEMBLE_WARNING_TYPE = Id.ENSEMBLE_WARNING
181
+ warning_message: str
182
+
183
+
173
184
  class EESnapshot(EnsembleBaseEvent):
174
185
  event_type: Id.EE_SNAPSHOT_TYPE = Id.EE_SNAPSHOT
175
186
  snapshot: Any
@@ -200,9 +211,15 @@ RealizationEvent = (
200
211
  | RealizationResubmit
201
212
  )
202
213
 
203
- EnsembleEvent = EnsembleStarted | EnsembleSucceeded | EnsembleFailed | EnsembleCancelled
214
+ EnsembleEvent = (
215
+ EnsembleStarted
216
+ | EnsembleSucceeded
217
+ | EnsembleFailed
218
+ | EnsembleCancelled
219
+ | EnsembleEvaluationWarning
220
+ )
204
221
 
205
- EEEvent = EESnapshot | EESnapshotUpdate
222
+ EEEvent = EESnapshot | EESnapshotUpdate | EnsembleEvaluationWarning
206
223
 
207
224
  SnapshotInputEvent = RealizationEvent | EnsembleEvent | FMEvent
208
225
 
ert/__main__.py CHANGED
@@ -35,8 +35,9 @@ from ert.mode_definitions import (
35
35
  )
36
36
  from ert.namespace import Namespace
37
37
  from ert.plugins import ErtRuntimePlugins, get_site_plugins, setup_site_logging
38
- from ert.run_models.multiple_data_assimilation import MultipleDataAssimilation
39
- from ert.services import StorageService, WebvizErt
38
+ from ert.run_models.multiple_data_assimilation import MultipleDataAssimilationConfig
39
+ from ert.services import ErtServer, WebvizErt
40
+ from ert.shared.status.utils import get_ert_memory_usage
40
41
  from ert.shared.storage.command import add_parser_options as ert_api_add_parser_options
41
42
  from ert.storage import ErtStorageException, ErtStoragePermissionError
42
43
  from ert.trace import trace, tracer
@@ -53,9 +54,9 @@ logger = logging.getLogger(__name__)
53
54
 
54
55
 
55
56
  def run_ert_storage(args: Namespace, _: ErtRuntimePlugins | None = None) -> None:
56
- with StorageService.start_server(
57
+ with ErtServer.start_server(
57
58
  verbose=True,
58
- project=ErtConfig.from_file(args.config).ens_path,
59
+ project=Path(ErtConfig.from_file(args.config).ens_path),
59
60
  parent_pid=os.getpid(),
60
61
  ) as server:
61
62
  server.wait()
@@ -80,7 +81,7 @@ def run_webviz_ert(args: Namespace, _: ErtRuntimePlugins | None = None) -> None:
80
81
  kwargs["ert_config"] = os.path.basename(args.config)
81
82
  kwargs["project"] = os.path.abspath(ens_path)
82
83
  try:
83
- with StorageService.init_service(project=os.path.abspath(ens_path)) as storage:
84
+ with ErtServer.init_service(project=Path(ens_path).absolute()) as storage:
84
85
  storage.wait_until_ready()
85
86
  print(
86
87
  """
@@ -501,7 +502,7 @@ def get_ert_parser(parser: ArgumentParser | None = None) -> ArgumentParser:
501
502
  es_mda_parser.add_argument(
502
503
  "--weights",
503
504
  type=valid_weights,
504
- default=MultipleDataAssimilation.default_weights,
505
+ default=MultipleDataAssimilationConfig.default_weights,
505
506
  help="Example custom relative weights: '8,4,2,1'. This means multiple data "
506
507
  "assimilation ensemble smoother will half the weight applied to the "
507
508
  "observation errors from one iteration to the next across 4 iterations.",
@@ -598,7 +599,7 @@ def ert_parser(parser: ArgumentParser | None, args: Sequence[str]) -> Namespace:
598
599
  def log_process_usage() -> None:
599
600
  try:
600
601
  usage = resource.getrusage(resource.RUSAGE_SELF)
601
- max_rss = ert.shared.status.utils.get_ert_memory_usage()
602
+ max_rss = get_ert_memory_usage()
602
603
 
603
604
  usage_dict: dict[str, int | float] = {
604
605
  "User time": usage.ru_utime,
@@ -61,10 +61,19 @@ def _copy_unupdated_parameters(
61
61
 
62
62
  # Copy the non-updated parameter groups from source to target
63
63
  # for each active realization
64
+ complete_df: pl.DataFrame | None = None
64
65
  for parameter_group in not_updated_parameter_groups:
65
- source_ensemble.experiment.parameter_configuration[
66
- parameter_group
67
- ].copy_parameters(source_ensemble, target_ensemble, iens_active_index)
66
+ data = source_ensemble.load_parameters(parameter_group, iens_active_index)
67
+ if isinstance(data, pl.DataFrame):
68
+ if complete_df is None:
69
+ complete_df = data
70
+ else:
71
+ complete_df = complete_df.join(data, on="realization")
72
+ else:
73
+ target_ensemble.save_parameters(dataset=data)
74
+
75
+ if complete_df is not None:
76
+ target_ensemble.save_parameters(complete_df)
68
77
 
69
78
 
70
79
  def _expand_wildcards(
ert/cli/main.py CHANGED
@@ -93,6 +93,7 @@ def run_cli(args: Namespace, runtime_plugins: ErtRuntimePlugins | None = None) -
93
93
  return
94
94
 
95
95
  status_queue: queue.SimpleQueue[StatusEvents] = queue.SimpleQueue()
96
+ using_local_queuesystem: bool = True
96
97
  try:
97
98
  with use_runtime_plugins(get_site_plugins()):
98
99
  model = create_model(
@@ -100,6 +101,9 @@ def run_cli(args: Namespace, runtime_plugins: ErtRuntimePlugins | None = None) -
100
101
  args,
101
102
  status_queue,
102
103
  )
104
+ using_local_queuesystem = (
105
+ ert_config.queue_config.queue_system == QueueSystem.LOCAL
106
+ )
103
107
  except ValueError as e:
104
108
  raise ErtCliError(f"{args.mode} was not valid, failed with: {e}") from e
105
109
 
@@ -116,17 +120,16 @@ def run_cli(args: Namespace, runtime_plugins: ErtRuntimePlugins | None = None) -
116
120
  f"and DESIGN_MATRIX ({model.active_realizations.count(True)})"
117
121
  )
118
122
 
119
- if args.port_range is None and model.queue_system == QueueSystem.LOCAL:
123
+ if args.port_range is None and using_local_queuesystem:
120
124
  # This is within the range for ephemeral ports as defined by
121
125
  # most unix flavors https://en.wikipedia.org/wiki/Ephemeral_port
122
126
  args.port_range = range(49152, 51819)
123
127
 
124
- use_ipc_protocol = model.queue_system == QueueSystem.LOCAL
125
128
  evaluator_server_config = EvaluatorServerConfig(
126
129
  port_range=None
127
130
  if args.port_range is None
128
131
  else (min(args.port_range), max(args.port_range) + 1),
129
- use_ipc_protocol=use_ipc_protocol,
132
+ use_ipc_protocol=using_local_queuesystem,
130
133
  )
131
134
 
132
135
  if model.check_if_runpath_exists():
ert/cli/monitor.py CHANGED
@@ -9,6 +9,7 @@ from typing import TextIO
9
9
  import humanize
10
10
  from tqdm import tqdm
11
11
 
12
+ from _ert.events import EnsembleEvaluationWarning
12
13
  from ert.ensemble_evaluator import (
13
14
  EndEvent,
14
15
  EnsembleSnapshot,
@@ -19,6 +20,7 @@ from ert.ensemble_evaluator import identifiers as ids
19
20
  from ert.ensemble_evaluator.state import (
20
21
  COLOR_FAILED,
21
22
  COLOR_FINISHED,
23
+ COLOR_WARNING,
22
24
  FORWARD_MODEL_STATE_FAILURE,
23
25
  REAL_STATE_TO_COLOR,
24
26
  )
@@ -96,6 +98,11 @@ class Monitor:
96
98
  | RunModelErrorEvent() as event
97
99
  ):
98
100
  event.write_as_csv(output_path)
101
+ case EnsembleEvaluationWarning(warning_message=msg):
102
+ print(
103
+ self._colorize(msg, color=COLOR_WARNING),
104
+ file=self._out,
105
+ )
99
106
 
100
107
  def _print_step_errors(self) -> None:
101
108
  failed_steps: dict[str | None, int] = {}
ert/config/__init__.py CHANGED
@@ -11,9 +11,8 @@ from .ensemble_config import EnsembleConfig
11
11
  from .ert_config import ErtConfig, forward_model_step_from_config_contents
12
12
  from .ert_plugin import ErtPlugin
13
13
  from .ert_script import ErtScript
14
- from .everest_constraints_config import EverestConstraintsConfig
15
- from .everest_objective_config import EverestObjectivesConfig
16
- from .ext_param_config import ExtParamConfig
14
+ from .everest_response import EverestConstraintsConfig, EverestObjectivesConfig
15
+ from .ext_param_config import ExtParamConfig, SamplerConfig
17
16
  from .external_ert_script import ExternalErtScript
18
17
  from .field import Field, field_transform
19
18
  from .forward_model_step import (
@@ -23,9 +22,13 @@ from .forward_model_step import (
23
22
  ForwardModelStepPlugin,
24
23
  ForwardModelStepValidationError,
25
24
  ForwardModelStepWarning,
25
+ SiteInstalledForwardModelStep,
26
+ SiteOrUserForwardModelStep,
27
+ UserInstalledForwardModelStep,
26
28
  )
27
29
  from .gen_data_config import GenDataConfig
28
30
  from .gen_kw_config import DataSource, GenKwConfig, PriorDict
31
+ from .known_response_types import KnownResponseTypes
29
32
  from .lint_file import lint_file
30
33
  from .model_config import ModelConfig
31
34
  from .parameter_config import ParameterCardinality, ParameterConfig, ParameterMetadata
@@ -45,6 +48,7 @@ from .queue_config import (
45
48
  QueueConfig,
46
49
  )
47
50
  from .response_config import InvalidResponseFile, ResponseConfig, ResponseMetadata
51
+ from .rft_config import RFTConfig
48
52
  from .summary_config import SummaryConfig
49
53
  from .surface_config import SurfaceConfig
50
54
  from .workflow import Workflow
@@ -105,6 +109,7 @@ __all__ = [
105
109
  "InversionTypeES",
106
110
  "KnownQueueOptions",
107
111
  "KnownQueueOptionsAdapter",
112
+ "KnownResponseTypes",
108
113
  "LegacyWorkflowConfigs",
109
114
  "LocalQueueOptions",
110
115
  "ModelConfig",
@@ -125,10 +130,15 @@ __all__ = [
125
130
  "PriorDict",
126
131
  "QueueConfig",
127
132
  "QueueSystem",
133
+ "RFTConfig",
128
134
  "ResponseConfig",
129
135
  "ResponseMetadata",
136
+ "SamplerConfig",
137
+ "SiteInstalledForwardModelStep",
138
+ "SiteOrUserForwardModelStep",
130
139
  "SummaryConfig",
131
140
  "SurfaceConfig",
141
+ "UserInstalledForwardModelStep",
132
142
  "WarningInfo",
133
143
  "Workflow",
134
144
  "WorkflowConfigs",
@@ -34,6 +34,7 @@ if TYPE_CHECKING:
34
34
 
35
35
 
36
36
  DEFAULT_TIME_DELTA = timedelta(seconds=30)
37
+ DEFAULT_LOCATION_RANGE_M = 3000
37
38
 
38
39
 
39
40
  def create_observation_dataframes(
@@ -286,6 +287,43 @@ def _get_restart(
286
287
  ) from err
287
288
 
288
289
 
290
+ def _has_localization(summary_dict: SummaryObservation) -> bool:
291
+ return any(
292
+ [
293
+ summary_dict.location_x is not None,
294
+ summary_dict.location_y is not None,
295
+ summary_dict.location_range is not None,
296
+ ]
297
+ )
298
+
299
+
300
+ def _validate_localization_values(summary_dict: SummaryObservation) -> None:
301
+ """The user must provide LOCATION_X and LOCATION_Y to use localization, while
302
+ unprovided LOCATION_RANGE should default to some value.
303
+
304
+ This method assumes the summary dict contains at least one LOCATION key.
305
+ """
306
+ if summary_dict.location_x is None or summary_dict.location_y is None:
307
+ loc_values = {
308
+ "LOCATION_X": summary_dict.location_x,
309
+ "LOCATION_Y": summary_dict.location_y,
310
+ "LOCATION_RANGE": summary_dict.location_range,
311
+ }
312
+ provided_loc_values = {k: v for k, v in loc_values.items() if v is not None}
313
+
314
+ provided_loc_values_string = ", ".join(
315
+ key.upper() for key in provided_loc_values
316
+ )
317
+ raise ObservationConfigError.with_context(
318
+ f"Localization for observation {summary_dict.name} is misconfigured.\n"
319
+ f"Only {provided_loc_values_string} were provided. To enable "
320
+ f"localization for an observation, ensure that both LOCATION_X and "
321
+ f"LOCATION_Y are defined - or remove LOCATION keywords to disable "
322
+ f"localization.",
323
+ summary_dict,
324
+ )
325
+
326
+
289
327
  def _handle_summary_observation(
290
328
  summary_dict: SummaryObservation,
291
329
  obs_key: str,
@@ -323,15 +361,23 @@ def _handle_summary_observation(
323
361
  "Observation uncertainty must be strictly > 0", summary_key
324
362
  ) from None
325
363
 
326
- return pl.DataFrame(
327
- {
328
- "response_key": [summary_key],
329
- "observation_key": [obs_key],
330
- "time": pl.Series([date]).dt.cast_time_unit("ms"),
331
- "observations": pl.Series([value], dtype=pl.Float32),
332
- "std": pl.Series([std_dev], dtype=pl.Float32),
333
- }
334
- )
364
+ data_dict = {
365
+ "response_key": [summary_key],
366
+ "observation_key": [obs_key],
367
+ "time": pl.Series([date]).dt.cast_time_unit("ms"),
368
+ "observations": pl.Series([value], dtype=pl.Float32),
369
+ "std": pl.Series([std_dev], dtype=pl.Float32),
370
+ }
371
+
372
+ if _has_localization(summary_dict):
373
+ _validate_localization_values(summary_dict)
374
+ data_dict["location_x"] = summary_dict.location_x
375
+ data_dict["location_y"] = summary_dict.location_y
376
+ data_dict["location_range"] = (
377
+ summary_dict.location_range or DEFAULT_LOCATION_RANGE_M
378
+ )
379
+
380
+ return pl.DataFrame(data_dict)
335
381
 
336
382
 
337
383
  def _handle_general_observation(
@@ -439,9 +485,11 @@ def _handle_general_observation(
439
485
  raise ObservationConfigError.with_context(
440
486
  f"Values ({values}), error ({stds}) and "
441
487
  f"index list ({indices}) must be of equal length",
442
- general_observation.obs_file
443
- if general_observation.obs_file is not None
444
- else "",
488
+ (
489
+ general_observation.obs_file
490
+ if general_observation.obs_file is not None
491
+ else ""
492
+ ),
445
493
  )
446
494
 
447
495
  if np.any(stds <= 0):
@@ -90,10 +90,13 @@ class _SummaryValues:
90
90
  name: str
91
91
  value: float
92
92
  key: str #: The :term:`summary key` in the summary response
93
+ location_x: float | None = None
94
+ location_y: float | None = None
95
+ location_range: float | None = None
93
96
 
94
97
 
95
98
  @dataclass
96
- class SummaryObservation(ObservationDate, ObservationError, _SummaryValues):
99
+ class SummaryObservation(ObservationDate, _SummaryValues, ObservationError):
97
100
  @classmethod
98
101
  def from_obs_dict(cls, directory: str, observation_dict: ObservationDict) -> Self:
99
102
  error_mode = ErrorModes.ABS
@@ -101,6 +104,7 @@ class SummaryObservation(ObservationDate, ObservationError, _SummaryValues):
101
104
 
102
105
  date_dict: ObservationDate = ObservationDate()
103
106
  float_values: dict[str, float] = {"ERROR_MIN": 0.1}
107
+ localization_values: dict[str, float] = {}
104
108
  for key, value in observation_dict.items():
105
109
  match key:
106
110
  case "type" | "name":
@@ -121,6 +125,12 @@ class SummaryObservation(ObservationDate, ObservationError, _SummaryValues):
121
125
  summary_key = value
122
126
  case "DATE":
123
127
  date_dict.date = value
128
+ case "LOCATION_X":
129
+ localization_values["x"] = validate_float(value, key)
130
+ case "LOCATION_Y":
131
+ localization_values["y"] = validate_float(value, key)
132
+ case "LOCATION_RANGE":
133
+ localization_values["range"] = validate_float(value, key)
124
134
  case _:
125
135
  raise _unknown_key_error(str(key), observation_dict["name"])
126
136
  if "VALUE" not in float_values:
@@ -137,6 +147,9 @@ class SummaryObservation(ObservationDate, ObservationError, _SummaryValues):
137
147
  error_min=float_values["ERROR_MIN"],
138
148
  key=summary_key,
139
149
  value=float_values["VALUE"],
150
+ location_x=localization_values.get("x"),
151
+ location_y=localization_values.get("y"),
152
+ location_range=localization_values.get("range"),
140
153
  **date_dict.__dict__,
141
154
  )
142
155
 
@@ -12,6 +12,7 @@ import warnings
12
12
  from collections.abc import Callable, Sequence
13
13
  from datetime import datetime, timedelta
14
14
  from enum import Enum, auto
15
+ from functools import lru_cache
15
16
 
16
17
  import numpy as np
17
18
  import numpy.typing as npt
@@ -83,29 +84,30 @@ class DateUnit(Enum):
83
84
  raise InvalidResponseFile(f"Unknown date unit {val}")
84
85
 
85
86
 
87
+ @lru_cache
86
88
  def _fetch_keys_to_matcher(fetch_keys: Sequence[str]) -> Callable[[str], bool]:
87
89
  """
88
90
  Transform the list of keys (with * used as repeated wildcard) into
89
91
  a matcher.
90
92
 
91
- >>> match = _fetch_keys_to_matcher([""])
93
+ >>> match = _fetch_keys_to_matcher(("",))
92
94
  >>> match("FOPR")
93
95
  False
94
96
 
95
- >>> match = _fetch_keys_to_matcher(["*"])
97
+ >>> match = _fetch_keys_to_matcher(("*",))
96
98
  >>> match("FOPR"), match("FO*")
97
99
  (True, True)
98
100
 
99
101
 
100
- >>> match = _fetch_keys_to_matcher(["F*PR"])
102
+ >>> match = _fetch_keys_to_matcher(("F*PR",))
101
103
  >>> match("WOPR"), match("FOPR"), match("FGPR"), match("SOIL")
102
104
  (False, True, True, False)
103
105
 
104
- >>> match = _fetch_keys_to_matcher(["WGOR:*"])
106
+ >>> match = _fetch_keys_to_matcher(("WGOR:*",))
105
107
  >>> match("FOPR"), match("WGOR:OP1"), match("WGOR:OP2"), match("WGOR")
106
108
  (False, True, True, False)
107
109
 
108
- >>> match = _fetch_keys_to_matcher(["FOPR", "FGPR"])
110
+ >>> match = _fetch_keys_to_matcher(("FOPR", "FGPR"))
109
111
  >>> match("FOPR"), match("FGPR"), match("WGOR:OP2"), match("WGOR")
110
112
  (True, True, False, False)
111
113
  """
@@ -138,7 +140,7 @@ def _read_spec(
138
140
  date_index = None
139
141
  date_unit_str = None
140
142
 
141
- should_load_key = _fetch_keys_to_matcher(fetch_keys)
143
+ should_load_key = _fetch_keys_to_matcher(tuple(fetch_keys))
142
144
 
143
145
  for i, kw in enumerate(keywords):
144
146
  try:
@@ -9,26 +9,18 @@ from pydantic import BaseModel, Field, model_validator
9
9
 
10
10
  from .ext_param_config import ExtParamConfig
11
11
  from .field import Field as FieldConfig
12
- from .gen_data_config import GenDataConfig
13
12
  from .gen_kw_config import GenKwConfig
13
+ from .known_response_types import KNOWN_ERT_RESPONSE_TYPES, KnownErtResponseTypes
14
14
  from .parameter_config import ParameterConfig
15
15
  from .parsing import ConfigDict, ConfigKeys, ConfigValidationError
16
16
  from .response_config import ResponseConfig
17
- from .summary_config import SummaryConfig
18
17
  from .surface_config import SurfaceConfig
19
18
 
20
- KnownResponseTypes = SummaryConfig | GenDataConfig
21
-
22
- _KNOWN_RESPONSE_TYPES = (
23
- SummaryConfig,
24
- GenDataConfig,
25
- )
26
-
27
19
  logger = logging.getLogger(__name__)
28
20
 
29
21
 
30
22
  class EnsembleConfig(BaseModel):
31
- response_configs: dict[str, KnownResponseTypes] = Field(default_factory=dict)
23
+ response_configs: dict[str, KnownErtResponseTypes] = Field(default_factory=dict)
32
24
  parameter_configs: dict[
33
25
  str, GenKwConfig | FieldConfig | SurfaceConfig | ExtParamConfig
34
26
  ] = Field(default_factory=dict)
@@ -131,9 +123,9 @@ class EnsembleConfig(BaseModel):
131
123
  + [make_field(f) for f in field_list]
132
124
  )
133
125
  EnsembleConfig._check_for_duplicate_gen_kw_param_names(gen_kw_cfgs)
134
- response_configs: list[KnownResponseTypes] = []
126
+ response_configs: list[KnownErtResponseTypes] = []
135
127
 
136
- for config_cls in _KNOWN_RESPONSE_TYPES:
128
+ for config_cls in KNOWN_ERT_RESPONSE_TYPES:
137
129
  instance = config_cls.from_config_dict(config_dict)
138
130
 
139
131
  if instance is not None and instance.keys:
@@ -151,13 +143,13 @@ class EnsembleConfig(BaseModel):
151
143
  return self.parameter_configs[key]
152
144
  elif key in self.response_configs:
153
145
  return self.response_configs[key]
154
- elif _config := next(
146
+ elif config := next(
155
147
  (c for c in self.response_configs.values() if key in c.keys), None
156
148
  ):
157
149
  # Only hit by blockfs migration
158
150
  # returns the same config for one call per
159
151
  # response type. Is later deduped before saving to json
160
- return _config
152
+ return config
161
153
  else:
162
154
  raise KeyError(f"The key:{key} is not in the ensemble configuration")
163
155
 
ert/config/ert_config.py CHANGED
@@ -36,6 +36,9 @@ from .forward_model_step import (
36
36
  ForwardModelStepJSON,
37
37
  ForwardModelStepValidationError,
38
38
  ForwardModelStepWarning,
39
+ SiteInstalledForwardModelStep,
40
+ SiteOrUserForwardModelStep,
41
+ UserInstalledForwardModelStep,
39
42
  )
40
43
  from .gen_data_config import GenDataConfig
41
44
  from .gen_kw_config import DataSource, GenKwConfig
@@ -133,6 +136,7 @@ def create_forward_model_json(
133
136
  env_pr_fm_step = {}
134
137
 
135
138
  context_substitutions = Substitutions(context)
139
+ real_iter_substituter = context_substitutions.real_iter_substituter(iens, itr)
136
140
 
137
141
  class Substituter:
138
142
  def __init__(self, fm_step: ForwardModelStep) -> None:
@@ -146,7 +150,7 @@ def create_forward_model_json(
146
150
  )
147
151
  self.copy_private_args = Substitutions(
148
152
  {
149
- key: context_substitutions.substitute_real_iter(val, iens, itr)
153
+ key: real_iter_substituter.substitute(val)
150
154
  for key, val in fm_step.private_args.items()
151
155
  }
152
156
  )
@@ -163,7 +167,7 @@ def create_forward_model_json(
163
167
  string = self.copy_private_args.substitute(
164
168
  string, self.substitution_context_hint, 1, warn_max_iter=False
165
169
  )
166
- return context_substitutions.substitute_real_iter(string, iens, itr)
170
+ return real_iter_substituter.substitute(string)
167
171
 
168
172
  def filter_env_dict(self, env_dict: dict[str, str]) -> dict[str, str] | None:
169
173
  substituted_dict = {}
@@ -518,18 +522,16 @@ def workflows_from_dict(
518
522
 
519
523
  def installed_forward_model_steps_from_dict(
520
524
  config_dict: ConfigDict,
521
- ) -> dict[str, ForwardModelStep]:
525
+ ) -> dict[str, UserInstalledForwardModelStep]:
522
526
  errors: list[ErrorInfo | ConfigValidationError] = []
523
- fm_steps: dict[str, ForwardModelStep] = {}
527
+ fm_steps: dict[str, UserInstalledForwardModelStep] = {}
524
528
  for name, (fm_step_config_file, config_contents) in config_dict.get(
525
529
  ConfigKeys.INSTALL_JOB, []
526
530
  ):
527
531
  fm_step_config_file = path.abspath(fm_step_config_file)
528
532
  try:
529
533
  new_fm_step = forward_model_step_from_config_contents(
530
- config_contents,
531
- name=name,
532
- config_file=fm_step_config_file,
534
+ config_contents, name=name, config_file=fm_step_config_file
533
535
  )
534
536
  except ConfigValidationError as e:
535
537
  errors.append(e)
@@ -692,6 +694,8 @@ def log_observation_keys(
692
694
 
693
695
  RESERVED_KEYWORDS = ["realization", "IENS", "ITER"]
694
696
 
697
+ USER_CONFIG_SCHEMA = init_user_config_schema()
698
+
695
699
 
696
700
  class ErtConfig(BaseModel):
697
701
  DEFAULT_ENSPATH: ClassVar[str] = "storage"
@@ -720,7 +724,7 @@ class ErtConfig(BaseModel):
720
724
 
721
725
  ert_templates: list[tuple[str, str]] = Field(default_factory=list)
722
726
 
723
- forward_model_steps: list[ForwardModelStep] = Field(default_factory=list)
727
+ forward_model_steps: list[SiteOrUserForwardModelStep] = Field(default_factory=list)
724
728
  runpath_config: ModelConfig = Field(default_factory=ModelConfig)
725
729
  user_config_file: str = "no_config"
726
730
  config_path: str = Field(init=False, default="")
@@ -826,7 +830,7 @@ class ErtConfig(BaseModel):
826
830
  def with_plugins(runtime_plugins: ErtRuntimePlugins) -> type[ErtConfig]:
827
831
  class ErtConfigWithPlugins(ErtConfig):
828
832
  PREINSTALLED_FORWARD_MODEL_STEPS: ClassVar[
829
- Mapping[str, ForwardModelStep]
833
+ Mapping[str, SiteInstalledForwardModelStep]
830
834
  ] = runtime_plugins.installed_forward_model_steps
831
835
  PREINSTALLED_WORKFLOWS = dict(runtime_plugins.installed_workflow_jobs)
832
836
  ENV_PR_FM_STEP: ClassVar[dict[str, dict[str, Any]]] = (
@@ -1303,7 +1307,7 @@ class ErtConfig(BaseModel):
1303
1307
  @classmethod
1304
1308
  def _read_user_config_contents(cls, user_config: str, file_name: str) -> ConfigDict:
1305
1309
  return parse_contents(
1306
- user_config, file_name=file_name, schema=init_user_config_schema()
1310
+ user_config, file_name=file_name, schema=USER_CONFIG_SCHEMA
1307
1311
  )
1308
1312
 
1309
1313
  @classmethod
@@ -1393,8 +1397,10 @@ def uppercase_subkeys_and_stringify_subvalues(
1393
1397
 
1394
1398
 
1395
1399
  def forward_model_step_from_config_contents(
1396
- config_contents: str, config_file: str, name: str | None = None
1397
- ) -> ForwardModelStep:
1400
+ config_contents: str,
1401
+ config_file: str,
1402
+ name: str | None = None,
1403
+ ) -> UserInstalledForwardModelStep:
1398
1404
  if name is None:
1399
1405
  name = os.path.basename(config_file)
1400
1406
 
@@ -1418,7 +1424,7 @@ def forward_model_step_from_config_contents(
1418
1424
  environment = {k: v for [k, v] in content_dict.get("ENV", [])}
1419
1425
  default_mapping = {k: v for [k, v] in content_dict.get("DEFAULT", [])}
1420
1426
 
1421
- return ForwardModelStep(
1427
+ return UserInstalledForwardModelStep(
1422
1428
  name=name,
1423
1429
  executable=content_dict["EXECUTABLE"],
1424
1430
  stdin_file=content_dict.get("STDIN"),
@@ -11,13 +11,15 @@ from .response_config import InvalidResponseFile, ResponseConfig, ResponseMetada
11
11
  from .responses_index import responses_index
12
12
 
13
13
 
14
- class EverestObjectivesConfig(ResponseConfig):
15
- type: Literal["everest_objectives"] = "everest_objectives"
16
- name: str = "everest_objectives"
14
+ class EverestResponse(ResponseConfig):
15
+ """Base class for Everest response configurations."""
16
+
17
17
  has_finalized_keys: bool = True
18
- weights: list[float | None]
19
18
  scales: list[float | None]
20
- objective_types: list[Literal["mean", "stddev"]]
19
+
20
+ @property
21
+ def primary_key(self) -> list[str]:
22
+ return []
21
23
 
22
24
  @property
23
25
  def metadata(self) -> list[ResponseMetadata]:
@@ -52,7 +54,6 @@ class EverestObjectivesConfig(ResponseConfig):
52
54
  )
53
55
 
54
56
  errors = []
55
-
56
57
  run_path_ = Path(run_path)
57
58
  datasets_per_name = []
58
59
 
@@ -86,13 +87,23 @@ class EverestObjectivesConfig(ResponseConfig):
86
87
  combined = pl.concat(datasets_per_name)
87
88
  return combined
88
89
 
89
- @property
90
- def response_type(self) -> str:
91
- return "everest_objectives"
92
90
 
93
- @property
94
- def primary_key(self) -> list[str]:
95
- return []
91
+ class EverestConstraintsConfig(EverestResponse):
92
+ type: Literal["everest_constraints"] = "everest_constraints"
93
+ name: str = "everest_constraints"
94
+ targets: list[float | None]
95
+ upper_bounds: list[float]
96
+ lower_bounds: list[float]
97
+
98
+
99
+ responses_index.add_response_type(EverestConstraintsConfig)
100
+
101
+
102
+ class EverestObjectivesConfig(EverestResponse):
103
+ type: Literal["everest_objectives"] = "everest_objectives"
104
+ name: str = "everest_objectives"
105
+ weights: list[float | None]
106
+ objective_types: list[Literal["mean", "stddev"]]
96
107
 
97
108
 
98
109
  responses_index.add_response_type(EverestObjectivesConfig)