ert 19.0.0rc4__py3-none-any.whl → 20.0.0b0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. ert/__main__.py +94 -63
  2. ert/analysis/_es_update.py +11 -14
  3. ert/config/__init__.py +3 -2
  4. ert/config/_create_observation_dataframes.py +51 -375
  5. ert/config/_observations.py +483 -200
  6. ert/config/_read_summary.py +4 -5
  7. ert/config/ert_config.py +53 -80
  8. ert/config/everest_control.py +40 -39
  9. ert/config/everest_response.py +1 -13
  10. ert/config/field.py +0 -72
  11. ert/config/forward_model_step.py +17 -1
  12. ert/config/gen_data_config.py +14 -17
  13. ert/config/observation_config_migrations.py +821 -0
  14. ert/config/parameter_config.py +18 -28
  15. ert/config/parsing/__init__.py +0 -1
  16. ert/config/parsing/_parse_zonemap.py +45 -0
  17. ert/config/parsing/config_keywords.py +1 -1
  18. ert/config/parsing/config_schema.py +2 -8
  19. ert/config/parsing/observations_parser.py +2 -0
  20. ert/config/response_config.py +5 -23
  21. ert/config/rft_config.py +44 -19
  22. ert/config/summary_config.py +1 -13
  23. ert/config/surface_config.py +0 -57
  24. ert/dark_storage/compute/misfits.py +0 -42
  25. ert/dark_storage/endpoints/__init__.py +0 -2
  26. ert/dark_storage/endpoints/experiments.py +2 -5
  27. ert/dark_storage/json_schema/experiment.py +1 -2
  28. ert/field_utils/__init__.py +0 -2
  29. ert/field_utils/field_utils.py +1 -117
  30. ert/gui/ertwidgets/listeditbox.py +9 -1
  31. ert/gui/ertwidgets/models/ertsummary.py +20 -6
  32. ert/gui/ertwidgets/pathchooser.py +9 -1
  33. ert/gui/ertwidgets/stringbox.py +11 -3
  34. ert/gui/ertwidgets/textbox.py +10 -3
  35. ert/gui/ertwidgets/validationsupport.py +19 -1
  36. ert/gui/main_window.py +11 -6
  37. ert/gui/simulation/experiment_panel.py +1 -1
  38. ert/gui/simulation/run_dialog.py +11 -1
  39. ert/gui/tools/manage_experiments/export_dialog.py +4 -0
  40. ert/gui/tools/manage_experiments/manage_experiments_panel.py +1 -0
  41. ert/gui/tools/manage_experiments/storage_info_widget.py +5 -2
  42. ert/gui/tools/manage_experiments/storage_widget.py +18 -3
  43. ert/gui/tools/plot/data_type_proxy_model.py +1 -1
  44. ert/gui/tools/plot/plot_api.py +35 -27
  45. ert/gui/tools/plot/plot_widget.py +5 -0
  46. ert/gui/tools/plot/plot_window.py +4 -7
  47. ert/run_models/ensemble_experiment.py +1 -3
  48. ert/run_models/ensemble_smoother.py +1 -3
  49. ert/run_models/everest_run_model.py +12 -13
  50. ert/run_models/initial_ensemble_run_model.py +19 -22
  51. ert/run_models/model_factory.py +7 -7
  52. ert/run_models/multiple_data_assimilation.py +1 -3
  53. ert/sample_prior.py +12 -14
  54. ert/services/__init__.py +7 -3
  55. ert/services/_storage_main.py +59 -22
  56. ert/services/ert_server.py +186 -24
  57. ert/shared/version.py +3 -3
  58. ert/storage/local_ensemble.py +46 -115
  59. ert/storage/local_experiment.py +0 -16
  60. ert/utils/__init__.py +20 -0
  61. ert/warnings/specific_warning_handler.py +3 -2
  62. {ert-19.0.0rc4.dist-info → ert-20.0.0b0.dist-info}/METADATA +4 -51
  63. {ert-19.0.0rc4.dist-info → ert-20.0.0b0.dist-info}/RECORD +75 -80
  64. everest/bin/everest_script.py +5 -5
  65. everest/bin/kill_script.py +2 -2
  66. everest/bin/monitor_script.py +2 -2
  67. everest/bin/utils.py +4 -4
  68. everest/detached/everserver.py +6 -6
  69. everest/gui/everest_client.py +0 -6
  70. everest/gui/main_window.py +2 -2
  71. everest/util/__init__.py +1 -19
  72. ert/dark_storage/compute/__init__.py +0 -0
  73. ert/dark_storage/endpoints/compute/__init__.py +0 -0
  74. ert/dark_storage/endpoints/compute/misfits.py +0 -95
  75. ert/services/_base_service.py +0 -387
  76. ert/services/webviz_ert_service.py +0 -20
  77. ert/shared/storage/command.py +0 -38
  78. ert/shared/storage/extraction.py +0 -42
  79. {ert-19.0.0rc4.dist-info → ert-20.0.0b0.dist-info}/WHEEL +0 -0
  80. {ert-19.0.0rc4.dist-info → ert-20.0.0b0.dist-info}/entry_points.txt +0 -0
  81. {ert-19.0.0rc4.dist-info → ert-20.0.0b0.dist-info}/licenses/COPYING +0 -0
  82. {ert-19.0.0rc4.dist-info → ert-20.0.0b0.dist-info}/top_level.txt +0 -0
@@ -1,17 +1,27 @@
1
+ from __future__ import annotations
2
+
3
+ import logging
1
4
  import os
2
5
  from collections.abc import Sequence
3
- from dataclasses import dataclass
6
+ from datetime import datetime
4
7
  from enum import StrEnum
5
- from itertools import starmap
6
- from typing import Any, Self
8
+ from typing import Annotated, Any, Literal, Self, assert_never
9
+
10
+ import numpy as np
11
+ import pandas as pd
12
+ from pydantic import BaseModel, Field
7
13
 
14
+ from ..validation import rangestring_to_list
8
15
  from .parsing import (
16
+ ConfigWarning,
9
17
  ErrorInfo,
10
18
  ObservationConfigError,
11
19
  ObservationDict,
12
20
  ObservationType,
13
21
  )
14
22
 
23
+ logger = logging.getLogger(__name__)
24
+
15
25
 
16
26
  class ErrorModes(StrEnum):
17
27
  REL = "REL"
@@ -19,102 +29,67 @@ class ErrorModes(StrEnum):
19
29
  RELMIN = "RELMIN"
20
30
 
21
31
 
22
- @dataclass
23
- class ObservationError:
24
- error_mode: ErrorModes
25
- error: float
26
- error_min: float
27
-
28
-
29
- @dataclass
30
- class Segment(ObservationError):
31
- name: str
32
- start: int
33
- stop: int
34
-
35
-
36
- @dataclass
37
- class HistoryObservation(ObservationError):
38
- name: str
39
- segments: list[Segment]
40
-
41
- @property
42
- def key(self) -> str:
43
- """The :term:`summary key` to be fetched from :ref:`refcase`."""
44
- # For history observations the key is also the name, ie.
45
- # "HISTORY_OBSERVATION FOPR" means to add the values from
46
- # the summary vector FOPRH in refcase as observations.
47
- return self.name
48
-
49
- @classmethod
50
- def from_obs_dict(cls, directory: str, observation_dict: ObservationDict) -> Self:
51
- error_mode = ErrorModes.RELMIN
52
- error = 0.1
53
- error_min = 0.1
54
- segments = []
55
- for key, value in observation_dict.items():
56
- match key:
57
- case "type" | "name":
58
- pass
59
- case "ERROR":
60
- error = validate_positive_float(value, key)
61
- case "ERROR_MIN":
62
- error_min = validate_positive_float(value, key)
63
- case "ERROR_MODE":
64
- error_mode = validate_error_mode(value)
65
- case "segments":
66
- segments = list(starmap(_validate_segment_dict, value))
67
- case _:
68
- raise _unknown_key_error(str(key), observation_dict["name"])
69
-
70
- return cls(
71
- name=observation_dict["name"],
72
- error_mode=error_mode,
73
- error=error,
74
- error_min=error_min,
75
- segments=segments,
76
- )
77
-
78
-
79
- @dataclass
80
- class ObservationDate:
81
- days: float | None = None
82
- hours: float | None = None
83
- date: str | None = None
84
- restart: int | None = None
85
-
86
-
87
- @dataclass
88
- class _SummaryValues:
32
+ class _SummaryValues(BaseModel):
33
+ type: Literal["summary_observation"] = "summary_observation"
89
34
  name: str
90
35
  value: float
36
+ error: float
91
37
  key: str #: The :term:`summary key` in the summary response
38
+ date: str
92
39
  east: float | None = None
93
40
  north: float | None = None
94
41
  radius: float | None = None
95
42
 
96
43
 
97
- @dataclass
98
- class SummaryObservation(ObservationDate, _SummaryValues, ObservationError):
44
+ def _parse_date(date_str: str) -> datetime:
45
+ try:
46
+ return datetime.fromisoformat(date_str)
47
+ except ValueError:
48
+ try:
49
+ date = datetime.strptime(date_str, "%d/%m/%Y")
50
+ except ValueError as err:
51
+ raise ObservationConfigError.with_context(
52
+ f"Unsupported date format {date_str}. Please use ISO date format",
53
+ date_str,
54
+ ) from err
55
+ else:
56
+ ConfigWarning.warn(
57
+ f"Deprecated time format {date_str}."
58
+ " Please use ISO date format YYYY-MM-DD",
59
+ date_str,
60
+ )
61
+ return date
62
+
63
+
64
+ class SummaryObservation(_SummaryValues):
99
65
  @classmethod
100
- def from_obs_dict(cls, directory: str, observation_dict: ObservationDict) -> Self:
66
+ def from_obs_dict(
67
+ cls, directory: str, observation_dict: ObservationDict
68
+ ) -> list[Self]:
101
69
  error_mode = ErrorModes.ABS
102
70
  summary_key = None
103
71
 
104
- date_dict: ObservationDate = ObservationDate()
72
+ date: str | None = None
105
73
  float_values: dict[str, float] = {"ERROR_MIN": 0.1}
106
- localization_values: dict[str, float | None] = {}
74
+ localization_dict: dict[LOCALIZATION_KEYS, float | None] = {}
107
75
  for key, value in observation_dict.items():
108
76
  match key:
109
77
  case "type" | "name":
110
78
  pass
111
- case "RESTART":
112
- date_dict.restart = validate_positive_int(value, key)
113
79
  case "ERROR" | "ERROR_MIN":
114
- float_values[str(key)] = validate_positive_float(value, key)
115
- case "DAYS" | "HOURS":
116
- setattr(
117
- date_dict, str(key).lower(), validate_positive_float(value, key)
80
+ float_values[str(key)] = validate_positive_float(
81
+ value, key, strictly_positive=True
82
+ )
83
+ case "DAYS" | "HOURS" | "RESTART":
84
+ raise ObservationConfigError.with_context(
85
+ (
86
+ "SUMMARY_OBSERVATION must use DATE to specify "
87
+ "date, DAYS | HOURS is no longer allowed. "
88
+ "Please run:\n ert convert_observations "
89
+ "<your_ert_config.ert>\nto migrate the observation config "
90
+ "to use the correct format."
91
+ ),
92
+ key,
118
93
  )
119
94
  case "VALUE":
120
95
  float_values[str(key)] = validate_float(value, key)
@@ -123,16 +98,10 @@ class SummaryObservation(ObservationDate, _SummaryValues, ObservationError):
123
98
  case "KEY":
124
99
  summary_key = value
125
100
  case "DATE":
126
- date_dict.date = value
101
+ date = value
127
102
  case "LOCALIZATION":
128
103
  validate_localization(value, observation_dict["name"])
129
- localization_values["east"] = validate_float(value["EAST"], key)
130
- localization_values["north"] = validate_float(value["NORTH"], key)
131
- localization_values["radius"] = (
132
- validate_float(value["RADIUS"], key)
133
- if "RADIUS" in value
134
- else None
135
- )
104
+ extract_localization_values(localization_dict, value, key)
136
105
  case _:
137
106
  raise _unknown_key_error(str(key), observation_dict["name"])
138
107
  if "VALUE" not in float_values:
@@ -142,82 +111,225 @@ class SummaryObservation(ObservationDate, _SummaryValues, ObservationError):
142
111
  if "ERROR" not in float_values:
143
112
  raise _missing_value_error(observation_dict["name"], "ERROR")
144
113
 
145
- return cls(
114
+ assert date is not None
115
+ # Raise errors if the date is off
116
+ parsed_date: datetime = _parse_date(date)
117
+ standardized_date = parsed_date.date().isoformat()
118
+
119
+ value = float_values["VALUE"]
120
+ input_error = float_values["ERROR"]
121
+ error_min = float_values["ERROR_MIN"]
122
+
123
+ error = input_error
124
+ match error_mode:
125
+ case ErrorModes.ABS:
126
+ error = validate_positive_float(
127
+ np.abs(input_error), summary_key, strictly_positive=True
128
+ )
129
+ case ErrorModes.REL:
130
+ error = validate_positive_float(
131
+ np.abs(value) * input_error, summary_key, strictly_positive=True
132
+ )
133
+ case ErrorModes.RELMIN:
134
+ error = validate_positive_float(
135
+ np.maximum(np.abs(value) * input_error, error_min),
136
+ summary_key,
137
+ strictly_positive=True,
138
+ )
139
+ case default:
140
+ assert_never(default)
141
+
142
+ obs_instance = cls(
146
143
  name=observation_dict["name"],
147
- error_mode=error_mode,
148
- error=float_values["ERROR"],
149
- error_min=float_values["ERROR_MIN"],
144
+ error=error,
150
145
  key=summary_key,
151
- value=float_values["VALUE"],
152
- east=localization_values.get("east"),
153
- north=localization_values.get("north"),
154
- radius=localization_values.get("radius"),
155
- **date_dict.__dict__,
146
+ value=value,
147
+ east=localization_dict.get("east"),
148
+ north=localization_dict.get("north"),
149
+ radius=localization_dict.get("radius"),
150
+ date=standardized_date,
156
151
  )
152
+ # Bypass pydantic discarding context
153
+ # only relevant for ERT config surfacing validation errors
154
+ # irrelevant for runmodels etc.
155
+ obs_instance.name = observation_dict["name"]
156
+
157
+ return [obs_instance]
157
158
 
158
159
 
159
- @dataclass
160
- class _GeneralObservation:
160
+ class _GeneralObservation(BaseModel):
161
+ type: Literal["general_observation"] = "general_observation"
161
162
  name: str
162
163
  data: str
163
- value: float | None = None
164
- error: float | None = None
165
- index_list: str | None = None
166
- index_file: str | None = None
167
- obs_file: str | None = None
164
+ value: float
165
+ error: float
166
+ restart: int
167
+ index: int
168
+ east: float | None = None
169
+ north: float | None = None
170
+ radius: float | None = None
168
171
 
169
172
 
170
- @dataclass
171
- class GeneralObservation(ObservationDate, _GeneralObservation):
173
+ class GeneralObservation(_GeneralObservation):
172
174
  @classmethod
173
- def from_obs_dict(cls, directory: str, observation_dict: ObservationDict) -> Self:
175
+ def from_obs_dict(
176
+ cls, directory: str, observation_dict: ObservationDict
177
+ ) -> list[Self]:
174
178
  try:
175
179
  data = observation_dict["DATA"]
176
180
  except KeyError as err:
177
181
  raise _missing_value_error(observation_dict["name"], "DATA") from err
178
182
 
179
- output = cls(name=observation_dict["name"], data=data)
180
- for key, value in observation_dict.items():
181
- match key:
182
- case "type" | "name":
183
- pass
184
- case "RESTART":
185
- output.restart = validate_positive_int(value, key)
186
- case "VALUE":
187
- output.value = validate_float(value, key)
188
- case "ERROR" | "DAYS" | "HOURS":
189
- setattr(
190
- output, str(key).lower(), validate_positive_float(value, key)
191
- )
192
- case "DATE" | "INDEX_LIST":
193
- setattr(output, str(key).lower(), value)
194
- case "OBS_FILE" | "INDEX_FILE":
195
- assert not isinstance(key, tuple)
196
- filename = value
197
- if not os.path.isabs(filename):
198
- filename = os.path.join(directory, filename)
199
- if not os.path.exists(filename):
200
- raise ObservationConfigError.with_context(
201
- "The following keywords did not"
202
- f" resolve to a valid path:\n {key}",
203
- value,
204
- )
205
- setattr(output, str(key).lower(), filename)
206
- case "DATA":
207
- output.data = value
208
- case _:
209
- raise _unknown_key_error(str(key), observation_dict["name"])
210
- if output.value is not None and output.error is None:
183
+ allowed = {
184
+ "type",
185
+ "name",
186
+ "RESTART",
187
+ "VALUE",
188
+ "ERROR",
189
+ "DATE",
190
+ "DAYS",
191
+ "HOURS",
192
+ "INDEX_LIST",
193
+ "OBS_FILE",
194
+ "INDEX_FILE",
195
+ "DATA",
196
+ }
197
+
198
+ extra = set(observation_dict.keys()) - allowed
199
+ if extra:
200
+ raise _unknown_key_error(str(next(iter(extra))), observation_dict["name"])
201
+
202
+ if any(k in observation_dict for k in ("DATE", "DAYS", "HOURS")):
203
+ bad_key = next(
204
+ k for k in ("DATE", "DAYS", "HOURS") if k in observation_dict
205
+ )
206
+ raise ObservationConfigError.with_context(
207
+ (
208
+ "GENERAL_OBSERVATION must use RESTART to specify "
209
+ "report step. Please run:\n ert convert_observations "
210
+ "<your_ert_config.ert>\nto migrate the observation config "
211
+ "to use the correct format."
212
+ ),
213
+ bad_key,
214
+ )
215
+
216
+ if "OBS_FILE" in observation_dict and (
217
+ "VALUE" in observation_dict or "ERROR" in observation_dict
218
+ ):
219
+ raise ObservationConfigError.with_context(
220
+ "GENERAL_OBSERVATION cannot contain both VALUE/ERROR and OBS_FILE",
221
+ observation_dict["name"],
222
+ )
223
+
224
+ if "INDEX_FILE" in observation_dict and "INDEX_LIST" in observation_dict:
225
+ raise ObservationConfigError.with_context(
226
+ (
227
+ "GENERAL_OBSERVATION "
228
+ f"{observation_dict['name']} has both INDEX_FILE and INDEX_LIST."
229
+ ),
230
+ observation_dict["name"],
231
+ )
232
+
233
+ restart = (
234
+ validate_positive_int(observation_dict["RESTART"], "RESTART")
235
+ if "RESTART" in observation_dict
236
+ else 0
237
+ )
238
+
239
+ if "OBS_FILE" not in observation_dict:
240
+ if "VALUE" in observation_dict and "ERROR" not in observation_dict:
241
+ raise ObservationConfigError.with_context(
242
+ f"For GENERAL_OBSERVATION {observation_dict['name']}, with"
243
+ f" VALUE = {observation_dict['VALUE']}, ERROR must also be given.",
244
+ observation_dict["name"],
245
+ )
246
+
247
+ if "VALUE" not in observation_dict and "ERROR" not in observation_dict:
248
+ raise ObservationConfigError.with_context(
249
+ "GENERAL_OBSERVATION must contain either VALUE "
250
+ "and ERROR or OBS_FILE",
251
+ context=observation_dict["name"],
252
+ )
253
+
254
+ obs_instance = cls(
255
+ name=observation_dict["name"],
256
+ data=data,
257
+ value=validate_float(observation_dict["VALUE"], "VALUE"),
258
+ error=validate_positive_float(
259
+ observation_dict["ERROR"], "ERROR", strictly_positive=True
260
+ ),
261
+ restart=restart,
262
+ index=0,
263
+ )
264
+ # Bypass pydantic discarding context
265
+ # only relevant for ERT config surfacing validation errors
266
+ # irrelevant for runmodels etc.
267
+ obs_instance.name = observation_dict["name"]
268
+ return [obs_instance]
269
+
270
+ obs_filename = _resolve_path(directory, observation_dict["OBS_FILE"])
271
+ try:
272
+ file_values = np.loadtxt(obs_filename, delimiter=None).ravel()
273
+ except ValueError as err:
274
+ raise ObservationConfigError.with_context(
275
+ f"Failed to read OBS_FILE {obs_filename}: {err}", obs_filename
276
+ ) from err
277
+ if len(file_values) % 2 != 0:
278
+ raise ObservationConfigError.with_context(
279
+ "Expected even number of values in GENERAL_OBSERVATION",
280
+ obs_filename,
281
+ )
282
+ values = file_values[::2]
283
+ stds = file_values[1::2]
284
+
285
+ if "INDEX_FILE" in observation_dict:
286
+ idx_file = _resolve_path(directory, observation_dict["INDEX_FILE"])
287
+ indices = np.loadtxt(idx_file, delimiter=None, dtype=np.int32).ravel()
288
+ elif "INDEX_LIST" in observation_dict:
289
+ indices = np.array(
290
+ sorted(rangestring_to_list(observation_dict["INDEX_LIST"])),
291
+ dtype=np.int32,
292
+ )
293
+ else:
294
+ indices = np.arange(len(values), dtype=np.int32)
295
+
296
+ if len({len(stds), len(values), len(indices)}) != 1:
297
+ raise ObservationConfigError.with_context(
298
+ (
299
+ "Values ("
300
+ f"{values}), error ({stds}) and index list ({indices}) "
301
+ "must be of equal length"
302
+ ),
303
+ observation_dict["name"],
304
+ )
305
+
306
+ if np.any(stds <= 0):
211
307
  raise ObservationConfigError.with_context(
212
- f"For GENERAL_OBSERVATION {observation_dict['name']}, with"
213
- f" VALUE = {output.value}, ERROR must also be given.",
308
+ "Observation uncertainty must be strictly > 0",
214
309
  observation_dict["name"],
215
310
  )
216
- return output
217
311
 
312
+ obs_instances: list[Self] = []
313
+ for _pos, (val, std, idx) in enumerate(zip(values, stds, indices, strict=True)):
314
+ # index should reflect the index provided by INDEX_FILE / INDEX_LIST
315
+ inst = cls(
316
+ name=observation_dict["name"],
317
+ data=data,
318
+ value=float(val),
319
+ error=float(std),
320
+ restart=restart,
321
+ index=int(idx),
322
+ )
323
+ # Bypass pydantic discarding context
324
+ # only relevant for ERT config surfacing validation errors
325
+ # irrelevant for runmodels etc.
326
+ inst.name = observation_dict["name"]
327
+ obs_instances.append(inst)
328
+ return obs_instances
218
329
 
219
- @dataclass
220
- class RFTObservation:
330
+
331
+ class RFTObservation(BaseModel):
332
+ type: Literal["rft_observation"] = "rft_observation"
221
333
  name: str
222
334
  well: str
223
335
  date: str
@@ -227,9 +339,93 @@ class RFTObservation:
227
339
  north: float
228
340
  east: float
229
341
  tvd: float
342
+ zone: str | None = None
343
+
344
+ @classmethod
345
+ def from_csv(
346
+ cls,
347
+ directory: str,
348
+ observation_dict: ObservationDict,
349
+ filename: str,
350
+ observed_property: str = "PRESSURE",
351
+ ) -> list[Self]:
352
+ if not os.path.isabs(filename):
353
+ filename = os.path.join(directory, filename)
354
+ if not os.path.exists(filename):
355
+ raise ObservationConfigError.with_context(
356
+ f"The CSV file ({filename}) does not exist or is not accessible.",
357
+ filename,
358
+ )
359
+ csv_file = pd.read_csv(
360
+ filename,
361
+ encoding="utf-8",
362
+ on_bad_lines="error",
363
+ )
364
+
365
+ required_columns = {
366
+ "WELL_NAME",
367
+ "DATE",
368
+ observed_property,
369
+ "ERROR",
370
+ "NORTH",
371
+ "EAST",
372
+ "TVD",
373
+ }
374
+ missing_required_columns = required_columns - set(csv_file.keys())
375
+ if missing_required_columns:
376
+ raise ObservationConfigError.with_context(
377
+ f"The rft observations file {filename} is missing required column(s) "
378
+ f"{', '.join(sorted(missing_required_columns))}.",
379
+ filename,
380
+ )
381
+
382
+ rft_observations = []
383
+ invalid_observations = []
384
+ for row in csv_file.itertuples(index=True):
385
+ rft_observation = cls(
386
+ name=f"{observation_dict['name']}[{row.Index}]",
387
+ well=str(row.WELL_NAME),
388
+ date=str(row.DATE),
389
+ property=observed_property,
390
+ value=validate_float(
391
+ str(getattr(row, observed_property)), observed_property
392
+ ),
393
+ error=validate_float(str(row.ERROR), "ERROR"),
394
+ north=validate_float(str(row.NORTH), "NORTH"),
395
+ east=validate_float(str(row.EAST), "EAST"),
396
+ tvd=validate_float(str(row.TVD), "TVD"),
397
+ zone=row.ZONE if "ZONE" in csv_file else None,
398
+ )
399
+ # A value of -1 and error of 0 is used by fmu.tools.rms create_rft_ertobs to
400
+ # indicate missing data. If encountered in an rft observations csv file
401
+ # it should raise an error and ask the user to remove invalid observations.
402
+ if rft_observation.value == -1 and rft_observation.error == 0:
403
+ invalid_observations.append(rft_observation)
404
+ else:
405
+ rft_observations.append(rft_observation)
406
+
407
+ if invalid_observations:
408
+ well_list = "\n - ".join(
409
+ [
410
+ f"{observation.well} at date {observation.date}"
411
+ for observation in invalid_observations
412
+ ]
413
+ )
414
+ raise ObservationConfigError.with_context(
415
+ (
416
+ f"Invalid value=-1 and error=0 detected in {filename} for "
417
+ f"well(s):\n - {well_list}\n"
418
+ "The invalid observation(s) must be removed from the file."
419
+ ),
420
+ filename,
421
+ )
422
+ return rft_observations
230
423
 
231
424
  @classmethod
232
- def from_obs_dict(cls, directory: str, observation_dict: ObservationDict) -> Self:
425
+ def from_obs_dict(
426
+ cls, directory: str, observation_dict: ObservationDict
427
+ ) -> list[Self]:
428
+ csv_filename = None
233
429
  well = None
234
430
  observed_property = None
235
431
  observed_value = None
@@ -238,6 +434,7 @@ class RFTObservation:
238
434
  north = None
239
435
  east = None
240
436
  tvd = None
437
+ zone = None
241
438
  for key, value in observation_dict.items():
242
439
  match key:
243
440
  case "type" | "name":
@@ -258,8 +455,19 @@ class RFTObservation:
258
455
  east = validate_float(value, key)
259
456
  case "TVD":
260
457
  tvd = validate_float(value, key)
458
+ case "CSV":
459
+ csv_filename = value
460
+ case "ZONE":
461
+ zone = value
261
462
  case _:
262
463
  raise _unknown_key_error(str(key), observation_dict["name"])
464
+ if csv_filename is not None:
465
+ return cls.from_csv(
466
+ directory,
467
+ observation_dict,
468
+ csv_filename,
469
+ observed_property or "PRESSURE",
470
+ )
263
471
  if well is None:
264
472
  raise _missing_value_error(observation_dict["name"], "WELL")
265
473
  if observed_value is None:
@@ -276,30 +484,106 @@ class RFTObservation:
276
484
  raise _missing_value_error(observation_dict["name"], "EAST")
277
485
  if tvd is None:
278
486
  raise _missing_value_error(observation_dict["name"], "TVD")
279
- return cls(
280
- observation_dict["name"],
281
- well,
282
- date,
283
- observed_property,
284
- observed_value,
285
- error,
286
- north,
287
- east,
288
- tvd,
487
+
488
+ obs_instance = cls(
489
+ name=observation_dict["name"],
490
+ well=well,
491
+ property=observed_property,
492
+ value=observed_value,
493
+ error=error,
494
+ date=date,
495
+ north=north,
496
+ east=east,
497
+ tvd=tvd,
498
+ zone=zone,
289
499
  )
290
500
 
501
+ # Bypass pydantic discarding context
502
+ # only relevant for ERT config surfacing validation errors
503
+ # irrelevant for runmodels etc.
504
+ obs_instance.name = observation_dict["name"]
505
+
506
+ return [obs_instance]
291
507
 
292
- Observation = (
293
- HistoryObservation | SummaryObservation | GeneralObservation | RFTObservation
294
- )
508
+
509
+ class BreakthroughObservation(BaseModel):
510
+ type: Literal["breakthrough"] = "breakthrough"
511
+ name: str
512
+ response_key: str
513
+ date: datetime
514
+ error: float
515
+ threshold: float
516
+ north: float | None
517
+ east: float | None
518
+ radius: float | None
519
+
520
+ @classmethod
521
+ def from_obs_dict(cls, directory: str, obs_dict: ObservationDict) -> list[Self]:
522
+ response_key = None
523
+ date = None
524
+ error = None
525
+ threshold = None
526
+ localization_dict: dict[LOCALIZATION_KEYS, float | None] = {}
527
+ for key, value in obs_dict.items():
528
+ match key:
529
+ case "type" | "name":
530
+ pass
531
+ case "KEY":
532
+ response_key = value
533
+ case "DATE":
534
+ date = value
535
+ case "ERROR":
536
+ error = validate_float(value, key)
537
+ case "THRESHOLD":
538
+ threshold = validate_float(value, key)
539
+ case "LOCALIZATION":
540
+ validate_localization(value, obs_dict["name"])
541
+ extract_localization_values(localization_dict, value, key)
542
+ case _:
543
+ raise _unknown_key_error(str(key), value)
544
+
545
+ if response_key is None:
546
+ raise _missing_value_error(obs_dict["name"], "KEY")
547
+ if date is None:
548
+ raise _missing_value_error(obs_dict["name"], "DATE")
549
+ if error is None:
550
+ raise _missing_value_error(obs_dict["name"], "ERROR")
551
+ if threshold is None:
552
+ raise _missing_value_error(obs_dict["name"], "THRESHOLD")
553
+
554
+ return [
555
+ cls(
556
+ name=obs_dict["name"],
557
+ response_key=response_key,
558
+ date=date,
559
+ error=error,
560
+ threshold=threshold,
561
+ north=localization_dict.get("north"),
562
+ east=localization_dict.get("east"),
563
+ radius=localization_dict.get("radius"),
564
+ )
565
+ ]
566
+
567
+
568
+ Observation = Annotated[
569
+ (
570
+ SummaryObservation
571
+ | GeneralObservation
572
+ | RFTObservation
573
+ | BreakthroughObservation
574
+ ),
575
+ Field(discriminator="type"),
576
+ ]
295
577
 
296
578
  _TYPE_TO_CLASS: dict[ObservationType, type[Observation]] = {
297
- ObservationType.HISTORY: HistoryObservation,
298
579
  ObservationType.SUMMARY: SummaryObservation,
299
580
  ObservationType.GENERAL: GeneralObservation,
300
581
  ObservationType.RFT: RFTObservation,
582
+ ObservationType.BREAKTHROUGH: BreakthroughObservation,
301
583
  }
302
584
 
585
+ LOCALIZATION_KEYS = Literal["east", "north", "radius"]
586
+
303
587
 
304
588
  def make_observations(
305
589
  directory: str, observation_dicts: Sequence[ObservationDict]
@@ -312,10 +596,20 @@ def make_observations(
312
596
  inp: The collection of statements to validate.
313
597
  """
314
598
  result: list[Observation] = []
315
- error_list: list[ErrorInfo] = []
599
+ error_list: list[ErrorInfo | ObservationConfigError] = []
316
600
  for obs_dict in observation_dicts:
601
+ if obs_dict["type"] == ObservationType.HISTORY:
602
+ msg = (
603
+ "HISTORY_OBSERVATION is deprecated, and must be specified "
604
+ "as SUMMARY_OBSERVATION. Run"
605
+ " ert convert_observations <ert_config.ert> to convert your "
606
+ "observations automatically"
607
+ )
608
+ logger.error(msg)
609
+ error_list.append(ObservationConfigError(msg))
610
+ continue
317
611
  try:
318
- result.append(
612
+ result.extend(
319
613
  _TYPE_TO_CLASS[obs_dict["type"]].from_obs_dict(directory, obs_dict)
320
614
  )
321
615
  except KeyError as err:
@@ -329,41 +623,6 @@ def make_observations(
329
623
  return result
330
624
 
331
625
 
332
- def _validate_segment_dict(name_token: str, inp: dict[str, Any]) -> Segment:
333
- start = None
334
- stop = None
335
- error_mode = ErrorModes.RELMIN
336
- error = 0.1
337
- error_min = 0.1
338
- for key, value in inp.items():
339
- match key:
340
- case "START":
341
- start = validate_int(value, key)
342
- case "STOP":
343
- stop = validate_int(value, key)
344
- case "ERROR":
345
- error = validate_positive_float(value, key)
346
- case "ERROR_MIN":
347
- error_min = validate_positive_float(value, key)
348
- case "ERROR_MODE":
349
- error_mode = validate_error_mode(value)
350
- case _:
351
- raise _unknown_key_error(key, name_token)
352
-
353
- if start is None:
354
- raise _missing_value_error(name_token, "START")
355
- if stop is None:
356
- raise _missing_value_error(name_token, "STOP")
357
- return Segment(
358
- name=name_token,
359
- start=start,
360
- stop=stop,
361
- error_mode=error_mode,
362
- error=error,
363
- error_min=error_min,
364
- )
365
-
366
-
367
626
  def validate_error_mode(inp: str) -> ErrorModes:
368
627
  if inp == "REL":
369
628
  return ErrorModes.REL
@@ -390,12 +649,15 @@ def validate_int(val: str, key: str) -> int:
390
649
  raise _conversion_error(key, val, "int") from err
391
650
 
392
651
 
393
- def validate_positive_float(val: str, key: str) -> float:
652
+ def validate_positive_float(
653
+ val: str, key: str, strictly_positive: bool = False
654
+ ) -> float:
394
655
  v = validate_float(val, key)
395
- if v < 0:
656
+ if v < 0 or (v <= 0 and strictly_positive):
396
657
  raise ObservationConfigError.with_context(
397
658
  f'Failed to validate "{val}" in {key}={val}.'
398
- f" {key} must be given a positive value.",
659
+ f" {key} must be given a "
660
+ f"{'strictly ' if strictly_positive else ''}positive value.",
399
661
  val,
400
662
  )
401
663
  return v
@@ -414,6 +676,16 @@ def validate_localization(val: dict[str, Any], obs_name: str) -> None:
414
676
  raise ObservationConfigError.from_collected(errors)
415
677
 
416
678
 
679
+ def extract_localization_values(
680
+ localization_dict: dict[LOCALIZATION_KEYS, float | None], value: Any, key: str
681
+ ) -> None:
682
+ localization_dict["east"] = validate_float(value["EAST"], key)
683
+ localization_dict["north"] = validate_float(value["NORTH"], key)
684
+ localization_dict["radius"] = (
685
+ validate_float(value["RADIUS"], key) if "RADIUS" in value else None
686
+ )
687
+
688
+
417
689
  def validate_positive_int(val: str, key: str) -> int:
418
690
  try:
419
691
  v = int(val)
@@ -434,6 +706,17 @@ def _missing_value_error(name_token: str, value_key: str) -> ObservationConfigEr
434
706
  )
435
707
 
436
708
 
709
+ def _resolve_path(directory: str, filename: str) -> str:
710
+ if not os.path.isabs(filename):
711
+ filename = os.path.join(directory, filename)
712
+ if not os.path.exists(filename):
713
+ raise ObservationConfigError.with_context(
714
+ f"The following keywords did not resolve to a valid path:\n {filename}",
715
+ filename,
716
+ )
717
+ return filename
718
+
719
+
437
720
  def _conversion_error(token: str, value: Any, type_name: str) -> ObservationConfigError:
438
721
  return ObservationConfigError.with_context(
439
722
  f'Could not convert {value} to {type_name}. Failed to validate "{value}"',