ert 19.0.1__py3-none-any.whl → 20.0.0b1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. ert/__main__.py +94 -63
  2. ert/analysis/_es_update.py +11 -14
  3. ert/cli/main.py +1 -1
  4. ert/config/__init__.py +3 -2
  5. ert/config/_create_observation_dataframes.py +52 -375
  6. ert/config/_observations.py +527 -200
  7. ert/config/_read_summary.py +4 -5
  8. ert/config/ert_config.py +52 -117
  9. ert/config/everest_control.py +40 -39
  10. ert/config/everest_response.py +3 -15
  11. ert/config/field.py +4 -76
  12. ert/config/forward_model_step.py +17 -1
  13. ert/config/gen_data_config.py +14 -17
  14. ert/config/observation_config_migrations.py +821 -0
  15. ert/config/parameter_config.py +18 -28
  16. ert/config/parsing/__init__.py +0 -1
  17. ert/config/parsing/_parse_zonemap.py +45 -0
  18. ert/config/parsing/config_keywords.py +1 -0
  19. ert/config/parsing/config_schema.py +2 -0
  20. ert/config/parsing/observations_parser.py +2 -0
  21. ert/config/response_config.py +5 -23
  22. ert/config/rft_config.py +129 -31
  23. ert/config/summary_config.py +1 -13
  24. ert/config/surface_config.py +0 -57
  25. ert/dark_storage/compute/misfits.py +0 -42
  26. ert/dark_storage/endpoints/__init__.py +0 -2
  27. ert/dark_storage/endpoints/experiments.py +2 -5
  28. ert/dark_storage/json_schema/experiment.py +1 -2
  29. ert/field_utils/__init__.py +0 -2
  30. ert/field_utils/field_utils.py +1 -117
  31. ert/gui/ertwidgets/listeditbox.py +9 -1
  32. ert/gui/ertwidgets/models/ertsummary.py +20 -6
  33. ert/gui/ertwidgets/pathchooser.py +9 -1
  34. ert/gui/ertwidgets/stringbox.py +11 -3
  35. ert/gui/ertwidgets/textbox.py +10 -3
  36. ert/gui/ertwidgets/validationsupport.py +19 -1
  37. ert/gui/main_window.py +11 -6
  38. ert/gui/simulation/experiment_panel.py +1 -1
  39. ert/gui/simulation/run_dialog.py +11 -1
  40. ert/gui/tools/manage_experiments/export_dialog.py +4 -0
  41. ert/gui/tools/manage_experiments/manage_experiments_panel.py +1 -0
  42. ert/gui/tools/manage_experiments/storage_info_widget.py +1 -1
  43. ert/gui/tools/manage_experiments/storage_widget.py +21 -4
  44. ert/gui/tools/plot/data_type_proxy_model.py +1 -1
  45. ert/gui/tools/plot/plot_api.py +35 -27
  46. ert/gui/tools/plot/plot_widget.py +5 -0
  47. ert/gui/tools/plot/plot_window.py +4 -7
  48. ert/run_models/ensemble_experiment.py +2 -9
  49. ert/run_models/ensemble_smoother.py +1 -9
  50. ert/run_models/everest_run_model.py +31 -23
  51. ert/run_models/initial_ensemble_run_model.py +19 -22
  52. ert/run_models/manual_update.py +11 -5
  53. ert/run_models/model_factory.py +7 -7
  54. ert/run_models/multiple_data_assimilation.py +3 -16
  55. ert/sample_prior.py +12 -14
  56. ert/scheduler/job.py +24 -4
  57. ert/services/__init__.py +7 -3
  58. ert/services/_storage_main.py +59 -22
  59. ert/services/ert_server.py +186 -24
  60. ert/shared/version.py +3 -3
  61. ert/storage/local_ensemble.py +50 -116
  62. ert/storage/local_experiment.py +94 -109
  63. ert/storage/local_storage.py +10 -12
  64. ert/storage/migration/to24.py +26 -0
  65. ert/storage/migration/to25.py +91 -0
  66. ert/utils/__init__.py +20 -0
  67. {ert-19.0.1.dist-info → ert-20.0.0b1.dist-info}/METADATA +4 -51
  68. {ert-19.0.1.dist-info → ert-20.0.0b1.dist-info}/RECORD +80 -83
  69. everest/bin/everest_script.py +5 -5
  70. everest/bin/kill_script.py +2 -2
  71. everest/bin/monitor_script.py +2 -2
  72. everest/bin/utils.py +4 -4
  73. everest/detached/everserver.py +6 -6
  74. everest/gui/everest_client.py +0 -6
  75. everest/gui/main_window.py +2 -2
  76. everest/util/__init__.py +1 -19
  77. ert/dark_storage/compute/__init__.py +0 -0
  78. ert/dark_storage/endpoints/compute/__init__.py +0 -0
  79. ert/dark_storage/endpoints/compute/misfits.py +0 -95
  80. ert/services/_base_service.py +0 -387
  81. ert/services/webviz_ert_service.py +0 -20
  82. ert/shared/storage/command.py +0 -38
  83. ert/shared/storage/extraction.py +0 -42
  84. {ert-19.0.1.dist-info → ert-20.0.0b1.dist-info}/WHEEL +0 -0
  85. {ert-19.0.1.dist-info → ert-20.0.0b1.dist-info}/entry_points.txt +0 -0
  86. {ert-19.0.1.dist-info → ert-20.0.0b1.dist-info}/licenses/COPYING +0 -0
  87. {ert-19.0.1.dist-info → ert-20.0.0b1.dist-info}/top_level.txt +0 -0
@@ -1,17 +1,27 @@
1
+ from __future__ import annotations
2
+
3
+ import logging
1
4
  import os
2
5
  from collections.abc import Sequence
3
- from dataclasses import dataclass
6
+ from datetime import datetime
4
7
  from enum import StrEnum
5
- from itertools import starmap
6
- from typing import Any, Self
8
+ from typing import Annotated, Any, Literal, Self, assert_never
9
+
10
+ import numpy as np
11
+ import pandas as pd
12
+ from pydantic import BaseModel, Field
7
13
 
14
+ from ..validation import rangestring_to_list
8
15
  from .parsing import (
16
+ ConfigWarning,
9
17
  ErrorInfo,
10
18
  ObservationConfigError,
11
19
  ObservationDict,
12
20
  ObservationType,
13
21
  )
14
22
 
23
+ logger = logging.getLogger(__name__)
24
+
15
25
 
16
26
  class ErrorModes(StrEnum):
17
27
  REL = "REL"
@@ -19,102 +29,67 @@ class ErrorModes(StrEnum):
19
29
  RELMIN = "RELMIN"
20
30
 
21
31
 
22
- @dataclass
23
- class ObservationError:
24
- error_mode: ErrorModes
25
- error: float
26
- error_min: float
27
-
28
-
29
- @dataclass
30
- class Segment(ObservationError):
31
- name: str
32
- start: int
33
- stop: int
34
-
35
-
36
- @dataclass
37
- class HistoryObservation(ObservationError):
38
- name: str
39
- segments: list[Segment]
40
-
41
- @property
42
- def key(self) -> str:
43
- """The :term:`summary key` to be fetched from :ref:`refcase`."""
44
- # For history observations the key is also the name, ie.
45
- # "HISTORY_OBSERVATION FOPR" means to add the values from
46
- # the summary vector FOPRH in refcase as observations.
47
- return self.name
48
-
49
- @classmethod
50
- def from_obs_dict(cls, directory: str, observation_dict: ObservationDict) -> Self:
51
- error_mode = ErrorModes.RELMIN
52
- error = 0.1
53
- error_min = 0.1
54
- segments = []
55
- for key, value in observation_dict.items():
56
- match key:
57
- case "type" | "name":
58
- pass
59
- case "ERROR":
60
- error = validate_positive_float(value, key)
61
- case "ERROR_MIN":
62
- error_min = validate_positive_float(value, key)
63
- case "ERROR_MODE":
64
- error_mode = validate_error_mode(value)
65
- case "segments":
66
- segments = list(starmap(_validate_segment_dict, value))
67
- case _:
68
- raise _unknown_key_error(str(key), observation_dict["name"])
69
-
70
- return cls(
71
- name=observation_dict["name"],
72
- error_mode=error_mode,
73
- error=error,
74
- error_min=error_min,
75
- segments=segments,
76
- )
77
-
78
-
79
- @dataclass
80
- class ObservationDate:
81
- days: float | None = None
82
- hours: float | None = None
83
- date: str | None = None
84
- restart: int | None = None
85
-
86
-
87
- @dataclass
88
- class _SummaryValues:
32
+ class _SummaryValues(BaseModel):
33
+ type: Literal["summary_observation"] = "summary_observation"
89
34
  name: str
90
35
  value: float
36
+ error: float
91
37
  key: str #: The :term:`summary key` in the summary response
38
+ date: str
92
39
  east: float | None = None
93
40
  north: float | None = None
94
41
  radius: float | None = None
95
42
 
96
43
 
97
- @dataclass
98
- class SummaryObservation(ObservationDate, _SummaryValues, ObservationError):
44
+ def _parse_date(date_str: str) -> datetime:
45
+ try:
46
+ return datetime.fromisoformat(date_str)
47
+ except ValueError:
48
+ try:
49
+ date = datetime.strptime(date_str, "%d/%m/%Y")
50
+ except ValueError as err:
51
+ raise ObservationConfigError.with_context(
52
+ f"Unsupported date format {date_str}. Please use ISO date format",
53
+ date_str,
54
+ ) from err
55
+ else:
56
+ ConfigWarning.warn(
57
+ f"Deprecated time format {date_str}."
58
+ " Please use ISO date format YYYY-MM-DD",
59
+ date_str,
60
+ )
61
+ return date
62
+
63
+
64
+ class SummaryObservation(_SummaryValues):
99
65
  @classmethod
100
- def from_obs_dict(cls, directory: str, observation_dict: ObservationDict) -> Self:
66
+ def from_obs_dict(
67
+ cls, directory: str, observation_dict: ObservationDict
68
+ ) -> list[Self]:
101
69
  error_mode = ErrorModes.ABS
102
70
  summary_key = None
103
71
 
104
- date_dict: ObservationDate = ObservationDate()
72
+ date: str | None = None
105
73
  float_values: dict[str, float] = {"ERROR_MIN": 0.1}
106
- localization_values: dict[str, float | None] = {}
74
+ localization_dict: dict[LOCALIZATION_KEYS, float | None] = {}
107
75
  for key, value in observation_dict.items():
108
76
  match key:
109
77
  case "type" | "name":
110
78
  pass
111
- case "RESTART":
112
- date_dict.restart = validate_positive_int(value, key)
113
79
  case "ERROR" | "ERROR_MIN":
114
- float_values[str(key)] = validate_positive_float(value, key)
115
- case "DAYS" | "HOURS":
116
- setattr(
117
- date_dict, str(key).lower(), validate_positive_float(value, key)
80
+ float_values[str(key)] = validate_positive_float(
81
+ value, key, strictly_positive=True
82
+ )
83
+ case "DAYS" | "HOURS" | "RESTART":
84
+ raise ObservationConfigError.with_context(
85
+ (
86
+ "SUMMARY_OBSERVATION must use DATE to specify "
87
+ "date, DAYS | HOURS is no longer allowed. "
88
+ "Please run:\n ert convert_observations "
89
+ "<your_ert_config.ert>\nto migrate the observation config "
90
+ "to use the correct format."
91
+ ),
92
+ key,
118
93
  )
119
94
  case "VALUE":
120
95
  float_values[str(key)] = validate_float(value, key)
@@ -123,16 +98,10 @@ class SummaryObservation(ObservationDate, _SummaryValues, ObservationError):
123
98
  case "KEY":
124
99
  summary_key = value
125
100
  case "DATE":
126
- date_dict.date = value
101
+ date = value
127
102
  case "LOCALIZATION":
128
103
  validate_localization(value, observation_dict["name"])
129
- localization_values["east"] = validate_float(value["EAST"], key)
130
- localization_values["north"] = validate_float(value["NORTH"], key)
131
- localization_values["radius"] = (
132
- validate_float(value["RADIUS"], key)
133
- if "RADIUS" in value
134
- else None
135
- )
104
+ extract_localization_values(localization_dict, value, key)
136
105
  case _:
137
106
  raise _unknown_key_error(str(key), observation_dict["name"])
138
107
  if "VALUE" not in float_values:
@@ -142,82 +111,231 @@ class SummaryObservation(ObservationDate, _SummaryValues, ObservationError):
142
111
  if "ERROR" not in float_values:
143
112
  raise _missing_value_error(observation_dict["name"], "ERROR")
144
113
 
145
- return cls(
114
+ assert date is not None
115
+ # Raise errors if the date is off
116
+ parsed_date: datetime = _parse_date(date)
117
+ standardized_date = parsed_date.date().isoformat()
118
+
119
+ value = float_values["VALUE"]
120
+ input_error = float_values["ERROR"]
121
+ error_min = float_values["ERROR_MIN"]
122
+
123
+ error = input_error
124
+ match error_mode:
125
+ case ErrorModes.ABS:
126
+ error = validate_positive_float(
127
+ np.abs(input_error), summary_key, strictly_positive=True
128
+ )
129
+ case ErrorModes.REL:
130
+ error = validate_positive_float(
131
+ np.abs(value) * input_error, summary_key, strictly_positive=True
132
+ )
133
+ case ErrorModes.RELMIN:
134
+ error = validate_positive_float(
135
+ np.maximum(np.abs(value) * input_error, error_min),
136
+ summary_key,
137
+ strictly_positive=True,
138
+ )
139
+ case default:
140
+ assert_never(default)
141
+
142
+ obs_instance = cls(
146
143
  name=observation_dict["name"],
147
- error_mode=error_mode,
148
- error=float_values["ERROR"],
149
- error_min=float_values["ERROR_MIN"],
144
+ error=error,
150
145
  key=summary_key,
151
- value=float_values["VALUE"],
152
- east=localization_values.get("east"),
153
- north=localization_values.get("north"),
154
- radius=localization_values.get("radius"),
155
- **date_dict.__dict__,
146
+ value=value,
147
+ east=localization_dict.get("east"),
148
+ north=localization_dict.get("north"),
149
+ radius=localization_dict.get("radius"),
150
+ date=standardized_date,
156
151
  )
152
+ # Bypass pydantic discarding context
153
+ # only relevant for ERT config surfacing validation errors
154
+ # irrelevant for runmodels etc.
155
+ obs_instance.name = observation_dict["name"]
157
156
 
157
+ return [obs_instance]
158
158
 
159
- @dataclass
160
- class _GeneralObservation:
159
+
160
+ class _GeneralObservation(BaseModel):
161
+ type: Literal["general_observation"] = "general_observation"
161
162
  name: str
162
163
  data: str
163
- value: float | None = None
164
- error: float | None = None
165
- index_list: str | None = None
166
- index_file: str | None = None
167
- obs_file: str | None = None
164
+ value: float
165
+ error: float
166
+ restart: int
167
+ index: int
168
+ east: float | None = None
169
+ north: float | None = None
170
+ radius: float | None = None
168
171
 
169
172
 
170
- @dataclass
171
- class GeneralObservation(ObservationDate, _GeneralObservation):
173
+ class GeneralObservation(_GeneralObservation):
172
174
  @classmethod
173
- def from_obs_dict(cls, directory: str, observation_dict: ObservationDict) -> Self:
175
+ def from_obs_dict(
176
+ cls, directory: str, observation_dict: ObservationDict
177
+ ) -> list[Self]:
174
178
  try:
175
179
  data = observation_dict["DATA"]
176
180
  except KeyError as err:
177
181
  raise _missing_value_error(observation_dict["name"], "DATA") from err
178
182
 
179
- output = cls(name=observation_dict["name"], data=data)
180
- for key, value in observation_dict.items():
181
- match key:
182
- case "type" | "name":
183
- pass
184
- case "RESTART":
185
- output.restart = validate_positive_int(value, key)
186
- case "VALUE":
187
- output.value = validate_float(value, key)
188
- case "ERROR" | "DAYS" | "HOURS":
189
- setattr(
190
- output, str(key).lower(), validate_positive_float(value, key)
191
- )
192
- case "DATE" | "INDEX_LIST":
193
- setattr(output, str(key).lower(), value)
194
- case "OBS_FILE" | "INDEX_FILE":
195
- assert not isinstance(key, tuple)
196
- filename = value
197
- if not os.path.isabs(filename):
198
- filename = os.path.join(directory, filename)
199
- if not os.path.exists(filename):
200
- raise ObservationConfigError.with_context(
201
- "The following keywords did not"
202
- f" resolve to a valid path:\n {key}",
203
- value,
204
- )
205
- setattr(output, str(key).lower(), filename)
206
- case "DATA":
207
- output.data = value
208
- case _:
209
- raise _unknown_key_error(str(key), observation_dict["name"])
210
- if output.value is not None and output.error is None:
183
+ allowed = {
184
+ "type",
185
+ "name",
186
+ "RESTART",
187
+ "VALUE",
188
+ "ERROR",
189
+ "DATE",
190
+ "DAYS",
191
+ "HOURS",
192
+ "INDEX_LIST",
193
+ "OBS_FILE",
194
+ "INDEX_FILE",
195
+ "DATA",
196
+ }
197
+
198
+ extra = set(observation_dict.keys()) - allowed
199
+ if extra:
200
+ raise _unknown_key_error(str(next(iter(extra))), observation_dict["name"])
201
+
202
+ if any(k in observation_dict for k in ("DATE", "DAYS", "HOURS")):
203
+ bad_key = next(
204
+ k for k in ("DATE", "DAYS", "HOURS") if k in observation_dict
205
+ )
206
+ raise ObservationConfigError.with_context(
207
+ (
208
+ "GENERAL_OBSERVATION must use RESTART to specify "
209
+ "report step. Please run:\n ert convert_observations "
210
+ "<your_ert_config.ert>\nto migrate the observation config "
211
+ "to use the correct format."
212
+ ),
213
+ bad_key,
214
+ )
215
+
216
+ if "OBS_FILE" in observation_dict and (
217
+ "VALUE" in observation_dict or "ERROR" in observation_dict
218
+ ):
219
+ raise ObservationConfigError.with_context(
220
+ "GENERAL_OBSERVATION cannot contain both VALUE/ERROR and OBS_FILE",
221
+ observation_dict["name"],
222
+ )
223
+
224
+ if "INDEX_FILE" in observation_dict and "INDEX_LIST" in observation_dict:
211
225
  raise ObservationConfigError.with_context(
212
- f"For GENERAL_OBSERVATION {observation_dict['name']}, with"
213
- f" VALUE = {output.value}, ERROR must also be given.",
226
+ (
227
+ "GENERAL_OBSERVATION "
228
+ f"{observation_dict['name']} has both INDEX_FILE and INDEX_LIST."
229
+ ),
214
230
  observation_dict["name"],
215
231
  )
216
- return output
217
232
 
233
+ restart = (
234
+ validate_positive_int(observation_dict["RESTART"], "RESTART")
235
+ if "RESTART" in observation_dict
236
+ else 0
237
+ )
238
+
239
+ if "OBS_FILE" not in observation_dict:
240
+ if "VALUE" in observation_dict and "ERROR" not in observation_dict:
241
+ raise ObservationConfigError.with_context(
242
+ f"For GENERAL_OBSERVATION {observation_dict['name']}, with"
243
+ f" VALUE = {observation_dict['VALUE']}, ERROR must also be given.",
244
+ observation_dict["name"],
245
+ )
246
+
247
+ if "VALUE" not in observation_dict and "ERROR" not in observation_dict:
248
+ raise ObservationConfigError.with_context(
249
+ "GENERAL_OBSERVATION must contain either VALUE "
250
+ "and ERROR or OBS_FILE",
251
+ context=observation_dict["name"],
252
+ )
253
+
254
+ obs_instance = cls(
255
+ name=observation_dict["name"],
256
+ data=data,
257
+ value=validate_float(observation_dict["VALUE"], "VALUE"),
258
+ error=validate_positive_float(
259
+ observation_dict["ERROR"], "ERROR", strictly_positive=True
260
+ ),
261
+ restart=restart,
262
+ index=0,
263
+ )
264
+ # Bypass pydantic discarding context
265
+ # only relevant for ERT config surfacing validation errors
266
+ # irrelevant for runmodels etc.
267
+ obs_instance.name = observation_dict["name"]
268
+ return [obs_instance]
269
+
270
+ obs_filename = _resolve_path(directory, observation_dict["OBS_FILE"])
271
+ try:
272
+ file_values = np.loadtxt(obs_filename, delimiter=None).ravel()
273
+ except ValueError as err:
274
+ raise ObservationConfigError.with_context(
275
+ f"Failed to read OBS_FILE {obs_filename}: {err}", obs_filename
276
+ ) from err
277
+ if len(file_values) % 2 != 0:
278
+ raise ObservationConfigError.with_context(
279
+ "Expected even number of values in GENERAL_OBSERVATION",
280
+ obs_filename,
281
+ )
282
+ values = file_values[::2]
283
+ stds = file_values[1::2]
284
+
285
+ if "INDEX_FILE" in observation_dict:
286
+ idx_file = _resolve_path(directory, observation_dict["INDEX_FILE"])
287
+ indices = np.loadtxt(idx_file, delimiter=None, dtype=np.int32).ravel()
288
+ elif "INDEX_LIST" in observation_dict:
289
+ indices = np.array(
290
+ sorted(rangestring_to_list(observation_dict["INDEX_LIST"])),
291
+ dtype=np.int32,
292
+ )
293
+ else:
294
+ indices = np.arange(len(values), dtype=np.int32)
218
295
 
219
- @dataclass
220
- class RFTObservation:
296
+ if len({len(stds), len(values), len(indices)}) != 1:
297
+ raise ObservationConfigError.with_context(
298
+ (
299
+ "Values ("
300
+ f"{values}), error ({stds}) and index list ({indices}) "
301
+ "must be of equal length"
302
+ ),
303
+ observation_dict["name"],
304
+ )
305
+
306
+ if np.any(stds <= 0):
307
+ raise ObservationConfigError.with_context(
308
+ "Observation uncertainty must be strictly > 0",
309
+ observation_dict["name"],
310
+ )
311
+
312
+ obs_instances: list[Self] = []
313
+ for _pos, (val, std, idx) in enumerate(zip(values, stds, indices, strict=True)):
314
+ # index should reflect the index provided by INDEX_FILE / INDEX_LIST
315
+ inst = cls(
316
+ name=observation_dict["name"],
317
+ data=data,
318
+ value=float(val),
319
+ error=float(std),
320
+ restart=restart,
321
+ index=int(idx),
322
+ )
323
+ # Bypass pydantic discarding context
324
+ # only relevant for ERT config surfacing validation errors
325
+ # irrelevant for runmodels etc.
326
+ inst.name = observation_dict["name"]
327
+ obs_instances.append(inst)
328
+ return obs_instances
329
+
330
+
331
+ class RFTObservation(BaseModel):
332
+ """Represents an RFT (Repeat Formation Tester) observation.
333
+
334
+ RFT observations are used to condition on pressure, saturation, or other
335
+ properties measured at specific well locations and times.
336
+ """
337
+
338
+ type: Literal["rft_observation"] = "rft_observation"
221
339
  name: str
222
340
  well: str
223
341
  date: str
@@ -227,9 +345,131 @@ class RFTObservation:
227
345
  north: float
228
346
  east: float
229
347
  tvd: float
348
+ zone: str | None = None
349
+
350
+ @classmethod
351
+ def from_csv(
352
+ cls,
353
+ directory: str,
354
+ observation_dict: ObservationDict,
355
+ filename: str,
356
+ observed_property: str = "PRESSURE",
357
+ ) -> list[Self]:
358
+ """Create RFT observations from a CSV file.
359
+
360
+ The CSV file must contain the following columns: WELL_NAME, DATE,
361
+ ERROR, NORTH, EAST, TVD, and a column for the observed property
362
+ (e.g., PRESSURE, SWAT). An optional ZONE column may also be present.
363
+
364
+ Args:
365
+ directory: Base directory for resolving relative file paths.
366
+ observation_dict: Dictionary containing the observation configuration.
367
+ filename: Path to the CSV file containing RFT observations.
368
+ observed_property: Property to observe (default: PRESSURE).
369
+
370
+ Returns:
371
+ List of RFTObservation instances created from the CSV file.
372
+
373
+ Raises:
374
+ ObservationConfigError: If the file is missing, inaccessible,
375
+ lacks required columns, or contains invalid observation values
376
+ (value=-1 and error=0).
377
+ """
378
+ if not os.path.isabs(filename):
379
+ filename = os.path.join(directory, filename)
380
+ if not os.path.exists(filename):
381
+ raise ObservationConfigError.with_context(
382
+ f"The CSV file ({filename}) does not exist or is not accessible.",
383
+ filename,
384
+ )
385
+ csv_file = pd.read_csv(
386
+ filename,
387
+ encoding="utf-8",
388
+ on_bad_lines="error",
389
+ )
390
+
391
+ required_columns = {
392
+ "WELL_NAME",
393
+ "DATE",
394
+ observed_property,
395
+ "ERROR",
396
+ "NORTH",
397
+ "EAST",
398
+ "TVD",
399
+ }
400
+ missing_required_columns = required_columns - set(csv_file.keys())
401
+ if missing_required_columns:
402
+ raise ObservationConfigError.with_context(
403
+ f"The rft observations file {filename} is missing required column(s) "
404
+ f"{', '.join(sorted(missing_required_columns))}.",
405
+ filename,
406
+ )
407
+
408
+ rft_observations = []
409
+ invalid_observations = []
410
+ for row in csv_file.itertuples(index=True):
411
+ rft_observation = cls(
412
+ name=f"{observation_dict['name']}[{row.Index}]",
413
+ well=str(row.WELL_NAME),
414
+ date=str(row.DATE),
415
+ property=observed_property,
416
+ value=validate_float(
417
+ str(getattr(row, observed_property)), observed_property
418
+ ),
419
+ error=validate_float(str(row.ERROR), "ERROR"),
420
+ north=validate_float(str(row.NORTH), "NORTH"),
421
+ east=validate_float(str(row.EAST), "EAST"),
422
+ tvd=validate_float(str(row.TVD), "TVD"),
423
+ zone=row.ZONE if "ZONE" in csv_file else None,
424
+ )
425
+ # A value of -1 and error of 0 is used by fmu.tools.rms create_rft_ertobs to
426
+ # indicate missing data. If encountered in an rft observations csv file
427
+ # it should raise an error and ask the user to remove invalid observations.
428
+ if rft_observation.value == -1 and rft_observation.error == 0:
429
+ invalid_observations.append(rft_observation)
430
+ else:
431
+ rft_observations.append(rft_observation)
432
+
433
+ if invalid_observations:
434
+ well_list = "\n - ".join(
435
+ [
436
+ f"{observation.well} at date {observation.date}"
437
+ for observation in invalid_observations
438
+ ]
439
+ )
440
+ raise ObservationConfigError.with_context(
441
+ (
442
+ f"Invalid value=-1 and error=0 detected in {filename} for "
443
+ f"well(s):\n - {well_list}\n"
444
+ "The invalid observation(s) must be removed from the file."
445
+ ),
446
+ filename,
447
+ )
448
+ return rft_observations
230
449
 
231
450
  @classmethod
232
- def from_obs_dict(cls, directory: str, observation_dict: ObservationDict) -> Self:
451
+ def from_obs_dict(
452
+ cls, directory: str, observation_dict: ObservationDict
453
+ ) -> list[Self]:
454
+ """Create RFT observations from an observation dictionary.
455
+
456
+ Supports two modes:
457
+ 1. CSV mode: Load observations from a CSV file specified by the CSV key.
458
+ 2. Direct mode: Create a single observation from individual keys
459
+ (WELL, PROPERTY, VALUE, ERROR, DATE, NORTH, EAST, TVD, ZONE).
460
+
461
+ Args:
462
+ directory: Base directory for resolving relative file paths.
463
+ observation_dict: Dictionary containing the observation configuration.
464
+
465
+ Returns:
466
+ List of RFTObservation instances. Returns multiple observations when
467
+ loading from CSV, or a single observation when using direct mode.
468
+
469
+ Raises:
470
+ ObservationConfigError: If required keys are missing or invalid.
471
+ """
472
+ csv_filename = None
233
473
  well = None
234
474
  observed_property = None
235
475
  observed_value = None
@@ -238,6 +478,7 @@ class RFTObservation:
238
478
  north = None
239
479
  east = None
240
480
  tvd = None
481
+ zone = None
241
482
  for key, value in observation_dict.items():
242
483
  match key:
243
484
  case "type" | "name":
@@ -258,8 +499,19 @@ class RFTObservation:
258
499
  east = validate_float(value, key)
259
500
  case "TVD":
260
501
  tvd = validate_float(value, key)
502
+ case "CSV":
503
+ csv_filename = value
504
+ case "ZONE":
505
+ zone = value
261
506
  case _:
262
507
  raise _unknown_key_error(str(key), observation_dict["name"])
508
+ if csv_filename is not None:
509
+ return cls.from_csv(
510
+ directory,
511
+ observation_dict,
512
+ csv_filename,
513
+ observed_property or "PRESSURE",
514
+ )
263
515
  if well is None:
264
516
  raise _missing_value_error(observation_dict["name"], "WELL")
265
517
  if observed_value is None:
@@ -276,30 +528,106 @@ class RFTObservation:
276
528
  raise _missing_value_error(observation_dict["name"], "EAST")
277
529
  if tvd is None:
278
530
  raise _missing_value_error(observation_dict["name"], "TVD")
279
- return cls(
280
- observation_dict["name"],
281
- well,
282
- date,
283
- observed_property,
284
- observed_value,
285
- error,
286
- north,
287
- east,
288
- tvd,
531
+
532
+ obs_instance = cls(
533
+ name=observation_dict["name"],
534
+ well=well,
535
+ property=observed_property,
536
+ value=observed_value,
537
+ error=error,
538
+ date=date,
539
+ north=north,
540
+ east=east,
541
+ tvd=tvd,
542
+ zone=zone,
289
543
  )
290
544
 
545
+ # Bypass pydantic discarding context
546
+ # only relevant for ERT config surfacing validation errors
547
+ # irrelevant for runmodels etc.
548
+ obs_instance.name = observation_dict["name"]
549
+
550
+ return [obs_instance]
551
+
552
+
553
+ class BreakthroughObservation(BaseModel):
554
+ type: Literal["breakthrough"] = "breakthrough"
555
+ name: str
556
+ response_key: str
557
+ date: datetime
558
+ error: float
559
+ threshold: float
560
+ north: float | None
561
+ east: float | None
562
+ radius: float | None
563
+
564
+ @classmethod
565
+ def from_obs_dict(cls, directory: str, obs_dict: ObservationDict) -> list[Self]:
566
+ response_key = None
567
+ date = None
568
+ error = None
569
+ threshold = None
570
+ localization_dict: dict[LOCALIZATION_KEYS, float | None] = {}
571
+ for key, value in obs_dict.items():
572
+ match key:
573
+ case "type" | "name":
574
+ pass
575
+ case "KEY":
576
+ response_key = value
577
+ case "DATE":
578
+ date = value
579
+ case "ERROR":
580
+ error = validate_float(value, key)
581
+ case "THRESHOLD":
582
+ threshold = validate_float(value, key)
583
+ case "LOCALIZATION":
584
+ validate_localization(value, obs_dict["name"])
585
+ extract_localization_values(localization_dict, value, key)
586
+ case _:
587
+ raise _unknown_key_error(str(key), value)
588
+
589
+ if response_key is None:
590
+ raise _missing_value_error(obs_dict["name"], "KEY")
591
+ if date is None:
592
+ raise _missing_value_error(obs_dict["name"], "DATE")
593
+ if error is None:
594
+ raise _missing_value_error(obs_dict["name"], "ERROR")
595
+ if threshold is None:
596
+ raise _missing_value_error(obs_dict["name"], "THRESHOLD")
597
+
598
+ return [
599
+ cls(
600
+ name=obs_dict["name"],
601
+ response_key=response_key,
602
+ date=date,
603
+ error=error,
604
+ threshold=threshold,
605
+ north=localization_dict.get("north"),
606
+ east=localization_dict.get("east"),
607
+ radius=localization_dict.get("radius"),
608
+ )
609
+ ]
291
610
 
292
- Observation = (
293
- HistoryObservation | SummaryObservation | GeneralObservation | RFTObservation
294
- )
611
+
612
+ Observation = Annotated[
613
+ (
614
+ SummaryObservation
615
+ | GeneralObservation
616
+ | RFTObservation
617
+ | BreakthroughObservation
618
+ ),
619
+ Field(discriminator="type"),
620
+ ]
295
621
 
296
622
  _TYPE_TO_CLASS: dict[ObservationType, type[Observation]] = {
297
- ObservationType.HISTORY: HistoryObservation,
298
623
  ObservationType.SUMMARY: SummaryObservation,
299
624
  ObservationType.GENERAL: GeneralObservation,
300
625
  ObservationType.RFT: RFTObservation,
626
+ ObservationType.BREAKTHROUGH: BreakthroughObservation,
301
627
  }
302
628
 
629
+ LOCALIZATION_KEYS = Literal["east", "north", "radius"]
630
+
303
631
 
304
632
  def make_observations(
305
633
  directory: str, observation_dicts: Sequence[ObservationDict]
@@ -312,10 +640,20 @@ def make_observations(
312
640
  inp: The collection of statements to validate.
313
641
  """
314
642
  result: list[Observation] = []
315
- error_list: list[ErrorInfo] = []
643
+ error_list: list[ErrorInfo | ObservationConfigError] = []
316
644
  for obs_dict in observation_dicts:
645
+ if obs_dict["type"] == ObservationType.HISTORY:
646
+ msg = (
647
+ "HISTORY_OBSERVATION is deprecated, and must be specified "
648
+ "as SUMMARY_OBSERVATION. Run"
649
+ " ert convert_observations <ert_config.ert> to convert your "
650
+ "observations automatically"
651
+ )
652
+ logger.error(msg)
653
+ error_list.append(ObservationConfigError(msg))
654
+ continue
317
655
  try:
318
- result.append(
656
+ result.extend(
319
657
  _TYPE_TO_CLASS[obs_dict["type"]].from_obs_dict(directory, obs_dict)
320
658
  )
321
659
  except KeyError as err:
@@ -329,41 +667,6 @@ def make_observations(
329
667
  return result
330
668
 
331
669
 
332
- def _validate_segment_dict(name_token: str, inp: dict[str, Any]) -> Segment:
333
- start = None
334
- stop = None
335
- error_mode = ErrorModes.RELMIN
336
- error = 0.1
337
- error_min = 0.1
338
- for key, value in inp.items():
339
- match key:
340
- case "START":
341
- start = validate_int(value, key)
342
- case "STOP":
343
- stop = validate_int(value, key)
344
- case "ERROR":
345
- error = validate_positive_float(value, key)
346
- case "ERROR_MIN":
347
- error_min = validate_positive_float(value, key)
348
- case "ERROR_MODE":
349
- error_mode = validate_error_mode(value)
350
- case _:
351
- raise _unknown_key_error(key, name_token)
352
-
353
- if start is None:
354
- raise _missing_value_error(name_token, "START")
355
- if stop is None:
356
- raise _missing_value_error(name_token, "STOP")
357
- return Segment(
358
- name=name_token,
359
- start=start,
360
- stop=stop,
361
- error_mode=error_mode,
362
- error=error,
363
- error_min=error_min,
364
- )
365
-
366
-
367
670
  def validate_error_mode(inp: str) -> ErrorModes:
368
671
  if inp == "REL":
369
672
  return ErrorModes.REL
@@ -390,12 +693,15 @@ def validate_int(val: str, key: str) -> int:
390
693
  raise _conversion_error(key, val, "int") from err
391
694
 
392
695
 
393
- def validate_positive_float(val: str, key: str) -> float:
696
+ def validate_positive_float(
697
+ val: str, key: str, strictly_positive: bool = False
698
+ ) -> float:
394
699
  v = validate_float(val, key)
395
- if v < 0:
700
+ if v < 0 or (v <= 0 and strictly_positive):
396
701
  raise ObservationConfigError.with_context(
397
702
  f'Failed to validate "{val}" in {key}={val}.'
398
- f" {key} must be given a positive value.",
703
+ f" {key} must be given a "
704
+ f"{'strictly ' if strictly_positive else ''}positive value.",
399
705
  val,
400
706
  )
401
707
  return v
@@ -414,6 +720,16 @@ def validate_localization(val: dict[str, Any], obs_name: str) -> None:
414
720
  raise ObservationConfigError.from_collected(errors)
415
721
 
416
722
 
723
+ def extract_localization_values(
724
+ localization_dict: dict[LOCALIZATION_KEYS, float | None], value: Any, key: str
725
+ ) -> None:
726
+ localization_dict["east"] = validate_float(value["EAST"], key)
727
+ localization_dict["north"] = validate_float(value["NORTH"], key)
728
+ localization_dict["radius"] = (
729
+ validate_float(value["RADIUS"], key) if "RADIUS" in value else None
730
+ )
731
+
732
+
417
733
  def validate_positive_int(val: str, key: str) -> int:
418
734
  try:
419
735
  v = int(val)
@@ -434,6 +750,17 @@ def _missing_value_error(name_token: str, value_key: str) -> ObservationConfigEr
434
750
  )
435
751
 
436
752
 
753
+ def _resolve_path(directory: str, filename: str) -> str:
754
+ if not os.path.isabs(filename):
755
+ filename = os.path.join(directory, filename)
756
+ if not os.path.exists(filename):
757
+ raise ObservationConfigError.with_context(
758
+ f"The following keywords did not resolve to a valid path:\n {filename}",
759
+ filename,
760
+ )
761
+ return filename
762
+
763
+
437
764
  def _conversion_error(token: str, value: Any, type_name: str) -> ObservationConfigError:
438
765
  return ObservationConfigError.with_context(
439
766
  f'Could not convert {value} to {type_name}. Failed to validate "{value}"',