ert 19.0.1__py3-none-any.whl → 20.0.0b1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ert/__main__.py +94 -63
- ert/analysis/_es_update.py +11 -14
- ert/cli/main.py +1 -1
- ert/config/__init__.py +3 -2
- ert/config/_create_observation_dataframes.py +52 -375
- ert/config/_observations.py +527 -200
- ert/config/_read_summary.py +4 -5
- ert/config/ert_config.py +52 -117
- ert/config/everest_control.py +40 -39
- ert/config/everest_response.py +3 -15
- ert/config/field.py +4 -76
- ert/config/forward_model_step.py +17 -1
- ert/config/gen_data_config.py +14 -17
- ert/config/observation_config_migrations.py +821 -0
- ert/config/parameter_config.py +18 -28
- ert/config/parsing/__init__.py +0 -1
- ert/config/parsing/_parse_zonemap.py +45 -0
- ert/config/parsing/config_keywords.py +1 -0
- ert/config/parsing/config_schema.py +2 -0
- ert/config/parsing/observations_parser.py +2 -0
- ert/config/response_config.py +5 -23
- ert/config/rft_config.py +129 -31
- ert/config/summary_config.py +1 -13
- ert/config/surface_config.py +0 -57
- ert/dark_storage/compute/misfits.py +0 -42
- ert/dark_storage/endpoints/__init__.py +0 -2
- ert/dark_storage/endpoints/experiments.py +2 -5
- ert/dark_storage/json_schema/experiment.py +1 -2
- ert/field_utils/__init__.py +0 -2
- ert/field_utils/field_utils.py +1 -117
- ert/gui/ertwidgets/listeditbox.py +9 -1
- ert/gui/ertwidgets/models/ertsummary.py +20 -6
- ert/gui/ertwidgets/pathchooser.py +9 -1
- ert/gui/ertwidgets/stringbox.py +11 -3
- ert/gui/ertwidgets/textbox.py +10 -3
- ert/gui/ertwidgets/validationsupport.py +19 -1
- ert/gui/main_window.py +11 -6
- ert/gui/simulation/experiment_panel.py +1 -1
- ert/gui/simulation/run_dialog.py +11 -1
- ert/gui/tools/manage_experiments/export_dialog.py +4 -0
- ert/gui/tools/manage_experiments/manage_experiments_panel.py +1 -0
- ert/gui/tools/manage_experiments/storage_info_widget.py +1 -1
- ert/gui/tools/manage_experiments/storage_widget.py +21 -4
- ert/gui/tools/plot/data_type_proxy_model.py +1 -1
- ert/gui/tools/plot/plot_api.py +35 -27
- ert/gui/tools/plot/plot_widget.py +5 -0
- ert/gui/tools/plot/plot_window.py +4 -7
- ert/run_models/ensemble_experiment.py +2 -9
- ert/run_models/ensemble_smoother.py +1 -9
- ert/run_models/everest_run_model.py +31 -23
- ert/run_models/initial_ensemble_run_model.py +19 -22
- ert/run_models/manual_update.py +11 -5
- ert/run_models/model_factory.py +7 -7
- ert/run_models/multiple_data_assimilation.py +3 -16
- ert/sample_prior.py +12 -14
- ert/scheduler/job.py +24 -4
- ert/services/__init__.py +7 -3
- ert/services/_storage_main.py +59 -22
- ert/services/ert_server.py +186 -24
- ert/shared/version.py +3 -3
- ert/storage/local_ensemble.py +50 -116
- ert/storage/local_experiment.py +94 -109
- ert/storage/local_storage.py +10 -12
- ert/storage/migration/to24.py +26 -0
- ert/storage/migration/to25.py +91 -0
- ert/utils/__init__.py +20 -0
- {ert-19.0.1.dist-info → ert-20.0.0b1.dist-info}/METADATA +4 -51
- {ert-19.0.1.dist-info → ert-20.0.0b1.dist-info}/RECORD +80 -83
- everest/bin/everest_script.py +5 -5
- everest/bin/kill_script.py +2 -2
- everest/bin/monitor_script.py +2 -2
- everest/bin/utils.py +4 -4
- everest/detached/everserver.py +6 -6
- everest/gui/everest_client.py +0 -6
- everest/gui/main_window.py +2 -2
- everest/util/__init__.py +1 -19
- ert/dark_storage/compute/__init__.py +0 -0
- ert/dark_storage/endpoints/compute/__init__.py +0 -0
- ert/dark_storage/endpoints/compute/misfits.py +0 -95
- ert/services/_base_service.py +0 -387
- ert/services/webviz_ert_service.py +0 -20
- ert/shared/storage/command.py +0 -38
- ert/shared/storage/extraction.py +0 -42
- {ert-19.0.1.dist-info → ert-20.0.0b1.dist-info}/WHEEL +0 -0
- {ert-19.0.1.dist-info → ert-20.0.0b1.dist-info}/entry_points.txt +0 -0
- {ert-19.0.1.dist-info → ert-20.0.0b1.dist-info}/licenses/COPYING +0 -0
- {ert-19.0.1.dist-info → ert-20.0.0b1.dist-info}/top_level.txt +0 -0
ert/config/_observations.py
CHANGED
|
@@ -1,17 +1,27 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import logging
|
|
1
4
|
import os
|
|
2
5
|
from collections.abc import Sequence
|
|
3
|
-
from
|
|
6
|
+
from datetime import datetime
|
|
4
7
|
from enum import StrEnum
|
|
5
|
-
from
|
|
6
|
-
|
|
8
|
+
from typing import Annotated, Any, Literal, Self, assert_never
|
|
9
|
+
|
|
10
|
+
import numpy as np
|
|
11
|
+
import pandas as pd
|
|
12
|
+
from pydantic import BaseModel, Field
|
|
7
13
|
|
|
14
|
+
from ..validation import rangestring_to_list
|
|
8
15
|
from .parsing import (
|
|
16
|
+
ConfigWarning,
|
|
9
17
|
ErrorInfo,
|
|
10
18
|
ObservationConfigError,
|
|
11
19
|
ObservationDict,
|
|
12
20
|
ObservationType,
|
|
13
21
|
)
|
|
14
22
|
|
|
23
|
+
logger = logging.getLogger(__name__)
|
|
24
|
+
|
|
15
25
|
|
|
16
26
|
class ErrorModes(StrEnum):
|
|
17
27
|
REL = "REL"
|
|
@@ -19,102 +29,67 @@ class ErrorModes(StrEnum):
|
|
|
19
29
|
RELMIN = "RELMIN"
|
|
20
30
|
|
|
21
31
|
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
error_mode: ErrorModes
|
|
25
|
-
error: float
|
|
26
|
-
error_min: float
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
@dataclass
|
|
30
|
-
class Segment(ObservationError):
|
|
31
|
-
name: str
|
|
32
|
-
start: int
|
|
33
|
-
stop: int
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
@dataclass
|
|
37
|
-
class HistoryObservation(ObservationError):
|
|
38
|
-
name: str
|
|
39
|
-
segments: list[Segment]
|
|
40
|
-
|
|
41
|
-
@property
|
|
42
|
-
def key(self) -> str:
|
|
43
|
-
"""The :term:`summary key` to be fetched from :ref:`refcase`."""
|
|
44
|
-
# For history observations the key is also the name, ie.
|
|
45
|
-
# "HISTORY_OBSERVATION FOPR" means to add the values from
|
|
46
|
-
# the summary vector FOPRH in refcase as observations.
|
|
47
|
-
return self.name
|
|
48
|
-
|
|
49
|
-
@classmethod
|
|
50
|
-
def from_obs_dict(cls, directory: str, observation_dict: ObservationDict) -> Self:
|
|
51
|
-
error_mode = ErrorModes.RELMIN
|
|
52
|
-
error = 0.1
|
|
53
|
-
error_min = 0.1
|
|
54
|
-
segments = []
|
|
55
|
-
for key, value in observation_dict.items():
|
|
56
|
-
match key:
|
|
57
|
-
case "type" | "name":
|
|
58
|
-
pass
|
|
59
|
-
case "ERROR":
|
|
60
|
-
error = validate_positive_float(value, key)
|
|
61
|
-
case "ERROR_MIN":
|
|
62
|
-
error_min = validate_positive_float(value, key)
|
|
63
|
-
case "ERROR_MODE":
|
|
64
|
-
error_mode = validate_error_mode(value)
|
|
65
|
-
case "segments":
|
|
66
|
-
segments = list(starmap(_validate_segment_dict, value))
|
|
67
|
-
case _:
|
|
68
|
-
raise _unknown_key_error(str(key), observation_dict["name"])
|
|
69
|
-
|
|
70
|
-
return cls(
|
|
71
|
-
name=observation_dict["name"],
|
|
72
|
-
error_mode=error_mode,
|
|
73
|
-
error=error,
|
|
74
|
-
error_min=error_min,
|
|
75
|
-
segments=segments,
|
|
76
|
-
)
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
@dataclass
|
|
80
|
-
class ObservationDate:
|
|
81
|
-
days: float | None = None
|
|
82
|
-
hours: float | None = None
|
|
83
|
-
date: str | None = None
|
|
84
|
-
restart: int | None = None
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
@dataclass
|
|
88
|
-
class _SummaryValues:
|
|
32
|
+
class _SummaryValues(BaseModel):
|
|
33
|
+
type: Literal["summary_observation"] = "summary_observation"
|
|
89
34
|
name: str
|
|
90
35
|
value: float
|
|
36
|
+
error: float
|
|
91
37
|
key: str #: The :term:`summary key` in the summary response
|
|
38
|
+
date: str
|
|
92
39
|
east: float | None = None
|
|
93
40
|
north: float | None = None
|
|
94
41
|
radius: float | None = None
|
|
95
42
|
|
|
96
43
|
|
|
97
|
-
|
|
98
|
-
|
|
44
|
+
def _parse_date(date_str: str) -> datetime:
|
|
45
|
+
try:
|
|
46
|
+
return datetime.fromisoformat(date_str)
|
|
47
|
+
except ValueError:
|
|
48
|
+
try:
|
|
49
|
+
date = datetime.strptime(date_str, "%d/%m/%Y")
|
|
50
|
+
except ValueError as err:
|
|
51
|
+
raise ObservationConfigError.with_context(
|
|
52
|
+
f"Unsupported date format {date_str}. Please use ISO date format",
|
|
53
|
+
date_str,
|
|
54
|
+
) from err
|
|
55
|
+
else:
|
|
56
|
+
ConfigWarning.warn(
|
|
57
|
+
f"Deprecated time format {date_str}."
|
|
58
|
+
" Please use ISO date format YYYY-MM-DD",
|
|
59
|
+
date_str,
|
|
60
|
+
)
|
|
61
|
+
return date
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
class SummaryObservation(_SummaryValues):
|
|
99
65
|
@classmethod
|
|
100
|
-
def from_obs_dict(
|
|
66
|
+
def from_obs_dict(
|
|
67
|
+
cls, directory: str, observation_dict: ObservationDict
|
|
68
|
+
) -> list[Self]:
|
|
101
69
|
error_mode = ErrorModes.ABS
|
|
102
70
|
summary_key = None
|
|
103
71
|
|
|
104
|
-
|
|
72
|
+
date: str | None = None
|
|
105
73
|
float_values: dict[str, float] = {"ERROR_MIN": 0.1}
|
|
106
|
-
|
|
74
|
+
localization_dict: dict[LOCALIZATION_KEYS, float | None] = {}
|
|
107
75
|
for key, value in observation_dict.items():
|
|
108
76
|
match key:
|
|
109
77
|
case "type" | "name":
|
|
110
78
|
pass
|
|
111
|
-
case "RESTART":
|
|
112
|
-
date_dict.restart = validate_positive_int(value, key)
|
|
113
79
|
case "ERROR" | "ERROR_MIN":
|
|
114
|
-
float_values[str(key)] = validate_positive_float(
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
80
|
+
float_values[str(key)] = validate_positive_float(
|
|
81
|
+
value, key, strictly_positive=True
|
|
82
|
+
)
|
|
83
|
+
case "DAYS" | "HOURS" | "RESTART":
|
|
84
|
+
raise ObservationConfigError.with_context(
|
|
85
|
+
(
|
|
86
|
+
"SUMMARY_OBSERVATION must use DATE to specify "
|
|
87
|
+
"date, DAYS | HOURS is no longer allowed. "
|
|
88
|
+
"Please run:\n ert convert_observations "
|
|
89
|
+
"<your_ert_config.ert>\nto migrate the observation config "
|
|
90
|
+
"to use the correct format."
|
|
91
|
+
),
|
|
92
|
+
key,
|
|
118
93
|
)
|
|
119
94
|
case "VALUE":
|
|
120
95
|
float_values[str(key)] = validate_float(value, key)
|
|
@@ -123,16 +98,10 @@ class SummaryObservation(ObservationDate, _SummaryValues, ObservationError):
|
|
|
123
98
|
case "KEY":
|
|
124
99
|
summary_key = value
|
|
125
100
|
case "DATE":
|
|
126
|
-
|
|
101
|
+
date = value
|
|
127
102
|
case "LOCALIZATION":
|
|
128
103
|
validate_localization(value, observation_dict["name"])
|
|
129
|
-
|
|
130
|
-
localization_values["north"] = validate_float(value["NORTH"], key)
|
|
131
|
-
localization_values["radius"] = (
|
|
132
|
-
validate_float(value["RADIUS"], key)
|
|
133
|
-
if "RADIUS" in value
|
|
134
|
-
else None
|
|
135
|
-
)
|
|
104
|
+
extract_localization_values(localization_dict, value, key)
|
|
136
105
|
case _:
|
|
137
106
|
raise _unknown_key_error(str(key), observation_dict["name"])
|
|
138
107
|
if "VALUE" not in float_values:
|
|
@@ -142,82 +111,231 @@ class SummaryObservation(ObservationDate, _SummaryValues, ObservationError):
|
|
|
142
111
|
if "ERROR" not in float_values:
|
|
143
112
|
raise _missing_value_error(observation_dict["name"], "ERROR")
|
|
144
113
|
|
|
145
|
-
|
|
114
|
+
assert date is not None
|
|
115
|
+
# Raise errors if the date is off
|
|
116
|
+
parsed_date: datetime = _parse_date(date)
|
|
117
|
+
standardized_date = parsed_date.date().isoformat()
|
|
118
|
+
|
|
119
|
+
value = float_values["VALUE"]
|
|
120
|
+
input_error = float_values["ERROR"]
|
|
121
|
+
error_min = float_values["ERROR_MIN"]
|
|
122
|
+
|
|
123
|
+
error = input_error
|
|
124
|
+
match error_mode:
|
|
125
|
+
case ErrorModes.ABS:
|
|
126
|
+
error = validate_positive_float(
|
|
127
|
+
np.abs(input_error), summary_key, strictly_positive=True
|
|
128
|
+
)
|
|
129
|
+
case ErrorModes.REL:
|
|
130
|
+
error = validate_positive_float(
|
|
131
|
+
np.abs(value) * input_error, summary_key, strictly_positive=True
|
|
132
|
+
)
|
|
133
|
+
case ErrorModes.RELMIN:
|
|
134
|
+
error = validate_positive_float(
|
|
135
|
+
np.maximum(np.abs(value) * input_error, error_min),
|
|
136
|
+
summary_key,
|
|
137
|
+
strictly_positive=True,
|
|
138
|
+
)
|
|
139
|
+
case default:
|
|
140
|
+
assert_never(default)
|
|
141
|
+
|
|
142
|
+
obs_instance = cls(
|
|
146
143
|
name=observation_dict["name"],
|
|
147
|
-
|
|
148
|
-
error=float_values["ERROR"],
|
|
149
|
-
error_min=float_values["ERROR_MIN"],
|
|
144
|
+
error=error,
|
|
150
145
|
key=summary_key,
|
|
151
|
-
value=
|
|
152
|
-
east=
|
|
153
|
-
north=
|
|
154
|
-
radius=
|
|
155
|
-
|
|
146
|
+
value=value,
|
|
147
|
+
east=localization_dict.get("east"),
|
|
148
|
+
north=localization_dict.get("north"),
|
|
149
|
+
radius=localization_dict.get("radius"),
|
|
150
|
+
date=standardized_date,
|
|
156
151
|
)
|
|
152
|
+
# Bypass pydantic discarding context
|
|
153
|
+
# only relevant for ERT config surfacing validation errors
|
|
154
|
+
# irrelevant for runmodels etc.
|
|
155
|
+
obs_instance.name = observation_dict["name"]
|
|
157
156
|
|
|
157
|
+
return [obs_instance]
|
|
158
158
|
|
|
159
|
-
|
|
160
|
-
class _GeneralObservation:
|
|
159
|
+
|
|
160
|
+
class _GeneralObservation(BaseModel):
|
|
161
|
+
type: Literal["general_observation"] = "general_observation"
|
|
161
162
|
name: str
|
|
162
163
|
data: str
|
|
163
|
-
value: float
|
|
164
|
-
error: float
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
164
|
+
value: float
|
|
165
|
+
error: float
|
|
166
|
+
restart: int
|
|
167
|
+
index: int
|
|
168
|
+
east: float | None = None
|
|
169
|
+
north: float | None = None
|
|
170
|
+
radius: float | None = None
|
|
168
171
|
|
|
169
172
|
|
|
170
|
-
|
|
171
|
-
class GeneralObservation(ObservationDate, _GeneralObservation):
|
|
173
|
+
class GeneralObservation(_GeneralObservation):
|
|
172
174
|
@classmethod
|
|
173
|
-
def from_obs_dict(
|
|
175
|
+
def from_obs_dict(
|
|
176
|
+
cls, directory: str, observation_dict: ObservationDict
|
|
177
|
+
) -> list[Self]:
|
|
174
178
|
try:
|
|
175
179
|
data = observation_dict["DATA"]
|
|
176
180
|
except KeyError as err:
|
|
177
181
|
raise _missing_value_error(observation_dict["name"], "DATA") from err
|
|
178
182
|
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
183
|
+
allowed = {
|
|
184
|
+
"type",
|
|
185
|
+
"name",
|
|
186
|
+
"RESTART",
|
|
187
|
+
"VALUE",
|
|
188
|
+
"ERROR",
|
|
189
|
+
"DATE",
|
|
190
|
+
"DAYS",
|
|
191
|
+
"HOURS",
|
|
192
|
+
"INDEX_LIST",
|
|
193
|
+
"OBS_FILE",
|
|
194
|
+
"INDEX_FILE",
|
|
195
|
+
"DATA",
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
extra = set(observation_dict.keys()) - allowed
|
|
199
|
+
if extra:
|
|
200
|
+
raise _unknown_key_error(str(next(iter(extra))), observation_dict["name"])
|
|
201
|
+
|
|
202
|
+
if any(k in observation_dict for k in ("DATE", "DAYS", "HOURS")):
|
|
203
|
+
bad_key = next(
|
|
204
|
+
k for k in ("DATE", "DAYS", "HOURS") if k in observation_dict
|
|
205
|
+
)
|
|
206
|
+
raise ObservationConfigError.with_context(
|
|
207
|
+
(
|
|
208
|
+
"GENERAL_OBSERVATION must use RESTART to specify "
|
|
209
|
+
"report step. Please run:\n ert convert_observations "
|
|
210
|
+
"<your_ert_config.ert>\nto migrate the observation config "
|
|
211
|
+
"to use the correct format."
|
|
212
|
+
),
|
|
213
|
+
bad_key,
|
|
214
|
+
)
|
|
215
|
+
|
|
216
|
+
if "OBS_FILE" in observation_dict and (
|
|
217
|
+
"VALUE" in observation_dict or "ERROR" in observation_dict
|
|
218
|
+
):
|
|
219
|
+
raise ObservationConfigError.with_context(
|
|
220
|
+
"GENERAL_OBSERVATION cannot contain both VALUE/ERROR and OBS_FILE",
|
|
221
|
+
observation_dict["name"],
|
|
222
|
+
)
|
|
223
|
+
|
|
224
|
+
if "INDEX_FILE" in observation_dict and "INDEX_LIST" in observation_dict:
|
|
211
225
|
raise ObservationConfigError.with_context(
|
|
212
|
-
|
|
213
|
-
|
|
226
|
+
(
|
|
227
|
+
"GENERAL_OBSERVATION "
|
|
228
|
+
f"{observation_dict['name']} has both INDEX_FILE and INDEX_LIST."
|
|
229
|
+
),
|
|
214
230
|
observation_dict["name"],
|
|
215
231
|
)
|
|
216
|
-
return output
|
|
217
232
|
|
|
233
|
+
restart = (
|
|
234
|
+
validate_positive_int(observation_dict["RESTART"], "RESTART")
|
|
235
|
+
if "RESTART" in observation_dict
|
|
236
|
+
else 0
|
|
237
|
+
)
|
|
238
|
+
|
|
239
|
+
if "OBS_FILE" not in observation_dict:
|
|
240
|
+
if "VALUE" in observation_dict and "ERROR" not in observation_dict:
|
|
241
|
+
raise ObservationConfigError.with_context(
|
|
242
|
+
f"For GENERAL_OBSERVATION {observation_dict['name']}, with"
|
|
243
|
+
f" VALUE = {observation_dict['VALUE']}, ERROR must also be given.",
|
|
244
|
+
observation_dict["name"],
|
|
245
|
+
)
|
|
246
|
+
|
|
247
|
+
if "VALUE" not in observation_dict and "ERROR" not in observation_dict:
|
|
248
|
+
raise ObservationConfigError.with_context(
|
|
249
|
+
"GENERAL_OBSERVATION must contain either VALUE "
|
|
250
|
+
"and ERROR or OBS_FILE",
|
|
251
|
+
context=observation_dict["name"],
|
|
252
|
+
)
|
|
253
|
+
|
|
254
|
+
obs_instance = cls(
|
|
255
|
+
name=observation_dict["name"],
|
|
256
|
+
data=data,
|
|
257
|
+
value=validate_float(observation_dict["VALUE"], "VALUE"),
|
|
258
|
+
error=validate_positive_float(
|
|
259
|
+
observation_dict["ERROR"], "ERROR", strictly_positive=True
|
|
260
|
+
),
|
|
261
|
+
restart=restart,
|
|
262
|
+
index=0,
|
|
263
|
+
)
|
|
264
|
+
# Bypass pydantic discarding context
|
|
265
|
+
# only relevant for ERT config surfacing validation errors
|
|
266
|
+
# irrelevant for runmodels etc.
|
|
267
|
+
obs_instance.name = observation_dict["name"]
|
|
268
|
+
return [obs_instance]
|
|
269
|
+
|
|
270
|
+
obs_filename = _resolve_path(directory, observation_dict["OBS_FILE"])
|
|
271
|
+
try:
|
|
272
|
+
file_values = np.loadtxt(obs_filename, delimiter=None).ravel()
|
|
273
|
+
except ValueError as err:
|
|
274
|
+
raise ObservationConfigError.with_context(
|
|
275
|
+
f"Failed to read OBS_FILE {obs_filename}: {err}", obs_filename
|
|
276
|
+
) from err
|
|
277
|
+
if len(file_values) % 2 != 0:
|
|
278
|
+
raise ObservationConfigError.with_context(
|
|
279
|
+
"Expected even number of values in GENERAL_OBSERVATION",
|
|
280
|
+
obs_filename,
|
|
281
|
+
)
|
|
282
|
+
values = file_values[::2]
|
|
283
|
+
stds = file_values[1::2]
|
|
284
|
+
|
|
285
|
+
if "INDEX_FILE" in observation_dict:
|
|
286
|
+
idx_file = _resolve_path(directory, observation_dict["INDEX_FILE"])
|
|
287
|
+
indices = np.loadtxt(idx_file, delimiter=None, dtype=np.int32).ravel()
|
|
288
|
+
elif "INDEX_LIST" in observation_dict:
|
|
289
|
+
indices = np.array(
|
|
290
|
+
sorted(rangestring_to_list(observation_dict["INDEX_LIST"])),
|
|
291
|
+
dtype=np.int32,
|
|
292
|
+
)
|
|
293
|
+
else:
|
|
294
|
+
indices = np.arange(len(values), dtype=np.int32)
|
|
218
295
|
|
|
219
|
-
|
|
220
|
-
|
|
296
|
+
if len({len(stds), len(values), len(indices)}) != 1:
|
|
297
|
+
raise ObservationConfigError.with_context(
|
|
298
|
+
(
|
|
299
|
+
"Values ("
|
|
300
|
+
f"{values}), error ({stds}) and index list ({indices}) "
|
|
301
|
+
"must be of equal length"
|
|
302
|
+
),
|
|
303
|
+
observation_dict["name"],
|
|
304
|
+
)
|
|
305
|
+
|
|
306
|
+
if np.any(stds <= 0):
|
|
307
|
+
raise ObservationConfigError.with_context(
|
|
308
|
+
"Observation uncertainty must be strictly > 0",
|
|
309
|
+
observation_dict["name"],
|
|
310
|
+
)
|
|
311
|
+
|
|
312
|
+
obs_instances: list[Self] = []
|
|
313
|
+
for _pos, (val, std, idx) in enumerate(zip(values, stds, indices, strict=True)):
|
|
314
|
+
# index should reflect the index provided by INDEX_FILE / INDEX_LIST
|
|
315
|
+
inst = cls(
|
|
316
|
+
name=observation_dict["name"],
|
|
317
|
+
data=data,
|
|
318
|
+
value=float(val),
|
|
319
|
+
error=float(std),
|
|
320
|
+
restart=restart,
|
|
321
|
+
index=int(idx),
|
|
322
|
+
)
|
|
323
|
+
# Bypass pydantic discarding context
|
|
324
|
+
# only relevant for ERT config surfacing validation errors
|
|
325
|
+
# irrelevant for runmodels etc.
|
|
326
|
+
inst.name = observation_dict["name"]
|
|
327
|
+
obs_instances.append(inst)
|
|
328
|
+
return obs_instances
|
|
329
|
+
|
|
330
|
+
|
|
331
|
+
class RFTObservation(BaseModel):
|
|
332
|
+
"""Represents an RFT (Repeat Formation Tester) observation.
|
|
333
|
+
|
|
334
|
+
RFT observations are used to condition on pressure, saturation, or other
|
|
335
|
+
properties measured at specific well locations and times.
|
|
336
|
+
"""
|
|
337
|
+
|
|
338
|
+
type: Literal["rft_observation"] = "rft_observation"
|
|
221
339
|
name: str
|
|
222
340
|
well: str
|
|
223
341
|
date: str
|
|
@@ -227,9 +345,131 @@ class RFTObservation:
|
|
|
227
345
|
north: float
|
|
228
346
|
east: float
|
|
229
347
|
tvd: float
|
|
348
|
+
zone: str | None = None
|
|
349
|
+
|
|
350
|
+
@classmethod
|
|
351
|
+
def from_csv(
|
|
352
|
+
cls,
|
|
353
|
+
directory: str,
|
|
354
|
+
observation_dict: ObservationDict,
|
|
355
|
+
filename: str,
|
|
356
|
+
observed_property: str = "PRESSURE",
|
|
357
|
+
) -> list[Self]:
|
|
358
|
+
"""Create RFT observations from a CSV file.
|
|
359
|
+
|
|
360
|
+
The CSV file must contain the following columns: WELL_NAME, DATE,
|
|
361
|
+
ERROR, NORTH, EAST, TVD, and a column for the observed property
|
|
362
|
+
(e.g., PRESSURE, SWAT). An optional ZONE column may also be present.
|
|
363
|
+
|
|
364
|
+
Args:
|
|
365
|
+
directory: Base directory for resolving relative file paths.
|
|
366
|
+
observation_dict: Dictionary containing the observation configuration.
|
|
367
|
+
filename: Path to the CSV file containing RFT observations.
|
|
368
|
+
observed_property: Property to observe (default: PRESSURE).
|
|
369
|
+
|
|
370
|
+
Returns:
|
|
371
|
+
List of RFTObservation instances created from the CSV file.
|
|
372
|
+
|
|
373
|
+
Raises:
|
|
374
|
+
ObservationConfigError: If the file is missing, inaccessible,
|
|
375
|
+
lacks required columns, or contains invalid observation values
|
|
376
|
+
(value=-1 and error=0).
|
|
377
|
+
"""
|
|
378
|
+
if not os.path.isabs(filename):
|
|
379
|
+
filename = os.path.join(directory, filename)
|
|
380
|
+
if not os.path.exists(filename):
|
|
381
|
+
raise ObservationConfigError.with_context(
|
|
382
|
+
f"The CSV file ({filename}) does not exist or is not accessible.",
|
|
383
|
+
filename,
|
|
384
|
+
)
|
|
385
|
+
csv_file = pd.read_csv(
|
|
386
|
+
filename,
|
|
387
|
+
encoding="utf-8",
|
|
388
|
+
on_bad_lines="error",
|
|
389
|
+
)
|
|
390
|
+
|
|
391
|
+
required_columns = {
|
|
392
|
+
"WELL_NAME",
|
|
393
|
+
"DATE",
|
|
394
|
+
observed_property,
|
|
395
|
+
"ERROR",
|
|
396
|
+
"NORTH",
|
|
397
|
+
"EAST",
|
|
398
|
+
"TVD",
|
|
399
|
+
}
|
|
400
|
+
missing_required_columns = required_columns - set(csv_file.keys())
|
|
401
|
+
if missing_required_columns:
|
|
402
|
+
raise ObservationConfigError.with_context(
|
|
403
|
+
f"The rft observations file {filename} is missing required column(s) "
|
|
404
|
+
f"{', '.join(sorted(missing_required_columns))}.",
|
|
405
|
+
filename,
|
|
406
|
+
)
|
|
407
|
+
|
|
408
|
+
rft_observations = []
|
|
409
|
+
invalid_observations = []
|
|
410
|
+
for row in csv_file.itertuples(index=True):
|
|
411
|
+
rft_observation = cls(
|
|
412
|
+
name=f"{observation_dict['name']}[{row.Index}]",
|
|
413
|
+
well=str(row.WELL_NAME),
|
|
414
|
+
date=str(row.DATE),
|
|
415
|
+
property=observed_property,
|
|
416
|
+
value=validate_float(
|
|
417
|
+
str(getattr(row, observed_property)), observed_property
|
|
418
|
+
),
|
|
419
|
+
error=validate_float(str(row.ERROR), "ERROR"),
|
|
420
|
+
north=validate_float(str(row.NORTH), "NORTH"),
|
|
421
|
+
east=validate_float(str(row.EAST), "EAST"),
|
|
422
|
+
tvd=validate_float(str(row.TVD), "TVD"),
|
|
423
|
+
zone=row.ZONE if "ZONE" in csv_file else None,
|
|
424
|
+
)
|
|
425
|
+
# A value of -1 and error of 0 is used by fmu.tools.rms create_rft_ertobs to
|
|
426
|
+
# indicate missing data. If encountered in an rft observations csv file
|
|
427
|
+
# it should raise an error and ask the user to remove invalid observations.
|
|
428
|
+
if rft_observation.value == -1 and rft_observation.error == 0:
|
|
429
|
+
invalid_observations.append(rft_observation)
|
|
430
|
+
else:
|
|
431
|
+
rft_observations.append(rft_observation)
|
|
432
|
+
|
|
433
|
+
if invalid_observations:
|
|
434
|
+
well_list = "\n - ".join(
|
|
435
|
+
[
|
|
436
|
+
f"{observation.well} at date {observation.date}"
|
|
437
|
+
for observation in invalid_observations
|
|
438
|
+
]
|
|
439
|
+
)
|
|
440
|
+
raise ObservationConfigError.with_context(
|
|
441
|
+
(
|
|
442
|
+
f"Invalid value=-1 and error=0 detected in {filename} for "
|
|
443
|
+
f"well(s):\n - {well_list}\n"
|
|
444
|
+
"The invalid observation(s) must be removed from the file."
|
|
445
|
+
),
|
|
446
|
+
filename,
|
|
447
|
+
)
|
|
448
|
+
return rft_observations
|
|
230
449
|
|
|
231
450
|
@classmethod
|
|
232
|
-
def from_obs_dict(
|
|
451
|
+
def from_obs_dict(
|
|
452
|
+
cls, directory: str, observation_dict: ObservationDict
|
|
453
|
+
) -> list[Self]:
|
|
454
|
+
"""Create RFT observations from an observation dictionary.
|
|
455
|
+
|
|
456
|
+
Supports two modes:
|
|
457
|
+
1. CSV mode: Load observations from a CSV file specified by the CSV key.
|
|
458
|
+
2. Direct mode: Create a single observation from individual keys
|
|
459
|
+
(WELL, PROPERTY, VALUE, ERROR, DATE, NORTH, EAST, TVD, ZONE).
|
|
460
|
+
|
|
461
|
+
Args:
|
|
462
|
+
directory: Base directory for resolving relative file paths.
|
|
463
|
+
observation_dict: Dictionary containing the observation configuration.
|
|
464
|
+
|
|
465
|
+
Returns:
|
|
466
|
+
List of RFTObservation instances. Returns multiple observations when
|
|
467
|
+
loading from CSV, or a single observation when using direct mode.
|
|
468
|
+
|
|
469
|
+
Raises:
|
|
470
|
+
ObservationConfigError: If required keys are missing or invalid.
|
|
471
|
+
"""
|
|
472
|
+
csv_filename = None
|
|
233
473
|
well = None
|
|
234
474
|
observed_property = None
|
|
235
475
|
observed_value = None
|
|
@@ -238,6 +478,7 @@ class RFTObservation:
|
|
|
238
478
|
north = None
|
|
239
479
|
east = None
|
|
240
480
|
tvd = None
|
|
481
|
+
zone = None
|
|
241
482
|
for key, value in observation_dict.items():
|
|
242
483
|
match key:
|
|
243
484
|
case "type" | "name":
|
|
@@ -258,8 +499,19 @@ class RFTObservation:
|
|
|
258
499
|
east = validate_float(value, key)
|
|
259
500
|
case "TVD":
|
|
260
501
|
tvd = validate_float(value, key)
|
|
502
|
+
case "CSV":
|
|
503
|
+
csv_filename = value
|
|
504
|
+
case "ZONE":
|
|
505
|
+
zone = value
|
|
261
506
|
case _:
|
|
262
507
|
raise _unknown_key_error(str(key), observation_dict["name"])
|
|
508
|
+
if csv_filename is not None:
|
|
509
|
+
return cls.from_csv(
|
|
510
|
+
directory,
|
|
511
|
+
observation_dict,
|
|
512
|
+
csv_filename,
|
|
513
|
+
observed_property or "PRESSURE",
|
|
514
|
+
)
|
|
263
515
|
if well is None:
|
|
264
516
|
raise _missing_value_error(observation_dict["name"], "WELL")
|
|
265
517
|
if observed_value is None:
|
|
@@ -276,30 +528,106 @@ class RFTObservation:
|
|
|
276
528
|
raise _missing_value_error(observation_dict["name"], "EAST")
|
|
277
529
|
if tvd is None:
|
|
278
530
|
raise _missing_value_error(observation_dict["name"], "TVD")
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
observed_property,
|
|
284
|
-
observed_value,
|
|
285
|
-
error,
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
531
|
+
|
|
532
|
+
obs_instance = cls(
|
|
533
|
+
name=observation_dict["name"],
|
|
534
|
+
well=well,
|
|
535
|
+
property=observed_property,
|
|
536
|
+
value=observed_value,
|
|
537
|
+
error=error,
|
|
538
|
+
date=date,
|
|
539
|
+
north=north,
|
|
540
|
+
east=east,
|
|
541
|
+
tvd=tvd,
|
|
542
|
+
zone=zone,
|
|
289
543
|
)
|
|
290
544
|
|
|
545
|
+
# Bypass pydantic discarding context
|
|
546
|
+
# only relevant for ERT config surfacing validation errors
|
|
547
|
+
# irrelevant for runmodels etc.
|
|
548
|
+
obs_instance.name = observation_dict["name"]
|
|
549
|
+
|
|
550
|
+
return [obs_instance]
|
|
551
|
+
|
|
552
|
+
|
|
553
|
+
class BreakthroughObservation(BaseModel):
|
|
554
|
+
type: Literal["breakthrough"] = "breakthrough"
|
|
555
|
+
name: str
|
|
556
|
+
response_key: str
|
|
557
|
+
date: datetime
|
|
558
|
+
error: float
|
|
559
|
+
threshold: float
|
|
560
|
+
north: float | None
|
|
561
|
+
east: float | None
|
|
562
|
+
radius: float | None
|
|
563
|
+
|
|
564
|
+
@classmethod
|
|
565
|
+
def from_obs_dict(cls, directory: str, obs_dict: ObservationDict) -> list[Self]:
|
|
566
|
+
response_key = None
|
|
567
|
+
date = None
|
|
568
|
+
error = None
|
|
569
|
+
threshold = None
|
|
570
|
+
localization_dict: dict[LOCALIZATION_KEYS, float | None] = {}
|
|
571
|
+
for key, value in obs_dict.items():
|
|
572
|
+
match key:
|
|
573
|
+
case "type" | "name":
|
|
574
|
+
pass
|
|
575
|
+
case "KEY":
|
|
576
|
+
response_key = value
|
|
577
|
+
case "DATE":
|
|
578
|
+
date = value
|
|
579
|
+
case "ERROR":
|
|
580
|
+
error = validate_float(value, key)
|
|
581
|
+
case "THRESHOLD":
|
|
582
|
+
threshold = validate_float(value, key)
|
|
583
|
+
case "LOCALIZATION":
|
|
584
|
+
validate_localization(value, obs_dict["name"])
|
|
585
|
+
extract_localization_values(localization_dict, value, key)
|
|
586
|
+
case _:
|
|
587
|
+
raise _unknown_key_error(str(key), value)
|
|
588
|
+
|
|
589
|
+
if response_key is None:
|
|
590
|
+
raise _missing_value_error(obs_dict["name"], "KEY")
|
|
591
|
+
if date is None:
|
|
592
|
+
raise _missing_value_error(obs_dict["name"], "DATE")
|
|
593
|
+
if error is None:
|
|
594
|
+
raise _missing_value_error(obs_dict["name"], "ERROR")
|
|
595
|
+
if threshold is None:
|
|
596
|
+
raise _missing_value_error(obs_dict["name"], "THRESHOLD")
|
|
597
|
+
|
|
598
|
+
return [
|
|
599
|
+
cls(
|
|
600
|
+
name=obs_dict["name"],
|
|
601
|
+
response_key=response_key,
|
|
602
|
+
date=date,
|
|
603
|
+
error=error,
|
|
604
|
+
threshold=threshold,
|
|
605
|
+
north=localization_dict.get("north"),
|
|
606
|
+
east=localization_dict.get("east"),
|
|
607
|
+
radius=localization_dict.get("radius"),
|
|
608
|
+
)
|
|
609
|
+
]
|
|
291
610
|
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
611
|
+
|
|
612
|
+
Observation = Annotated[
|
|
613
|
+
(
|
|
614
|
+
SummaryObservation
|
|
615
|
+
| GeneralObservation
|
|
616
|
+
| RFTObservation
|
|
617
|
+
| BreakthroughObservation
|
|
618
|
+
),
|
|
619
|
+
Field(discriminator="type"),
|
|
620
|
+
]
|
|
295
621
|
|
|
296
622
|
_TYPE_TO_CLASS: dict[ObservationType, type[Observation]] = {
|
|
297
|
-
ObservationType.HISTORY: HistoryObservation,
|
|
298
623
|
ObservationType.SUMMARY: SummaryObservation,
|
|
299
624
|
ObservationType.GENERAL: GeneralObservation,
|
|
300
625
|
ObservationType.RFT: RFTObservation,
|
|
626
|
+
ObservationType.BREAKTHROUGH: BreakthroughObservation,
|
|
301
627
|
}
|
|
302
628
|
|
|
629
|
+
LOCALIZATION_KEYS = Literal["east", "north", "radius"]
|
|
630
|
+
|
|
303
631
|
|
|
304
632
|
def make_observations(
|
|
305
633
|
directory: str, observation_dicts: Sequence[ObservationDict]
|
|
@@ -312,10 +640,20 @@ def make_observations(
|
|
|
312
640
|
inp: The collection of statements to validate.
|
|
313
641
|
"""
|
|
314
642
|
result: list[Observation] = []
|
|
315
|
-
error_list: list[ErrorInfo] = []
|
|
643
|
+
error_list: list[ErrorInfo | ObservationConfigError] = []
|
|
316
644
|
for obs_dict in observation_dicts:
|
|
645
|
+
if obs_dict["type"] == ObservationType.HISTORY:
|
|
646
|
+
msg = (
|
|
647
|
+
"HISTORY_OBSERVATION is deprecated, and must be specified "
|
|
648
|
+
"as SUMMARY_OBSERVATION. Run"
|
|
649
|
+
" ert convert_observations <ert_config.ert> to convert your "
|
|
650
|
+
"observations automatically"
|
|
651
|
+
)
|
|
652
|
+
logger.error(msg)
|
|
653
|
+
error_list.append(ObservationConfigError(msg))
|
|
654
|
+
continue
|
|
317
655
|
try:
|
|
318
|
-
result.
|
|
656
|
+
result.extend(
|
|
319
657
|
_TYPE_TO_CLASS[obs_dict["type"]].from_obs_dict(directory, obs_dict)
|
|
320
658
|
)
|
|
321
659
|
except KeyError as err:
|
|
@@ -329,41 +667,6 @@ def make_observations(
|
|
|
329
667
|
return result
|
|
330
668
|
|
|
331
669
|
|
|
332
|
-
def _validate_segment_dict(name_token: str, inp: dict[str, Any]) -> Segment:
|
|
333
|
-
start = None
|
|
334
|
-
stop = None
|
|
335
|
-
error_mode = ErrorModes.RELMIN
|
|
336
|
-
error = 0.1
|
|
337
|
-
error_min = 0.1
|
|
338
|
-
for key, value in inp.items():
|
|
339
|
-
match key:
|
|
340
|
-
case "START":
|
|
341
|
-
start = validate_int(value, key)
|
|
342
|
-
case "STOP":
|
|
343
|
-
stop = validate_int(value, key)
|
|
344
|
-
case "ERROR":
|
|
345
|
-
error = validate_positive_float(value, key)
|
|
346
|
-
case "ERROR_MIN":
|
|
347
|
-
error_min = validate_positive_float(value, key)
|
|
348
|
-
case "ERROR_MODE":
|
|
349
|
-
error_mode = validate_error_mode(value)
|
|
350
|
-
case _:
|
|
351
|
-
raise _unknown_key_error(key, name_token)
|
|
352
|
-
|
|
353
|
-
if start is None:
|
|
354
|
-
raise _missing_value_error(name_token, "START")
|
|
355
|
-
if stop is None:
|
|
356
|
-
raise _missing_value_error(name_token, "STOP")
|
|
357
|
-
return Segment(
|
|
358
|
-
name=name_token,
|
|
359
|
-
start=start,
|
|
360
|
-
stop=stop,
|
|
361
|
-
error_mode=error_mode,
|
|
362
|
-
error=error,
|
|
363
|
-
error_min=error_min,
|
|
364
|
-
)
|
|
365
|
-
|
|
366
|
-
|
|
367
670
|
def validate_error_mode(inp: str) -> ErrorModes:
|
|
368
671
|
if inp == "REL":
|
|
369
672
|
return ErrorModes.REL
|
|
@@ -390,12 +693,15 @@ def validate_int(val: str, key: str) -> int:
|
|
|
390
693
|
raise _conversion_error(key, val, "int") from err
|
|
391
694
|
|
|
392
695
|
|
|
393
|
-
def validate_positive_float(
|
|
696
|
+
def validate_positive_float(
|
|
697
|
+
val: str, key: str, strictly_positive: bool = False
|
|
698
|
+
) -> float:
|
|
394
699
|
v = validate_float(val, key)
|
|
395
|
-
if v < 0:
|
|
700
|
+
if v < 0 or (v <= 0 and strictly_positive):
|
|
396
701
|
raise ObservationConfigError.with_context(
|
|
397
702
|
f'Failed to validate "{val}" in {key}={val}.'
|
|
398
|
-
f" {key} must be given a
|
|
703
|
+
f" {key} must be given a "
|
|
704
|
+
f"{'strictly ' if strictly_positive else ''}positive value.",
|
|
399
705
|
val,
|
|
400
706
|
)
|
|
401
707
|
return v
|
|
@@ -414,6 +720,16 @@ def validate_localization(val: dict[str, Any], obs_name: str) -> None:
|
|
|
414
720
|
raise ObservationConfigError.from_collected(errors)
|
|
415
721
|
|
|
416
722
|
|
|
723
|
+
def extract_localization_values(
|
|
724
|
+
localization_dict: dict[LOCALIZATION_KEYS, float | None], value: Any, key: str
|
|
725
|
+
) -> None:
|
|
726
|
+
localization_dict["east"] = validate_float(value["EAST"], key)
|
|
727
|
+
localization_dict["north"] = validate_float(value["NORTH"], key)
|
|
728
|
+
localization_dict["radius"] = (
|
|
729
|
+
validate_float(value["RADIUS"], key) if "RADIUS" in value else None
|
|
730
|
+
)
|
|
731
|
+
|
|
732
|
+
|
|
417
733
|
def validate_positive_int(val: str, key: str) -> int:
|
|
418
734
|
try:
|
|
419
735
|
v = int(val)
|
|
@@ -434,6 +750,17 @@ def _missing_value_error(name_token: str, value_key: str) -> ObservationConfigEr
|
|
|
434
750
|
)
|
|
435
751
|
|
|
436
752
|
|
|
753
|
+
def _resolve_path(directory: str, filename: str) -> str:
|
|
754
|
+
if not os.path.isabs(filename):
|
|
755
|
+
filename = os.path.join(directory, filename)
|
|
756
|
+
if not os.path.exists(filename):
|
|
757
|
+
raise ObservationConfigError.with_context(
|
|
758
|
+
f"The following keywords did not resolve to a valid path:\n {filename}",
|
|
759
|
+
filename,
|
|
760
|
+
)
|
|
761
|
+
return filename
|
|
762
|
+
|
|
763
|
+
|
|
437
764
|
def _conversion_error(token: str, value: Any, type_name: str) -> ObservationConfigError:
|
|
438
765
|
return ObservationConfigError.with_context(
|
|
439
766
|
f'Could not convert {value} to {type_name}. Failed to validate "{value}"',
|