ert 19.0.0rc4__py3-none-any.whl → 20.0.0b0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ert/__main__.py +94 -63
- ert/analysis/_es_update.py +11 -14
- ert/config/__init__.py +3 -2
- ert/config/_create_observation_dataframes.py +51 -375
- ert/config/_observations.py +483 -200
- ert/config/_read_summary.py +4 -5
- ert/config/ert_config.py +53 -80
- ert/config/everest_control.py +40 -39
- ert/config/everest_response.py +1 -13
- ert/config/field.py +0 -72
- ert/config/forward_model_step.py +17 -1
- ert/config/gen_data_config.py +14 -17
- ert/config/observation_config_migrations.py +821 -0
- ert/config/parameter_config.py +18 -28
- ert/config/parsing/__init__.py +0 -1
- ert/config/parsing/_parse_zonemap.py +45 -0
- ert/config/parsing/config_keywords.py +1 -1
- ert/config/parsing/config_schema.py +2 -8
- ert/config/parsing/observations_parser.py +2 -0
- ert/config/response_config.py +5 -23
- ert/config/rft_config.py +44 -19
- ert/config/summary_config.py +1 -13
- ert/config/surface_config.py +0 -57
- ert/dark_storage/compute/misfits.py +0 -42
- ert/dark_storage/endpoints/__init__.py +0 -2
- ert/dark_storage/endpoints/experiments.py +2 -5
- ert/dark_storage/json_schema/experiment.py +1 -2
- ert/field_utils/__init__.py +0 -2
- ert/field_utils/field_utils.py +1 -117
- ert/gui/ertwidgets/listeditbox.py +9 -1
- ert/gui/ertwidgets/models/ertsummary.py +20 -6
- ert/gui/ertwidgets/pathchooser.py +9 -1
- ert/gui/ertwidgets/stringbox.py +11 -3
- ert/gui/ertwidgets/textbox.py +10 -3
- ert/gui/ertwidgets/validationsupport.py +19 -1
- ert/gui/main_window.py +11 -6
- ert/gui/simulation/experiment_panel.py +1 -1
- ert/gui/simulation/run_dialog.py +11 -1
- ert/gui/tools/manage_experiments/export_dialog.py +4 -0
- ert/gui/tools/manage_experiments/manage_experiments_panel.py +1 -0
- ert/gui/tools/manage_experiments/storage_info_widget.py +5 -2
- ert/gui/tools/manage_experiments/storage_widget.py +18 -3
- ert/gui/tools/plot/data_type_proxy_model.py +1 -1
- ert/gui/tools/plot/plot_api.py +35 -27
- ert/gui/tools/plot/plot_widget.py +5 -0
- ert/gui/tools/plot/plot_window.py +4 -7
- ert/run_models/ensemble_experiment.py +1 -3
- ert/run_models/ensemble_smoother.py +1 -3
- ert/run_models/everest_run_model.py +12 -13
- ert/run_models/initial_ensemble_run_model.py +19 -22
- ert/run_models/model_factory.py +7 -7
- ert/run_models/multiple_data_assimilation.py +1 -3
- ert/sample_prior.py +12 -14
- ert/services/__init__.py +7 -3
- ert/services/_storage_main.py +59 -22
- ert/services/ert_server.py +186 -24
- ert/shared/version.py +3 -3
- ert/storage/local_ensemble.py +46 -115
- ert/storage/local_experiment.py +0 -16
- ert/utils/__init__.py +20 -0
- ert/warnings/specific_warning_handler.py +3 -2
- {ert-19.0.0rc4.dist-info → ert-20.0.0b0.dist-info}/METADATA +4 -51
- {ert-19.0.0rc4.dist-info → ert-20.0.0b0.dist-info}/RECORD +75 -80
- everest/bin/everest_script.py +5 -5
- everest/bin/kill_script.py +2 -2
- everest/bin/monitor_script.py +2 -2
- everest/bin/utils.py +4 -4
- everest/detached/everserver.py +6 -6
- everest/gui/everest_client.py +0 -6
- everest/gui/main_window.py +2 -2
- everest/util/__init__.py +1 -19
- ert/dark_storage/compute/__init__.py +0 -0
- ert/dark_storage/endpoints/compute/__init__.py +0 -0
- ert/dark_storage/endpoints/compute/misfits.py +0 -95
- ert/services/_base_service.py +0 -387
- ert/services/webviz_ert_service.py +0 -20
- ert/shared/storage/command.py +0 -38
- ert/shared/storage/extraction.py +0 -42
- {ert-19.0.0rc4.dist-info → ert-20.0.0b0.dist-info}/WHEEL +0 -0
- {ert-19.0.0rc4.dist-info → ert-20.0.0b0.dist-info}/entry_points.txt +0 -0
- {ert-19.0.0rc4.dist-info → ert-20.0.0b0.dist-info}/licenses/COPYING +0 -0
- {ert-19.0.0rc4.dist-info → ert-20.0.0b0.dist-info}/top_level.txt +0 -0
ert/config/_observations.py
CHANGED
|
@@ -1,17 +1,27 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import logging
|
|
1
4
|
import os
|
|
2
5
|
from collections.abc import Sequence
|
|
3
|
-
from
|
|
6
|
+
from datetime import datetime
|
|
4
7
|
from enum import StrEnum
|
|
5
|
-
from
|
|
6
|
-
|
|
8
|
+
from typing import Annotated, Any, Literal, Self, assert_never
|
|
9
|
+
|
|
10
|
+
import numpy as np
|
|
11
|
+
import pandas as pd
|
|
12
|
+
from pydantic import BaseModel, Field
|
|
7
13
|
|
|
14
|
+
from ..validation import rangestring_to_list
|
|
8
15
|
from .parsing import (
|
|
16
|
+
ConfigWarning,
|
|
9
17
|
ErrorInfo,
|
|
10
18
|
ObservationConfigError,
|
|
11
19
|
ObservationDict,
|
|
12
20
|
ObservationType,
|
|
13
21
|
)
|
|
14
22
|
|
|
23
|
+
logger = logging.getLogger(__name__)
|
|
24
|
+
|
|
15
25
|
|
|
16
26
|
class ErrorModes(StrEnum):
|
|
17
27
|
REL = "REL"
|
|
@@ -19,102 +29,67 @@ class ErrorModes(StrEnum):
|
|
|
19
29
|
RELMIN = "RELMIN"
|
|
20
30
|
|
|
21
31
|
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
error_mode: ErrorModes
|
|
25
|
-
error: float
|
|
26
|
-
error_min: float
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
@dataclass
|
|
30
|
-
class Segment(ObservationError):
|
|
31
|
-
name: str
|
|
32
|
-
start: int
|
|
33
|
-
stop: int
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
@dataclass
|
|
37
|
-
class HistoryObservation(ObservationError):
|
|
38
|
-
name: str
|
|
39
|
-
segments: list[Segment]
|
|
40
|
-
|
|
41
|
-
@property
|
|
42
|
-
def key(self) -> str:
|
|
43
|
-
"""The :term:`summary key` to be fetched from :ref:`refcase`."""
|
|
44
|
-
# For history observations the key is also the name, ie.
|
|
45
|
-
# "HISTORY_OBSERVATION FOPR" means to add the values from
|
|
46
|
-
# the summary vector FOPRH in refcase as observations.
|
|
47
|
-
return self.name
|
|
48
|
-
|
|
49
|
-
@classmethod
|
|
50
|
-
def from_obs_dict(cls, directory: str, observation_dict: ObservationDict) -> Self:
|
|
51
|
-
error_mode = ErrorModes.RELMIN
|
|
52
|
-
error = 0.1
|
|
53
|
-
error_min = 0.1
|
|
54
|
-
segments = []
|
|
55
|
-
for key, value in observation_dict.items():
|
|
56
|
-
match key:
|
|
57
|
-
case "type" | "name":
|
|
58
|
-
pass
|
|
59
|
-
case "ERROR":
|
|
60
|
-
error = validate_positive_float(value, key)
|
|
61
|
-
case "ERROR_MIN":
|
|
62
|
-
error_min = validate_positive_float(value, key)
|
|
63
|
-
case "ERROR_MODE":
|
|
64
|
-
error_mode = validate_error_mode(value)
|
|
65
|
-
case "segments":
|
|
66
|
-
segments = list(starmap(_validate_segment_dict, value))
|
|
67
|
-
case _:
|
|
68
|
-
raise _unknown_key_error(str(key), observation_dict["name"])
|
|
69
|
-
|
|
70
|
-
return cls(
|
|
71
|
-
name=observation_dict["name"],
|
|
72
|
-
error_mode=error_mode,
|
|
73
|
-
error=error,
|
|
74
|
-
error_min=error_min,
|
|
75
|
-
segments=segments,
|
|
76
|
-
)
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
@dataclass
|
|
80
|
-
class ObservationDate:
|
|
81
|
-
days: float | None = None
|
|
82
|
-
hours: float | None = None
|
|
83
|
-
date: str | None = None
|
|
84
|
-
restart: int | None = None
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
@dataclass
|
|
88
|
-
class _SummaryValues:
|
|
32
|
+
class _SummaryValues(BaseModel):
|
|
33
|
+
type: Literal["summary_observation"] = "summary_observation"
|
|
89
34
|
name: str
|
|
90
35
|
value: float
|
|
36
|
+
error: float
|
|
91
37
|
key: str #: The :term:`summary key` in the summary response
|
|
38
|
+
date: str
|
|
92
39
|
east: float | None = None
|
|
93
40
|
north: float | None = None
|
|
94
41
|
radius: float | None = None
|
|
95
42
|
|
|
96
43
|
|
|
97
|
-
|
|
98
|
-
|
|
44
|
+
def _parse_date(date_str: str) -> datetime:
|
|
45
|
+
try:
|
|
46
|
+
return datetime.fromisoformat(date_str)
|
|
47
|
+
except ValueError:
|
|
48
|
+
try:
|
|
49
|
+
date = datetime.strptime(date_str, "%d/%m/%Y")
|
|
50
|
+
except ValueError as err:
|
|
51
|
+
raise ObservationConfigError.with_context(
|
|
52
|
+
f"Unsupported date format {date_str}. Please use ISO date format",
|
|
53
|
+
date_str,
|
|
54
|
+
) from err
|
|
55
|
+
else:
|
|
56
|
+
ConfigWarning.warn(
|
|
57
|
+
f"Deprecated time format {date_str}."
|
|
58
|
+
" Please use ISO date format YYYY-MM-DD",
|
|
59
|
+
date_str,
|
|
60
|
+
)
|
|
61
|
+
return date
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
class SummaryObservation(_SummaryValues):
|
|
99
65
|
@classmethod
|
|
100
|
-
def from_obs_dict(
|
|
66
|
+
def from_obs_dict(
|
|
67
|
+
cls, directory: str, observation_dict: ObservationDict
|
|
68
|
+
) -> list[Self]:
|
|
101
69
|
error_mode = ErrorModes.ABS
|
|
102
70
|
summary_key = None
|
|
103
71
|
|
|
104
|
-
|
|
72
|
+
date: str | None = None
|
|
105
73
|
float_values: dict[str, float] = {"ERROR_MIN": 0.1}
|
|
106
|
-
|
|
74
|
+
localization_dict: dict[LOCALIZATION_KEYS, float | None] = {}
|
|
107
75
|
for key, value in observation_dict.items():
|
|
108
76
|
match key:
|
|
109
77
|
case "type" | "name":
|
|
110
78
|
pass
|
|
111
|
-
case "RESTART":
|
|
112
|
-
date_dict.restart = validate_positive_int(value, key)
|
|
113
79
|
case "ERROR" | "ERROR_MIN":
|
|
114
|
-
float_values[str(key)] = validate_positive_float(
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
80
|
+
float_values[str(key)] = validate_positive_float(
|
|
81
|
+
value, key, strictly_positive=True
|
|
82
|
+
)
|
|
83
|
+
case "DAYS" | "HOURS" | "RESTART":
|
|
84
|
+
raise ObservationConfigError.with_context(
|
|
85
|
+
(
|
|
86
|
+
"SUMMARY_OBSERVATION must use DATE to specify "
|
|
87
|
+
"date, DAYS | HOURS is no longer allowed. "
|
|
88
|
+
"Please run:\n ert convert_observations "
|
|
89
|
+
"<your_ert_config.ert>\nto migrate the observation config "
|
|
90
|
+
"to use the correct format."
|
|
91
|
+
),
|
|
92
|
+
key,
|
|
118
93
|
)
|
|
119
94
|
case "VALUE":
|
|
120
95
|
float_values[str(key)] = validate_float(value, key)
|
|
@@ -123,16 +98,10 @@ class SummaryObservation(ObservationDate, _SummaryValues, ObservationError):
|
|
|
123
98
|
case "KEY":
|
|
124
99
|
summary_key = value
|
|
125
100
|
case "DATE":
|
|
126
|
-
|
|
101
|
+
date = value
|
|
127
102
|
case "LOCALIZATION":
|
|
128
103
|
validate_localization(value, observation_dict["name"])
|
|
129
|
-
|
|
130
|
-
localization_values["north"] = validate_float(value["NORTH"], key)
|
|
131
|
-
localization_values["radius"] = (
|
|
132
|
-
validate_float(value["RADIUS"], key)
|
|
133
|
-
if "RADIUS" in value
|
|
134
|
-
else None
|
|
135
|
-
)
|
|
104
|
+
extract_localization_values(localization_dict, value, key)
|
|
136
105
|
case _:
|
|
137
106
|
raise _unknown_key_error(str(key), observation_dict["name"])
|
|
138
107
|
if "VALUE" not in float_values:
|
|
@@ -142,82 +111,225 @@ class SummaryObservation(ObservationDate, _SummaryValues, ObservationError):
|
|
|
142
111
|
if "ERROR" not in float_values:
|
|
143
112
|
raise _missing_value_error(observation_dict["name"], "ERROR")
|
|
144
113
|
|
|
145
|
-
|
|
114
|
+
assert date is not None
|
|
115
|
+
# Raise errors if the date is off
|
|
116
|
+
parsed_date: datetime = _parse_date(date)
|
|
117
|
+
standardized_date = parsed_date.date().isoformat()
|
|
118
|
+
|
|
119
|
+
value = float_values["VALUE"]
|
|
120
|
+
input_error = float_values["ERROR"]
|
|
121
|
+
error_min = float_values["ERROR_MIN"]
|
|
122
|
+
|
|
123
|
+
error = input_error
|
|
124
|
+
match error_mode:
|
|
125
|
+
case ErrorModes.ABS:
|
|
126
|
+
error = validate_positive_float(
|
|
127
|
+
np.abs(input_error), summary_key, strictly_positive=True
|
|
128
|
+
)
|
|
129
|
+
case ErrorModes.REL:
|
|
130
|
+
error = validate_positive_float(
|
|
131
|
+
np.abs(value) * input_error, summary_key, strictly_positive=True
|
|
132
|
+
)
|
|
133
|
+
case ErrorModes.RELMIN:
|
|
134
|
+
error = validate_positive_float(
|
|
135
|
+
np.maximum(np.abs(value) * input_error, error_min),
|
|
136
|
+
summary_key,
|
|
137
|
+
strictly_positive=True,
|
|
138
|
+
)
|
|
139
|
+
case default:
|
|
140
|
+
assert_never(default)
|
|
141
|
+
|
|
142
|
+
obs_instance = cls(
|
|
146
143
|
name=observation_dict["name"],
|
|
147
|
-
|
|
148
|
-
error=float_values["ERROR"],
|
|
149
|
-
error_min=float_values["ERROR_MIN"],
|
|
144
|
+
error=error,
|
|
150
145
|
key=summary_key,
|
|
151
|
-
value=
|
|
152
|
-
east=
|
|
153
|
-
north=
|
|
154
|
-
radius=
|
|
155
|
-
|
|
146
|
+
value=value,
|
|
147
|
+
east=localization_dict.get("east"),
|
|
148
|
+
north=localization_dict.get("north"),
|
|
149
|
+
radius=localization_dict.get("radius"),
|
|
150
|
+
date=standardized_date,
|
|
156
151
|
)
|
|
152
|
+
# Bypass pydantic discarding context
|
|
153
|
+
# only relevant for ERT config surfacing validation errors
|
|
154
|
+
# irrelevant for runmodels etc.
|
|
155
|
+
obs_instance.name = observation_dict["name"]
|
|
156
|
+
|
|
157
|
+
return [obs_instance]
|
|
157
158
|
|
|
158
159
|
|
|
159
|
-
|
|
160
|
-
|
|
160
|
+
class _GeneralObservation(BaseModel):
|
|
161
|
+
type: Literal["general_observation"] = "general_observation"
|
|
161
162
|
name: str
|
|
162
163
|
data: str
|
|
163
|
-
value: float
|
|
164
|
-
error: float
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
164
|
+
value: float
|
|
165
|
+
error: float
|
|
166
|
+
restart: int
|
|
167
|
+
index: int
|
|
168
|
+
east: float | None = None
|
|
169
|
+
north: float | None = None
|
|
170
|
+
radius: float | None = None
|
|
168
171
|
|
|
169
172
|
|
|
170
|
-
|
|
171
|
-
class GeneralObservation(ObservationDate, _GeneralObservation):
|
|
173
|
+
class GeneralObservation(_GeneralObservation):
|
|
172
174
|
@classmethod
|
|
173
|
-
def from_obs_dict(
|
|
175
|
+
def from_obs_dict(
|
|
176
|
+
cls, directory: str, observation_dict: ObservationDict
|
|
177
|
+
) -> list[Self]:
|
|
174
178
|
try:
|
|
175
179
|
data = observation_dict["DATA"]
|
|
176
180
|
except KeyError as err:
|
|
177
181
|
raise _missing_value_error(observation_dict["name"], "DATA") from err
|
|
178
182
|
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
183
|
+
allowed = {
|
|
184
|
+
"type",
|
|
185
|
+
"name",
|
|
186
|
+
"RESTART",
|
|
187
|
+
"VALUE",
|
|
188
|
+
"ERROR",
|
|
189
|
+
"DATE",
|
|
190
|
+
"DAYS",
|
|
191
|
+
"HOURS",
|
|
192
|
+
"INDEX_LIST",
|
|
193
|
+
"OBS_FILE",
|
|
194
|
+
"INDEX_FILE",
|
|
195
|
+
"DATA",
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
extra = set(observation_dict.keys()) - allowed
|
|
199
|
+
if extra:
|
|
200
|
+
raise _unknown_key_error(str(next(iter(extra))), observation_dict["name"])
|
|
201
|
+
|
|
202
|
+
if any(k in observation_dict for k in ("DATE", "DAYS", "HOURS")):
|
|
203
|
+
bad_key = next(
|
|
204
|
+
k for k in ("DATE", "DAYS", "HOURS") if k in observation_dict
|
|
205
|
+
)
|
|
206
|
+
raise ObservationConfigError.with_context(
|
|
207
|
+
(
|
|
208
|
+
"GENERAL_OBSERVATION must use RESTART to specify "
|
|
209
|
+
"report step. Please run:\n ert convert_observations "
|
|
210
|
+
"<your_ert_config.ert>\nto migrate the observation config "
|
|
211
|
+
"to use the correct format."
|
|
212
|
+
),
|
|
213
|
+
bad_key,
|
|
214
|
+
)
|
|
215
|
+
|
|
216
|
+
if "OBS_FILE" in observation_dict and (
|
|
217
|
+
"VALUE" in observation_dict or "ERROR" in observation_dict
|
|
218
|
+
):
|
|
219
|
+
raise ObservationConfigError.with_context(
|
|
220
|
+
"GENERAL_OBSERVATION cannot contain both VALUE/ERROR and OBS_FILE",
|
|
221
|
+
observation_dict["name"],
|
|
222
|
+
)
|
|
223
|
+
|
|
224
|
+
if "INDEX_FILE" in observation_dict and "INDEX_LIST" in observation_dict:
|
|
225
|
+
raise ObservationConfigError.with_context(
|
|
226
|
+
(
|
|
227
|
+
"GENERAL_OBSERVATION "
|
|
228
|
+
f"{observation_dict['name']} has both INDEX_FILE and INDEX_LIST."
|
|
229
|
+
),
|
|
230
|
+
observation_dict["name"],
|
|
231
|
+
)
|
|
232
|
+
|
|
233
|
+
restart = (
|
|
234
|
+
validate_positive_int(observation_dict["RESTART"], "RESTART")
|
|
235
|
+
if "RESTART" in observation_dict
|
|
236
|
+
else 0
|
|
237
|
+
)
|
|
238
|
+
|
|
239
|
+
if "OBS_FILE" not in observation_dict:
|
|
240
|
+
if "VALUE" in observation_dict and "ERROR" not in observation_dict:
|
|
241
|
+
raise ObservationConfigError.with_context(
|
|
242
|
+
f"For GENERAL_OBSERVATION {observation_dict['name']}, with"
|
|
243
|
+
f" VALUE = {observation_dict['VALUE']}, ERROR must also be given.",
|
|
244
|
+
observation_dict["name"],
|
|
245
|
+
)
|
|
246
|
+
|
|
247
|
+
if "VALUE" not in observation_dict and "ERROR" not in observation_dict:
|
|
248
|
+
raise ObservationConfigError.with_context(
|
|
249
|
+
"GENERAL_OBSERVATION must contain either VALUE "
|
|
250
|
+
"and ERROR or OBS_FILE",
|
|
251
|
+
context=observation_dict["name"],
|
|
252
|
+
)
|
|
253
|
+
|
|
254
|
+
obs_instance = cls(
|
|
255
|
+
name=observation_dict["name"],
|
|
256
|
+
data=data,
|
|
257
|
+
value=validate_float(observation_dict["VALUE"], "VALUE"),
|
|
258
|
+
error=validate_positive_float(
|
|
259
|
+
observation_dict["ERROR"], "ERROR", strictly_positive=True
|
|
260
|
+
),
|
|
261
|
+
restart=restart,
|
|
262
|
+
index=0,
|
|
263
|
+
)
|
|
264
|
+
# Bypass pydantic discarding context
|
|
265
|
+
# only relevant for ERT config surfacing validation errors
|
|
266
|
+
# irrelevant for runmodels etc.
|
|
267
|
+
obs_instance.name = observation_dict["name"]
|
|
268
|
+
return [obs_instance]
|
|
269
|
+
|
|
270
|
+
obs_filename = _resolve_path(directory, observation_dict["OBS_FILE"])
|
|
271
|
+
try:
|
|
272
|
+
file_values = np.loadtxt(obs_filename, delimiter=None).ravel()
|
|
273
|
+
except ValueError as err:
|
|
274
|
+
raise ObservationConfigError.with_context(
|
|
275
|
+
f"Failed to read OBS_FILE {obs_filename}: {err}", obs_filename
|
|
276
|
+
) from err
|
|
277
|
+
if len(file_values) % 2 != 0:
|
|
278
|
+
raise ObservationConfigError.with_context(
|
|
279
|
+
"Expected even number of values in GENERAL_OBSERVATION",
|
|
280
|
+
obs_filename,
|
|
281
|
+
)
|
|
282
|
+
values = file_values[::2]
|
|
283
|
+
stds = file_values[1::2]
|
|
284
|
+
|
|
285
|
+
if "INDEX_FILE" in observation_dict:
|
|
286
|
+
idx_file = _resolve_path(directory, observation_dict["INDEX_FILE"])
|
|
287
|
+
indices = np.loadtxt(idx_file, delimiter=None, dtype=np.int32).ravel()
|
|
288
|
+
elif "INDEX_LIST" in observation_dict:
|
|
289
|
+
indices = np.array(
|
|
290
|
+
sorted(rangestring_to_list(observation_dict["INDEX_LIST"])),
|
|
291
|
+
dtype=np.int32,
|
|
292
|
+
)
|
|
293
|
+
else:
|
|
294
|
+
indices = np.arange(len(values), dtype=np.int32)
|
|
295
|
+
|
|
296
|
+
if len({len(stds), len(values), len(indices)}) != 1:
|
|
297
|
+
raise ObservationConfigError.with_context(
|
|
298
|
+
(
|
|
299
|
+
"Values ("
|
|
300
|
+
f"{values}), error ({stds}) and index list ({indices}) "
|
|
301
|
+
"must be of equal length"
|
|
302
|
+
),
|
|
303
|
+
observation_dict["name"],
|
|
304
|
+
)
|
|
305
|
+
|
|
306
|
+
if np.any(stds <= 0):
|
|
211
307
|
raise ObservationConfigError.with_context(
|
|
212
|
-
|
|
213
|
-
f" VALUE = {output.value}, ERROR must also be given.",
|
|
308
|
+
"Observation uncertainty must be strictly > 0",
|
|
214
309
|
observation_dict["name"],
|
|
215
310
|
)
|
|
216
|
-
return output
|
|
217
311
|
|
|
312
|
+
obs_instances: list[Self] = []
|
|
313
|
+
for _pos, (val, std, idx) in enumerate(zip(values, stds, indices, strict=True)):
|
|
314
|
+
# index should reflect the index provided by INDEX_FILE / INDEX_LIST
|
|
315
|
+
inst = cls(
|
|
316
|
+
name=observation_dict["name"],
|
|
317
|
+
data=data,
|
|
318
|
+
value=float(val),
|
|
319
|
+
error=float(std),
|
|
320
|
+
restart=restart,
|
|
321
|
+
index=int(idx),
|
|
322
|
+
)
|
|
323
|
+
# Bypass pydantic discarding context
|
|
324
|
+
# only relevant for ERT config surfacing validation errors
|
|
325
|
+
# irrelevant for runmodels etc.
|
|
326
|
+
inst.name = observation_dict["name"]
|
|
327
|
+
obs_instances.append(inst)
|
|
328
|
+
return obs_instances
|
|
218
329
|
|
|
219
|
-
|
|
220
|
-
class RFTObservation:
|
|
330
|
+
|
|
331
|
+
class RFTObservation(BaseModel):
|
|
332
|
+
type: Literal["rft_observation"] = "rft_observation"
|
|
221
333
|
name: str
|
|
222
334
|
well: str
|
|
223
335
|
date: str
|
|
@@ -227,9 +339,93 @@ class RFTObservation:
|
|
|
227
339
|
north: float
|
|
228
340
|
east: float
|
|
229
341
|
tvd: float
|
|
342
|
+
zone: str | None = None
|
|
343
|
+
|
|
344
|
+
@classmethod
|
|
345
|
+
def from_csv(
|
|
346
|
+
cls,
|
|
347
|
+
directory: str,
|
|
348
|
+
observation_dict: ObservationDict,
|
|
349
|
+
filename: str,
|
|
350
|
+
observed_property: str = "PRESSURE",
|
|
351
|
+
) -> list[Self]:
|
|
352
|
+
if not os.path.isabs(filename):
|
|
353
|
+
filename = os.path.join(directory, filename)
|
|
354
|
+
if not os.path.exists(filename):
|
|
355
|
+
raise ObservationConfigError.with_context(
|
|
356
|
+
f"The CSV file ({filename}) does not exist or is not accessible.",
|
|
357
|
+
filename,
|
|
358
|
+
)
|
|
359
|
+
csv_file = pd.read_csv(
|
|
360
|
+
filename,
|
|
361
|
+
encoding="utf-8",
|
|
362
|
+
on_bad_lines="error",
|
|
363
|
+
)
|
|
364
|
+
|
|
365
|
+
required_columns = {
|
|
366
|
+
"WELL_NAME",
|
|
367
|
+
"DATE",
|
|
368
|
+
observed_property,
|
|
369
|
+
"ERROR",
|
|
370
|
+
"NORTH",
|
|
371
|
+
"EAST",
|
|
372
|
+
"TVD",
|
|
373
|
+
}
|
|
374
|
+
missing_required_columns = required_columns - set(csv_file.keys())
|
|
375
|
+
if missing_required_columns:
|
|
376
|
+
raise ObservationConfigError.with_context(
|
|
377
|
+
f"The rft observations file {filename} is missing required column(s) "
|
|
378
|
+
f"{', '.join(sorted(missing_required_columns))}.",
|
|
379
|
+
filename,
|
|
380
|
+
)
|
|
381
|
+
|
|
382
|
+
rft_observations = []
|
|
383
|
+
invalid_observations = []
|
|
384
|
+
for row in csv_file.itertuples(index=True):
|
|
385
|
+
rft_observation = cls(
|
|
386
|
+
name=f"{observation_dict['name']}[{row.Index}]",
|
|
387
|
+
well=str(row.WELL_NAME),
|
|
388
|
+
date=str(row.DATE),
|
|
389
|
+
property=observed_property,
|
|
390
|
+
value=validate_float(
|
|
391
|
+
str(getattr(row, observed_property)), observed_property
|
|
392
|
+
),
|
|
393
|
+
error=validate_float(str(row.ERROR), "ERROR"),
|
|
394
|
+
north=validate_float(str(row.NORTH), "NORTH"),
|
|
395
|
+
east=validate_float(str(row.EAST), "EAST"),
|
|
396
|
+
tvd=validate_float(str(row.TVD), "TVD"),
|
|
397
|
+
zone=row.ZONE if "ZONE" in csv_file else None,
|
|
398
|
+
)
|
|
399
|
+
# A value of -1 and error of 0 is used by fmu.tools.rms create_rft_ertobs to
|
|
400
|
+
# indicate missing data. If encountered in an rft observations csv file
|
|
401
|
+
# it should raise an error and ask the user to remove invalid observations.
|
|
402
|
+
if rft_observation.value == -1 and rft_observation.error == 0:
|
|
403
|
+
invalid_observations.append(rft_observation)
|
|
404
|
+
else:
|
|
405
|
+
rft_observations.append(rft_observation)
|
|
406
|
+
|
|
407
|
+
if invalid_observations:
|
|
408
|
+
well_list = "\n - ".join(
|
|
409
|
+
[
|
|
410
|
+
f"{observation.well} at date {observation.date}"
|
|
411
|
+
for observation in invalid_observations
|
|
412
|
+
]
|
|
413
|
+
)
|
|
414
|
+
raise ObservationConfigError.with_context(
|
|
415
|
+
(
|
|
416
|
+
f"Invalid value=-1 and error=0 detected in {filename} for "
|
|
417
|
+
f"well(s):\n - {well_list}\n"
|
|
418
|
+
"The invalid observation(s) must be removed from the file."
|
|
419
|
+
),
|
|
420
|
+
filename,
|
|
421
|
+
)
|
|
422
|
+
return rft_observations
|
|
230
423
|
|
|
231
424
|
@classmethod
|
|
232
|
-
def from_obs_dict(
|
|
425
|
+
def from_obs_dict(
|
|
426
|
+
cls, directory: str, observation_dict: ObservationDict
|
|
427
|
+
) -> list[Self]:
|
|
428
|
+
csv_filename = None
|
|
233
429
|
well = None
|
|
234
430
|
observed_property = None
|
|
235
431
|
observed_value = None
|
|
@@ -238,6 +434,7 @@ class RFTObservation:
|
|
|
238
434
|
north = None
|
|
239
435
|
east = None
|
|
240
436
|
tvd = None
|
|
437
|
+
zone = None
|
|
241
438
|
for key, value in observation_dict.items():
|
|
242
439
|
match key:
|
|
243
440
|
case "type" | "name":
|
|
@@ -258,8 +455,19 @@ class RFTObservation:
|
|
|
258
455
|
east = validate_float(value, key)
|
|
259
456
|
case "TVD":
|
|
260
457
|
tvd = validate_float(value, key)
|
|
458
|
+
case "CSV":
|
|
459
|
+
csv_filename = value
|
|
460
|
+
case "ZONE":
|
|
461
|
+
zone = value
|
|
261
462
|
case _:
|
|
262
463
|
raise _unknown_key_error(str(key), observation_dict["name"])
|
|
464
|
+
if csv_filename is not None:
|
|
465
|
+
return cls.from_csv(
|
|
466
|
+
directory,
|
|
467
|
+
observation_dict,
|
|
468
|
+
csv_filename,
|
|
469
|
+
observed_property or "PRESSURE",
|
|
470
|
+
)
|
|
263
471
|
if well is None:
|
|
264
472
|
raise _missing_value_error(observation_dict["name"], "WELL")
|
|
265
473
|
if observed_value is None:
|
|
@@ -276,30 +484,106 @@ class RFTObservation:
|
|
|
276
484
|
raise _missing_value_error(observation_dict["name"], "EAST")
|
|
277
485
|
if tvd is None:
|
|
278
486
|
raise _missing_value_error(observation_dict["name"], "TVD")
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
observed_property,
|
|
284
|
-
observed_value,
|
|
285
|
-
error,
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
487
|
+
|
|
488
|
+
obs_instance = cls(
|
|
489
|
+
name=observation_dict["name"],
|
|
490
|
+
well=well,
|
|
491
|
+
property=observed_property,
|
|
492
|
+
value=observed_value,
|
|
493
|
+
error=error,
|
|
494
|
+
date=date,
|
|
495
|
+
north=north,
|
|
496
|
+
east=east,
|
|
497
|
+
tvd=tvd,
|
|
498
|
+
zone=zone,
|
|
289
499
|
)
|
|
290
500
|
|
|
501
|
+
# Bypass pydantic discarding context
|
|
502
|
+
# only relevant for ERT config surfacing validation errors
|
|
503
|
+
# irrelevant for runmodels etc.
|
|
504
|
+
obs_instance.name = observation_dict["name"]
|
|
505
|
+
|
|
506
|
+
return [obs_instance]
|
|
291
507
|
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
508
|
+
|
|
509
|
+
class BreakthroughObservation(BaseModel):
|
|
510
|
+
type: Literal["breakthrough"] = "breakthrough"
|
|
511
|
+
name: str
|
|
512
|
+
response_key: str
|
|
513
|
+
date: datetime
|
|
514
|
+
error: float
|
|
515
|
+
threshold: float
|
|
516
|
+
north: float | None
|
|
517
|
+
east: float | None
|
|
518
|
+
radius: float | None
|
|
519
|
+
|
|
520
|
+
@classmethod
|
|
521
|
+
def from_obs_dict(cls, directory: str, obs_dict: ObservationDict) -> list[Self]:
|
|
522
|
+
response_key = None
|
|
523
|
+
date = None
|
|
524
|
+
error = None
|
|
525
|
+
threshold = None
|
|
526
|
+
localization_dict: dict[LOCALIZATION_KEYS, float | None] = {}
|
|
527
|
+
for key, value in obs_dict.items():
|
|
528
|
+
match key:
|
|
529
|
+
case "type" | "name":
|
|
530
|
+
pass
|
|
531
|
+
case "KEY":
|
|
532
|
+
response_key = value
|
|
533
|
+
case "DATE":
|
|
534
|
+
date = value
|
|
535
|
+
case "ERROR":
|
|
536
|
+
error = validate_float(value, key)
|
|
537
|
+
case "THRESHOLD":
|
|
538
|
+
threshold = validate_float(value, key)
|
|
539
|
+
case "LOCALIZATION":
|
|
540
|
+
validate_localization(value, obs_dict["name"])
|
|
541
|
+
extract_localization_values(localization_dict, value, key)
|
|
542
|
+
case _:
|
|
543
|
+
raise _unknown_key_error(str(key), value)
|
|
544
|
+
|
|
545
|
+
if response_key is None:
|
|
546
|
+
raise _missing_value_error(obs_dict["name"], "KEY")
|
|
547
|
+
if date is None:
|
|
548
|
+
raise _missing_value_error(obs_dict["name"], "DATE")
|
|
549
|
+
if error is None:
|
|
550
|
+
raise _missing_value_error(obs_dict["name"], "ERROR")
|
|
551
|
+
if threshold is None:
|
|
552
|
+
raise _missing_value_error(obs_dict["name"], "THRESHOLD")
|
|
553
|
+
|
|
554
|
+
return [
|
|
555
|
+
cls(
|
|
556
|
+
name=obs_dict["name"],
|
|
557
|
+
response_key=response_key,
|
|
558
|
+
date=date,
|
|
559
|
+
error=error,
|
|
560
|
+
threshold=threshold,
|
|
561
|
+
north=localization_dict.get("north"),
|
|
562
|
+
east=localization_dict.get("east"),
|
|
563
|
+
radius=localization_dict.get("radius"),
|
|
564
|
+
)
|
|
565
|
+
]
|
|
566
|
+
|
|
567
|
+
|
|
568
|
+
Observation = Annotated[
|
|
569
|
+
(
|
|
570
|
+
SummaryObservation
|
|
571
|
+
| GeneralObservation
|
|
572
|
+
| RFTObservation
|
|
573
|
+
| BreakthroughObservation
|
|
574
|
+
),
|
|
575
|
+
Field(discriminator="type"),
|
|
576
|
+
]
|
|
295
577
|
|
|
296
578
|
_TYPE_TO_CLASS: dict[ObservationType, type[Observation]] = {
|
|
297
|
-
ObservationType.HISTORY: HistoryObservation,
|
|
298
579
|
ObservationType.SUMMARY: SummaryObservation,
|
|
299
580
|
ObservationType.GENERAL: GeneralObservation,
|
|
300
581
|
ObservationType.RFT: RFTObservation,
|
|
582
|
+
ObservationType.BREAKTHROUGH: BreakthroughObservation,
|
|
301
583
|
}
|
|
302
584
|
|
|
585
|
+
LOCALIZATION_KEYS = Literal["east", "north", "radius"]
|
|
586
|
+
|
|
303
587
|
|
|
304
588
|
def make_observations(
|
|
305
589
|
directory: str, observation_dicts: Sequence[ObservationDict]
|
|
@@ -312,10 +596,20 @@ def make_observations(
|
|
|
312
596
|
inp: The collection of statements to validate.
|
|
313
597
|
"""
|
|
314
598
|
result: list[Observation] = []
|
|
315
|
-
error_list: list[ErrorInfo] = []
|
|
599
|
+
error_list: list[ErrorInfo | ObservationConfigError] = []
|
|
316
600
|
for obs_dict in observation_dicts:
|
|
601
|
+
if obs_dict["type"] == ObservationType.HISTORY:
|
|
602
|
+
msg = (
|
|
603
|
+
"HISTORY_OBSERVATION is deprecated, and must be specified "
|
|
604
|
+
"as SUMMARY_OBSERVATION. Run"
|
|
605
|
+
" ert convert_observations <ert_config.ert> to convert your "
|
|
606
|
+
"observations automatically"
|
|
607
|
+
)
|
|
608
|
+
logger.error(msg)
|
|
609
|
+
error_list.append(ObservationConfigError(msg))
|
|
610
|
+
continue
|
|
317
611
|
try:
|
|
318
|
-
result.
|
|
612
|
+
result.extend(
|
|
319
613
|
_TYPE_TO_CLASS[obs_dict["type"]].from_obs_dict(directory, obs_dict)
|
|
320
614
|
)
|
|
321
615
|
except KeyError as err:
|
|
@@ -329,41 +623,6 @@ def make_observations(
|
|
|
329
623
|
return result
|
|
330
624
|
|
|
331
625
|
|
|
332
|
-
def _validate_segment_dict(name_token: str, inp: dict[str, Any]) -> Segment:
|
|
333
|
-
start = None
|
|
334
|
-
stop = None
|
|
335
|
-
error_mode = ErrorModes.RELMIN
|
|
336
|
-
error = 0.1
|
|
337
|
-
error_min = 0.1
|
|
338
|
-
for key, value in inp.items():
|
|
339
|
-
match key:
|
|
340
|
-
case "START":
|
|
341
|
-
start = validate_int(value, key)
|
|
342
|
-
case "STOP":
|
|
343
|
-
stop = validate_int(value, key)
|
|
344
|
-
case "ERROR":
|
|
345
|
-
error = validate_positive_float(value, key)
|
|
346
|
-
case "ERROR_MIN":
|
|
347
|
-
error_min = validate_positive_float(value, key)
|
|
348
|
-
case "ERROR_MODE":
|
|
349
|
-
error_mode = validate_error_mode(value)
|
|
350
|
-
case _:
|
|
351
|
-
raise _unknown_key_error(key, name_token)
|
|
352
|
-
|
|
353
|
-
if start is None:
|
|
354
|
-
raise _missing_value_error(name_token, "START")
|
|
355
|
-
if stop is None:
|
|
356
|
-
raise _missing_value_error(name_token, "STOP")
|
|
357
|
-
return Segment(
|
|
358
|
-
name=name_token,
|
|
359
|
-
start=start,
|
|
360
|
-
stop=stop,
|
|
361
|
-
error_mode=error_mode,
|
|
362
|
-
error=error,
|
|
363
|
-
error_min=error_min,
|
|
364
|
-
)
|
|
365
|
-
|
|
366
|
-
|
|
367
626
|
def validate_error_mode(inp: str) -> ErrorModes:
|
|
368
627
|
if inp == "REL":
|
|
369
628
|
return ErrorModes.REL
|
|
@@ -390,12 +649,15 @@ def validate_int(val: str, key: str) -> int:
|
|
|
390
649
|
raise _conversion_error(key, val, "int") from err
|
|
391
650
|
|
|
392
651
|
|
|
393
|
-
def validate_positive_float(
|
|
652
|
+
def validate_positive_float(
|
|
653
|
+
val: str, key: str, strictly_positive: bool = False
|
|
654
|
+
) -> float:
|
|
394
655
|
v = validate_float(val, key)
|
|
395
|
-
if v < 0:
|
|
656
|
+
if v < 0 or (v <= 0 and strictly_positive):
|
|
396
657
|
raise ObservationConfigError.with_context(
|
|
397
658
|
f'Failed to validate "{val}" in {key}={val}.'
|
|
398
|
-
f" {key} must be given a
|
|
659
|
+
f" {key} must be given a "
|
|
660
|
+
f"{'strictly ' if strictly_positive else ''}positive value.",
|
|
399
661
|
val,
|
|
400
662
|
)
|
|
401
663
|
return v
|
|
@@ -414,6 +676,16 @@ def validate_localization(val: dict[str, Any], obs_name: str) -> None:
|
|
|
414
676
|
raise ObservationConfigError.from_collected(errors)
|
|
415
677
|
|
|
416
678
|
|
|
679
|
+
def extract_localization_values(
|
|
680
|
+
localization_dict: dict[LOCALIZATION_KEYS, float | None], value: Any, key: str
|
|
681
|
+
) -> None:
|
|
682
|
+
localization_dict["east"] = validate_float(value["EAST"], key)
|
|
683
|
+
localization_dict["north"] = validate_float(value["NORTH"], key)
|
|
684
|
+
localization_dict["radius"] = (
|
|
685
|
+
validate_float(value["RADIUS"], key) if "RADIUS" in value else None
|
|
686
|
+
)
|
|
687
|
+
|
|
688
|
+
|
|
417
689
|
def validate_positive_int(val: str, key: str) -> int:
|
|
418
690
|
try:
|
|
419
691
|
v = int(val)
|
|
@@ -434,6 +706,17 @@ def _missing_value_error(name_token: str, value_key: str) -> ObservationConfigEr
|
|
|
434
706
|
)
|
|
435
707
|
|
|
436
708
|
|
|
709
|
+
def _resolve_path(directory: str, filename: str) -> str:
|
|
710
|
+
if not os.path.isabs(filename):
|
|
711
|
+
filename = os.path.join(directory, filename)
|
|
712
|
+
if not os.path.exists(filename):
|
|
713
|
+
raise ObservationConfigError.with_context(
|
|
714
|
+
f"The following keywords did not resolve to a valid path:\n {filename}",
|
|
715
|
+
filename,
|
|
716
|
+
)
|
|
717
|
+
return filename
|
|
718
|
+
|
|
719
|
+
|
|
437
720
|
def _conversion_error(token: str, value: Any, type_name: str) -> ObservationConfigError:
|
|
438
721
|
return ObservationConfigError.with_context(
|
|
439
722
|
f'Could not convert {value} to {type_name}. Failed to validate "{value}"',
|