ert 17.0.0__py3-none-any.whl → 19.0.0rc2__py3-none-any.whl

ert/config/_create_observation_dataframes.py

@@ -18,6 +18,7 @@ from ._observations import (
     Observation,
     ObservationDate,
     ObservationError,
+    RFTObservation,
     SummaryObservation,
 )
 from .gen_data_config import GenDataConfig
@@ -28,18 +29,21 @@ from .parsing import (
     ObservationConfigError,
 )
 from .refcase import Refcase
+from .rft_config import RFTConfig
 
 if TYPE_CHECKING:
     import numpy.typing as npt
 
 
 DEFAULT_TIME_DELTA = timedelta(seconds=30)
+DEFAULT_LOCATION_RANGE_M = 3000
 
 
 def create_observation_dataframes(
     observations: Sequence[Observation],
     refcase: Refcase | None,
     gen_data_config: GenDataConfig | None,
+    rft_config: RFTConfig | None,
     time_map: list[datetime] | None,
     history: HistorySource,
 ) -> dict[str, pl.DataFrame]:
@@ -55,7 +59,6 @@ def create_observation_dataframes(
     config_errors: list[ErrorInfo] = []
     grouped: dict[str, list[pl.DataFrame]] = defaultdict(list)
     for obs in observations:
-        obs_name = obs.name
         try:
             match obs:
                 case HistoryObservation():
@@ -63,7 +66,7 @@ def create_observation_dataframes(
                         _handle_history_observation(
                             refcase,
                             obs,
-                            obs_name,
+                            obs.name,
                             history,
                             time_len,
                         )
@@ -72,7 +75,7 @@ def create_observation_dataframes(
                     grouped["summary"].append(
                         _handle_summary_observation(
                             obs,
-                            obs_name,
+                            obs.name,
                             obs_time_list,
                             bool(refcase),
                         )
@@ -82,11 +85,18 @@ def create_observation_dataframes(
                         _handle_general_observation(
                             gen_data_config,
                             obs,
-                            obs_name,
+                            obs.name,
                             obs_time_list,
                             bool(refcase),
                         )
                     )
+                case RFTObservation():
+                    if rft_config is None:
+                        raise TypeError(
+                            "create_observation_dataframes requires "
+                            "rft_config is not None when using RFTObservation"
+                        )
+                    grouped["rft"].append(_handle_rft_observation(rft_config, obs))
                 case default:
                     assert_never(default)
         except ObservationConfigError as err:
@@ -286,6 +296,43 @@ def _get_restart(
         ) from err
 
 
+def _has_localization(summary_dict: SummaryObservation) -> bool:
+    return any(
+        [
+            summary_dict.location_x is not None,
+            summary_dict.location_y is not None,
+            summary_dict.location_range is not None,
+        ]
+    )
+
+
+def _validate_localization_values(summary_dict: SummaryObservation) -> None:
+    """The user must provide LOCATION_X and LOCATION_Y to use localization, while
+    unprovided LOCATION_RANGE should default to some value.
+
+    This method assumes the summary dict contains at least one LOCATION key.
+    """
+    if summary_dict.location_x is None or summary_dict.location_y is None:
+        loc_values = {
+            "LOCATION_X": summary_dict.location_x,
+            "LOCATION_Y": summary_dict.location_y,
+            "LOCATION_RANGE": summary_dict.location_range,
+        }
+        provided_loc_values = {k: v for k, v in loc_values.items() if v is not None}
+
+        provided_loc_values_string = ", ".join(
+            key.upper() for key in provided_loc_values
+        )
+        raise ObservationConfigError.with_context(
+            f"Localization for observation {summary_dict.name} is misconfigured.\n"
+            f"Only {provided_loc_values_string} were provided. To enable "
+            f"localization for an observation, ensure that both LOCATION_X and "
+            f"LOCATION_Y are defined - or remove LOCATION keywords to disable "
+            f"localization.",
+            summary_dict,
+        )
+
+
 def _handle_summary_observation(
     summary_dict: SummaryObservation,
     obs_key: str,
@@ -323,15 +370,23 @@ def _handle_summary_observation(
             "Observation uncertainty must be strictly > 0", summary_key
         ) from None
 
-    return pl.DataFrame(
-        {
-            "response_key": [summary_key],
-            "observation_key": [obs_key],
-            "time": pl.Series([date]).dt.cast_time_unit("ms"),
-            "observations": pl.Series([value], dtype=pl.Float32),
-            "std": pl.Series([std_dev], dtype=pl.Float32),
-        }
-    )
+    data_dict = {
+        "response_key": [summary_key],
+        "observation_key": [obs_key],
+        "time": pl.Series([date]).dt.cast_time_unit("ms"),
+        "observations": pl.Series([value], dtype=pl.Float32),
+        "std": pl.Series([std_dev], dtype=pl.Float32),
+    }
+
+    if _has_localization(summary_dict):
+        _validate_localization_values(summary_dict)
+        data_dict["location_x"] = summary_dict.location_x
+        data_dict["location_y"] = summary_dict.location_y
+        data_dict["location_range"] = (
+            summary_dict.location_range or DEFAULT_LOCATION_RANGE_M
+        )
+
+    return pl.DataFrame(data_dict)
 
 
 def _handle_general_observation(
@@ -414,10 +469,8 @@ def _handle_general_observation(
         stds = file_values[1::2]
 
     else:
-        assert (
-            general_observation.value is not None
-            and general_observation.error is not None
-        )
+        assert general_observation.value is not None
+        assert general_observation.error is not None
         values = np.array([general_observation.value])
         stds = np.array([general_observation.error])
 
@@ -439,9 +492,11 @@ def _handle_general_observation(
         raise ObservationConfigError.with_context(
             f"Values ({values}), error ({stds}) and "
             f"index list ({indices}) must be of equal length",
-            general_observation.obs_file
-            if general_observation.obs_file is not None
-            else "",
+            (
+                general_observation.obs_file
+                if general_observation.obs_file is not None
+                else ""
+            ),
         )
 
     if np.any(stds <= 0):
@@ -461,3 +516,45 @@ def _handle_general_observation(
             "std": pl.Series(stds, dtype=pl.Float32),
         }
     )
+
+
+def _handle_rft_observation(
+    rft_config: RFTConfig,
+    rft_observation: RFTObservation,
+) -> pl.DataFrame:
+    location = (rft_observation.east, rft_observation.north, rft_observation.tvd)
+    if location not in rft_config.locations:
+        rft_config.locations.append(location)
+
+    data_to_read = rft_config.data_to_read
+    if rft_observation.well not in data_to_read:
+        rft_config.data_to_read[rft_observation.well] = {}
+
+    well_dict = data_to_read[rft_observation.well]
+    if rft_observation.date not in well_dict:
+        well_dict[rft_observation.date] = []
+
+    property_list = well_dict[rft_observation.date]
+    if rft_observation.property not in property_list:
+        property_list.append(rft_observation.property)
+
+    if rft_observation.error <= 0.0:
+        raise ObservationConfigError.with_context(
+            "Observation uncertainty must be strictly > 0", rft_observation.well
+        )
+
+    return pl.DataFrame(
+        {
+            "response_key": (
+                f"{rft_observation.well}:"
+                f"{rft_observation.date}:"
+                f"{rft_observation.property}"
+            ),
+            "observation_key": rft_observation.name,
+            "east": pl.Series([location[0]], dtype=pl.Float32),
+            "north": pl.Series([location[1]], dtype=pl.Float32),
+            "tvd": pl.Series([location[2]], dtype=pl.Float32),
+            "observations": pl.Series([rft_observation.value], dtype=pl.Float32),
+            "std": pl.Series([rft_observation.error], dtype=pl.Float32),
+        }
+    )

ert/config/_get_num_cpu.py

@@ -156,7 +156,7 @@ class _Parser:
     def next_line(self) -> Iterator[str]: ...
 
     @overload
-    def next_line(self, __default: T) -> Iterator[str] | T: ...
+    def next_line(self, __default: T, /) -> Iterator[str] | T: ...
 
     def next_line(self, *args: T) -> Iterator[str] | T:
         self.line_number += 1

ert/config/_observations.py

@@ -90,10 +90,13 @@ class _SummaryValues:
     name: str
     value: float
     key: str  #: The :term:`summary key` in the summary response
+    location_x: float | None = None
+    location_y: float | None = None
+    location_range: float | None = None
 
 
 @dataclass
-class SummaryObservation(ObservationDate, ObservationError, _SummaryValues):
+class SummaryObservation(ObservationDate, _SummaryValues, ObservationError):
     @classmethod
     def from_obs_dict(cls, directory: str, observation_dict: ObservationDict) -> Self:
         error_mode = ErrorModes.ABS
@@ -101,6 +104,7 @@ class SummaryObservation(ObservationDate, ObservationError, _SummaryValues):
 
         date_dict: ObservationDate = ObservationDate()
         float_values: dict[str, float] = {"ERROR_MIN": 0.1}
+        localization_values: dict[str, float] = {}
         for key, value in observation_dict.items():
             match key:
                 case "type" | "name":
@@ -121,6 +125,12 @@ class SummaryObservation(ObservationDate, ObservationError, _SummaryValues):
                     summary_key = value
                 case "DATE":
                     date_dict.date = value
+                case "LOCATION_X":
+                    localization_values["x"] = validate_float(value, key)
+                case "LOCATION_Y":
+                    localization_values["y"] = validate_float(value, key)
+                case "LOCATION_RANGE":
+                    localization_values["range"] = validate_float(value, key)
                 case _:
                     raise _unknown_key_error(str(key), observation_dict["name"])
         if "VALUE" not in float_values:
@@ -137,6 +147,9 @@ class SummaryObservation(ObservationDate, ObservationError, _SummaryValues):
             error_min=float_values["ERROR_MIN"],
             key=summary_key,
             value=float_values["VALUE"],
+            location_x=localization_values.get("x"),
+            location_y=localization_values.get("y"),
+            location_range=localization_values.get("range"),
             **date_dict.__dict__,
         )
 
@@ -201,12 +214,88 @@ class GeneralObservation(ObservationDate, _GeneralObservation):
         return output
 
 
-Observation = HistoryObservation | SummaryObservation | GeneralObservation
+@dataclass
+class RFTObservation:
+    name: str
+    well: str
+    date: str
+    property: str
+    value: float
+    error: float
+    north: float
+    east: float
+    tvd: float
+
+    @classmethod
+    def from_obs_dict(cls, directory: str, observation_dict: ObservationDict) -> Self:
+        well = None
+        observed_property = None
+        observed_value = None
+        error = None
+        date = None
+        north = None
+        east = None
+        tvd = None
+        for key, value in observation_dict.items():
+            match key:
+                case "type" | "name":
+                    pass
+                case "WELL":
+                    well = value
+                case "PROPERTY":
+                    observed_property = value
+                case "VALUE":
+                    observed_value = validate_float(value, key)
+                case "ERROR":
+                    error = validate_float(value, key)
+                case "DATE":
+                    date = value
+                case "NORTH":
+                    north = validate_float(value, key)
+                case "EAST":
+                    east = validate_float(value, key)
+                case "TVD":
+                    tvd = validate_float(value, key)
+                case _:
+                    raise _unknown_key_error(str(key), observation_dict["name"])
+        if well is None:
+            raise _missing_value_error(observation_dict["name"], "WELL")
+        if observed_value is None:
+            raise _missing_value_error(observation_dict["name"], "VALUE")
+        if observed_property is None:
+            raise _missing_value_error(observation_dict["name"], "PROPERTY")
+        if error is None:
+            raise _missing_value_error(observation_dict["name"], "ERROR")
+        if date is None:
+            raise _missing_value_error(observation_dict["name"], "DATE")
+        if north is None:
+            raise _missing_value_error(observation_dict["name"], "NORTH")
+        if east is None:
+            raise _missing_value_error(observation_dict["name"], "EAST")
+        if tvd is None:
+            raise _missing_value_error(observation_dict["name"], "TVD")
+        return cls(
+            observation_dict["name"],
+            well,
+            date,
+            observed_property,
+            observed_value,
+            error,
+            north,
+            east,
+            tvd,
+        )
+
+
+Observation = (
+    HistoryObservation | SummaryObservation | GeneralObservation | RFTObservation
+)
 
 _TYPE_TO_CLASS: dict[ObservationType, type[Observation]] = {
     ObservationType.HISTORY: HistoryObservation,
     ObservationType.SUMMARY: SummaryObservation,
     ObservationType.GENERAL: GeneralObservation,
+    ObservationType.RFT: RFTObservation,
 }
 
 

ert/config/_read_summary.py

@@ -12,6 +12,7 @@ import warnings
 from collections.abc import Callable, Sequence
 from datetime import datetime, timedelta
 from enum import Enum, auto
+from functools import lru_cache
 
 import numpy as np
 import numpy.typing as npt
@@ -83,29 +84,30 @@ class DateUnit(Enum):
         raise InvalidResponseFile(f"Unknown date unit {val}")
 
 
+@lru_cache
 def _fetch_keys_to_matcher(fetch_keys: Sequence[str]) -> Callable[[str], bool]:
     """
     Transform the list of keys (with * used as repeated wildcard) into
     a matcher.
 
-    >>> match = _fetch_keys_to_matcher([""])
+    >>> match = _fetch_keys_to_matcher(("",))
     >>> match("FOPR")
     False
 
-    >>> match = _fetch_keys_to_matcher(["*"])
+    >>> match = _fetch_keys_to_matcher(("*",))
     >>> match("FOPR"), match("FO*")
     (True, True)
 
 
-    >>> match = _fetch_keys_to_matcher(["F*PR"])
+    >>> match = _fetch_keys_to_matcher(("F*PR",))
     >>> match("WOPR"), match("FOPR"), match("FGPR"), match("SOIL")
     (False, True, True, False)
 
-    >>> match = _fetch_keys_to_matcher(["WGOR:*"])
+    >>> match = _fetch_keys_to_matcher(("WGOR:*",))
     >>> match("FOPR"), match("WGOR:OP1"), match("WGOR:OP2"), match("WGOR")
     (False, True, True, False)
 
-    >>> match = _fetch_keys_to_matcher(["FOPR", "FGPR"])
+    >>> match = _fetch_keys_to_matcher(("FOPR", "FGPR"))
     >>> match("FOPR"), match("FGPR"), match("WGOR:OP2"), match("WGOR")
     (True, True, False, False)
     """
@@ -138,7 +140,7 @@ def _read_spec(
     date_index = None
     date_unit_str = None
 
-    should_load_key = _fetch_keys_to_matcher(fetch_keys)
+    should_load_key = _fetch_keys_to_matcher(tuple(fetch_keys))
 
     for i, kw in enumerate(keywords):
         try:

ert/config/design_matrix.py

@@ -25,6 +25,7 @@ class DesignMatrix:
     xls_filename: Path
     design_sheet: str
     default_sheet: str | None
+    priority_source: str = "design_matrix"
 
     def __post_init__(self) -> None:
         try:
@@ -33,6 +34,9 @@ class DesignMatrix:
                 self.design_matrix_df,
                 self.parameter_configurations,
             ) = self.read_and_validate_design_matrix()
+            self.parameter_priority = {
+                cfg.name: self.priority_source for cfg in self.parameter_configurations
+            }
         except (ValueError, AttributeError) as exc:
             raise ConfigValidationError.with_context(
                 f"Error reading design matrix {self.xls_filename}"
@@ -45,7 +49,7 @@ class DesignMatrix:
     def from_config_list(cls, config_list: list[str | dict[str, str]]) -> DesignMatrix:
         filename = Path(cast(str, config_list[0]))
         options = cast(dict[str, str], config_list[1])
-        valid_options = ["DESIGN_SHEET", "DEFAULT_SHEET"]
+        valid_options = ["DESIGN_SHEET", "DEFAULT_SHEET", "PRIORITY"]
         option_errors = [
             ErrorInfo(
                 f"Option {option} is not a valid DESIGN_MATRIX option. "
@@ -59,6 +63,7 @@ class DesignMatrix:
             raise ConfigValidationError.from_collected(option_errors)
         design_sheet = options.get("DESIGN_SHEET", "DesignSheet")
         default_sheet = options.get("DEFAULT_SHEET", None)
+        priority_source = options.get("PRIORITY", DataSource.DESIGN_MATRIX)
         errors = []
         if filename.suffix not in {
             ".xlsx",
@@ -75,6 +80,13 @@ class DesignMatrix:
                     "DESIGN_SHEET and DEFAULT_SHEET can not point to the same sheet."
                 ).set_context(config_list)
             )
+        if priority_source not in {DataSource.DESIGN_MATRIX, DataSource.SAMPLED}:
+            errors.append(
+                ErrorInfo(
+                    f"PRIORITY must be either '{DataSource.DESIGN_MATRIX}'"
+                    f" or '{DataSource.SAMPLED}' priority is '{priority_source}'"
+                ).set_context(config_list)
+            )
         if errors:
             raise ConfigValidationError.from_collected(errors)
         assert design_sheet is not None
@@ -82,6 +94,7 @@ class DesignMatrix:
             xls_filename=filename,
             design_sheet=design_sheet,
             default_sheet=default_sheet,
+            priority_source=priority_source,
         )
 
     def merge_with_other(self, dm_other: DesignMatrix) -> None:
@@ -99,24 +112,17 @@ class DesignMatrix:
         common_keys = set(
             self.design_matrix_df.select(pl.exclude("realization")).columns
         ) & set(dm_other.design_matrix_df.columns)
-        non_identical_cols = set()
         if common_keys:
-            for key in common_keys:
-                if not self.design_matrix_df.select(key).equals(
-                    dm_other.design_matrix_df.select(key)
-                ):
-                    non_identical_cols.add(key)
-            if non_identical_cols:
-                errors.append(
-                    ErrorInfo(
-                        f"Design Matrices '{self.xls_filename.name} "
-                        f"({self.design_sheet} {self.default_sheet or ''})' and "
-                        f"'{dm_other.xls_filename.name} ({dm_other.design_sheet} "
-                        f"{dm_other.default_sheet or ''})' "
-                        "contains non identical columns with the same name: "
-                        f"{non_identical_cols}!"
-                    )
+            errors.append(
+                ErrorInfo(
+                    f"Design Matrices '{self.xls_filename.name} "
+                    f"({self.design_sheet} {self.default_sheet or ''})' and "
+                    f"'{dm_other.xls_filename.name} ({dm_other.design_sheet} "
+                    f"{dm_other.default_sheet or ''})' "
+                    "contains columns with the same name: "
+                    f"{common_keys}!"
                 )
+            )
 
         if errors:
             raise ConfigValidationError.from_collected(errors)
@@ -125,9 +131,7 @@ class DesignMatrix:
             self.design_matrix_df = pl.concat(
                 [
                     self.design_matrix_df,
-                    dm_other.design_matrix_df.select(
-                        pl.exclude([*list(common_keys), "realization"])
-                    ),
+                    dm_other.design_matrix_df.select(pl.exclude(["realization"])),
                 ],
                 how="horizontal",
             )
@@ -145,6 +149,7 @@ class DesignMatrix:
             for cfg in dm_other.parameter_configurations
             if cfg.name not in common_keys
         )
+        self.parameter_priority.update(dm_other.parameter_priority)
 
     def merge_with_existing_parameters(
         self, existing_parameters: list[ParameterConfig]
@@ -166,11 +171,33 @@ class DesignMatrix:
 
         for param_cfg in existing_parameters:
             if isinstance(param_cfg, GenKwConfig) and param_cfg.name in design_cfgs:
-                param_cfg.input_source = DataSource.DESIGN_MATRIX
-                param_cfg.update = False
-                param_cfg.distribution = RawSettings()
                 del design_cfgs[param_cfg.name]
-            new_param_configs += [param_cfg]
+                input_source = DataSource(
+                    self.parameter_priority.get(
+                        param_cfg.name, DataSource.DESIGN_MATRIX.value
+                    )
+                )
+                new_param_configs += [
+                    GenKwConfig(
+                        name=param_cfg.name,
+                        update=(
+                            input_source == DataSource.SAMPLED and param_cfg.update
+                        ),
+                        distribution=(
+                            RawSettings()
+                            if input_source == DataSource.DESIGN_MATRIX
+                            else param_cfg.distribution
+                        ),
+                        group=(
+                            DESIGN_MATRIX_GROUP
+                            if input_source == DataSource.DESIGN_MATRIX
+                            else param_cfg.group
+                        ),
+                        input_source=input_source,
+                    ),
+                ]
+            else:
+                new_param_configs += [param_cfg]
         if design_cfgs.values():
             new_param_configs += list(design_cfgs.values())
         return new_param_configs

ert/config/distribution.py

@@ -17,7 +17,7 @@ class TransSettingsValidation(BaseModel):
     model_config = {"extra": "forbid"}
 
     @classmethod
-    def create(cls: type[T], *args: Any, **kwargs: Any) -> T:
+    def create(cls, *args: Any, **kwargs: Any) -> Self:
         return cls(*args, **kwargs)
 
     @classmethod

ert/config/ensemble_config.py

@@ -7,30 +7,22 @@ from typing import Self
 
 from pydantic import BaseModel, Field, model_validator
 
-from .ext_param_config import ExtParamConfig
+from .everest_control import EverestControl
 from .field import Field as FieldConfig
-from .gen_data_config import GenDataConfig
 from .gen_kw_config import GenKwConfig
+from .known_response_types import KNOWN_ERT_RESPONSE_TYPES, KnownErtResponseTypes
 from .parameter_config import ParameterConfig
 from .parsing import ConfigDict, ConfigKeys, ConfigValidationError
 from .response_config import ResponseConfig
-from .summary_config import SummaryConfig
 from .surface_config import SurfaceConfig
 
-KnownResponseTypes = SummaryConfig | GenDataConfig
-
-_KNOWN_RESPONSE_TYPES = (
-    SummaryConfig,
-    GenDataConfig,
-)
-
 logger = logging.getLogger(__name__)
 
 
 class EnsembleConfig(BaseModel):
-    response_configs: dict[str, KnownResponseTypes] = Field(default_factory=dict)
+    response_configs: dict[str, KnownErtResponseTypes] = Field(default_factory=dict)
     parameter_configs: dict[
-        str, GenKwConfig | FieldConfig | SurfaceConfig | ExtParamConfig
+        str, GenKwConfig | FieldConfig | SurfaceConfig | EverestControl
     ] = Field(default_factory=dict)
 
     @model_validator(mode="after")
@@ -131,16 +123,16 @@ class EnsembleConfig(BaseModel):
             + [make_field(f) for f in field_list]
         )
         EnsembleConfig._check_for_duplicate_gen_kw_param_names(gen_kw_cfgs)
-        response_configs: list[KnownResponseTypes] = []
+        response_configs: list[KnownErtResponseTypes] = []
 
-        for config_cls in _KNOWN_RESPONSE_TYPES:
+        for config_cls in KNOWN_ERT_RESPONSE_TYPES:
             instance = config_cls.from_config_dict(config_dict)
 
             if instance is not None and instance.keys:
                 response_configs.append(instance)
 
         return cls(
-            response_configs={response.name: response for response in response_configs},
+            response_configs={response.type: response for response in response_configs},
             parameter_configs={
                 parameter.name: parameter for parameter in parameter_configs
             },
@@ -151,13 +143,13 @@ class EnsembleConfig(BaseModel):
             return self.parameter_configs[key]
         elif key in self.response_configs:
             return self.response_configs[key]
-        elif _config := next(
+        elif config := next(
             (c for c in self.response_configs.values() if key in c.keys), None
         ):
             # Only hit by blockfs migration
             # returns the same config for one call per
             # response type. Is later deduped before saving to json
-            return _config
+            return config
         else:
             raise KeyError(f"The key:{key} is not in the ensemble configuration")