ert 19.0.0rc0__py3-none-any.whl → 19.0.0rc1__py3-none-any.whl

ert/config/_observations.py

@@ -1,11 +1,14 @@
+from __future__ import annotations
+
+import logging
 import os
-from collections import Counter
 from collections.abc import Sequence
 from dataclasses import dataclass
 from enum import StrEnum
-from itertools import starmap
 from typing import Any, Self
 
+import pandas as pd
+
 from .parsing import (
     ErrorInfo,
     ObservationConfigError,
@@ -13,6 +16,8 @@ from .parsing import (
     ObservationType,
 )
 
+logger = logging.getLogger(__name__)
+
 
 class ErrorModes(StrEnum):
     REL = "REL"
@@ -20,102 +25,52 @@ class ErrorModes(StrEnum):
     RELMIN = "RELMIN"
 
 
-@dataclass
-class ObservationError:
-    error_mode: ErrorModes
-    error: float
-    error_min: float
-
-
-@dataclass
-class Segment(ObservationError):
-    name: str
-    start: int
-    stop: int
-
-
-@dataclass
-class HistoryObservation(ObservationError):
-    name: str
-    segments: list[Segment]
-
-    @property
-    def key(self) -> str:
-        """The :term:`summary key` to be fetched from :ref:`refcase`."""
-        # For history observations the key is also the name, ie.
-        # "HISTORY_OBSERVATION FOPR" means to add the values from
-        # the summary vector FOPRH in refcase as observations.
-        return self.name
-
-    @classmethod
-    def from_obs_dict(cls, directory: str, observation_dict: ObservationDict) -> Self:
-        error_mode = ErrorModes.RELMIN
-        error = 0.1
-        error_min = 0.1
-        segments = []
-        for key, value in observation_dict.items():
-            match key:
-                case "type" | "name":
-                    pass
-                case "ERROR":
-                    error = validate_positive_float(value, key)
-                case "ERROR_MIN":
-                    error_min = validate_positive_float(value, key)
-                case "ERROR_MODE":
-                    error_mode = validate_error_mode(value)
-                case "segments":
-                    segments = list(starmap(_validate_segment_dict, value))
-                case _:
-                    raise _unknown_key_error(str(key), observation_dict["name"])
-
-        return cls(
-            name=observation_dict["name"],
-            error_mode=error_mode,
-            error=error,
-            error_min=error_min,
-            segments=segments,
-        )
-
-
-@dataclass
-class ObservationDate:
-    days: float | None = None
-    hours: float | None = None
-    date: str | None = None
-    restart: int | None = None
-
-
 @dataclass
 class _SummaryValues:
     name: str
     value: float
     key: str  #: The :term:`summary key` in the summary response
+    date: str
     location_x: float | None = None
     location_y: float | None = None
     location_range: float | None = None
 
 
 @dataclass
-class SummaryObservation(ObservationDate, _SummaryValues, ObservationError):
+class ObservationError:
+    error_mode: ErrorModes
+    error: float
+    error_min: float
+
+
+@dataclass
+class SummaryObservation(_SummaryValues, ObservationError):
     @classmethod
-    def from_obs_dict(cls, directory: str, observation_dict: ObservationDict) -> Self:
+    def from_obs_dict(
+        cls, directory: str, observation_dict: ObservationDict
+    ) -> list[Self]:
         error_mode = ErrorModes.ABS
         summary_key = None
 
-        date_dict: ObservationDate = ObservationDate()
+        date: str | None = None
         float_values: dict[str, float] = {"ERROR_MIN": 0.1}
-        localization_values: dict[str, float] = {}
+        localization_values: dict[str, float | None] = {}
         for key, value in observation_dict.items():
             match key:
                 case "type" | "name":
                     pass
-                case "RESTART":
-                    date_dict.restart = validate_positive_int(value, key)
                 case "ERROR" | "ERROR_MIN":
                     float_values[str(key)] = validate_positive_float(value, key)
-                case "DAYS" | "HOURS":
-                    setattr(
-                        date_dict, str(key).lower(), validate_positive_float(value, key)
+                case "DAYS" | "HOURS" | "RESTART":
+                    raise ObservationConfigError.with_context(
+                        (
+                            "SUMMARY_OBSERVATION must use DATE to specify "
+                            "date, DAYS | HOURS is no longer allowed. "
+                            "Please run:\n ert convert_observations "
+                            "<your_ert_config.ert>\nto migrate the observation config "
+                            "to use the correct format."
+                        ),
+                        key,
                     )
                 case "VALUE":
                     float_values[str(key)] = validate_float(value, key)
@@ -124,13 +79,16 @@ class SummaryObservation(ObservationDate, _SummaryValues, ObservationError):
                 case "KEY":
                     summary_key = value
                 case "DATE":
-                    date_dict.date = value
-                case "LOCATION_X":
-                    localization_values["x"] = validate_float(value, key)
-                case "LOCATION_Y":
-                    localization_values["y"] = validate_float(value, key)
-                case "LOCATION_RANGE":
-                    localization_values["range"] = validate_float(value, key)
+                    date = value
+                case "LOCALIZATION":
+                    validate_localization(value, observation_dict["name"])
+                    localization_values["x"] = validate_float(value["EAST"], key)
+                    localization_values["y"] = validate_float(value["NORTH"], key)
+                    localization_values["range"] = (
+                        validate_float(value["RADIUS"], key)
+                        if "RADIUS" in value
+                        else None
+                    )
                 case _:
                     raise _unknown_key_error(str(key), observation_dict["name"])
         if "VALUE" not in float_values:
@@ -140,18 +98,21 @@ class SummaryObservation(ObservationDate, _SummaryValues, ObservationError):
         if "ERROR" not in float_values:
             raise _missing_value_error(observation_dict["name"], "ERROR")
 
-        return cls(
-            name=observation_dict["name"],
-            error_mode=error_mode,
-            error=float_values["ERROR"],
-            error_min=float_values["ERROR_MIN"],
-            key=summary_key,
-            value=float_values["VALUE"],
-            location_x=localization_values.get("x"),
-            location_y=localization_values.get("y"),
-            location_range=localization_values.get("range"),
-            **date_dict.__dict__,
-        )
+        assert date is not None
+        return [
+            cls(
+                name=observation_dict["name"],
+                error_mode=error_mode,
+                error=float_values["ERROR"],
+                error_min=float_values["ERROR_MIN"],
+                key=summary_key,
+                value=float_values["VALUE"],
+                location_x=localization_values.get("x"),
+                location_y=localization_values.get("y"),
+                location_range=localization_values.get("range"),
+                date=date,
+            )
+        ]
 
 
 @dataclass
@@ -163,12 +124,15 @@ class _GeneralObservation:
     index_list: str | None = None
     index_file: str | None = None
     obs_file: str | None = None
+    restart: int | None = None
 
 
 @dataclass
-class GeneralObservation(ObservationDate, _GeneralObservation):
+class GeneralObservation(_GeneralObservation):
     @classmethod
-    def from_obs_dict(cls, directory: str, observation_dict: ObservationDict) -> Self:
+    def from_obs_dict(
+        cls, directory: str, observation_dict: ObservationDict
+    ) -> list[Self]:
         try:
             data = observation_dict["DATA"]
         except KeyError as err:
@@ -183,12 +147,22 @@ class GeneralObservation(ObservationDate, _GeneralObservation):
                     output.restart = validate_positive_int(value, key)
                 case "VALUE":
                     output.value = validate_float(value, key)
-                case "ERROR" | "DAYS" | "HOURS":
+                case "ERROR":
                     setattr(
                         output, str(key).lower(), validate_positive_float(value, key)
                     )
-                case "DATE" | "INDEX_LIST":
-                    setattr(output, str(key).lower(), value)
+                case "DATE" | "DAYS" | "HOURS":
+                    raise ObservationConfigError.with_context(
+                        (
+                            "GENERAL_OBSERVATION must use RESTART to specify "
+                            "report step. Please run:\n ert convert_observations "
+                            "<your_ert_config.ert>\nto migrate the observation config "
+                            "to use the correct format."
+                        ),
+                        key,
+                    )
+                case "INDEX_LIST":
+                    output.index_list = value
                 case "OBS_FILE" | "INDEX_FILE":
                     assert not isinstance(key, tuple)
                     filename = value
@@ -211,7 +185,8 @@ class GeneralObservation(ObservationDate, _GeneralObservation):
                 f" VALUE = {output.value}, ERROR must also be given.",
                 observation_dict["name"],
             )
-        return output
+
+        return [output]
 
 
 @dataclass
@@ -227,7 +202,63 @@ class RFTObservation:
     tvd: float
 
     @classmethod
-    def from_obs_dict(cls, directory: str, observation_dict: ObservationDict) -> Self:
+    def from_csv(
+        cls,
+        directory: str,
+        observation_dict: ObservationDict,
+        filename: str,
+        observed_property: str = "PRESSURE",
+    ) -> list[Self]:
+        if not os.path.isabs(filename):
+            filename = os.path.join(directory, filename)
+        if not os.path.exists(filename):
+            raise ObservationConfigError.with_context(
+                f"The CSV file ({filename}) does not exist or is not accessible.",
+                filename,
+            )
+        csv_file = pd.read_csv(
+            filename,
+            encoding="utf-8",
+            on_bad_lines="error",
+        )
+
+        required_columns = {
+            "WELL_NAME",
+            "DATE",
+            observed_property,
+            "ERROR",
+            "NORTH",
+            "EAST",
+            "TVD",
+        }
+        missing_required_columns = required_columns - set(csv_file.keys())
+        if missing_required_columns:
+            raise ObservationConfigError.with_context(
+                f"The rft observations file {filename} is missing required column(s) "
+                f"{', '.join(sorted(missing_required_columns))}.",
+                filename,
+            )
+
+        return [
+            cls(
+                f"{observation_dict['name']}[{row.Index}]",
+                str(row.WELL_NAME),
+                str(row.DATE),
+                observed_property,
+                validate_float(str(getattr(row, observed_property)), observed_property),
+                validate_float(str(row.ERROR), "ERROR"),
+                validate_float(str(row.NORTH), "NORTH"),
+                validate_float(str(row.EAST), "EAST"),
+                validate_float(str(row.TVD), "TVD"),
+            )
+            for row in csv_file.itertuples(index=True)
+        ]
+
+    @classmethod
+    def from_obs_dict(
+        cls, directory: str, observation_dict: ObservationDict
+    ) -> list[Self]:
+        csv_filename = None
         well = None
         observed_property = None
         observed_value = None
@@ -256,8 +287,17 @@ class RFTObservation:
                     east = validate_float(value, key)
                 case "TVD":
                     tvd = validate_float(value, key)
+                case "CSV":
+                    csv_filename = value
                 case _:
                     raise _unknown_key_error(str(key), observation_dict["name"])
+        if csv_filename is not None:
+            return cls.from_csv(
+                directory,
+                observation_dict,
+                csv_filename,
+                observed_property or "PRESSURE",
+            )
         if well is None:
             raise _missing_value_error(observation_dict["name"], "WELL")
         if observed_value is None:
@@ -274,25 +314,24 @@ class RFTObservation:
             raise _missing_value_error(observation_dict["name"], "EAST")
         if tvd is None:
             raise _missing_value_error(observation_dict["name"], "TVD")
-        return cls(
-            observation_dict["name"],
-            well,
-            date,
-            observed_property,
-            observed_value,
-            error,
-            north,
-            east,
-            tvd,
-        )
+        return [
+            cls(
+                observation_dict["name"],
+                well,
+                date,
+                observed_property,
+                observed_value,
+                error,
+                north,
+                east,
+                tvd,
+            )
+        ]
 
 
-Observation = (
-    HistoryObservation | SummaryObservation | GeneralObservation | RFTObservation
-)
+Observation = SummaryObservation | GeneralObservation | RFTObservation
 
 _TYPE_TO_CLASS: dict[ObservationType, type[Observation]] = {
-    ObservationType.HISTORY: HistoryObservation,
     ObservationType.SUMMARY: SummaryObservation,
     ObservationType.GENERAL: GeneralObservation,
     ObservationType.RFT: RFTObservation,
@@ -310,10 +349,20 @@ def make_observations(
         inp: The collection of statements to validate.
     """
     result: list[Observation] = []
-    error_list: list[ErrorInfo] = []
+    error_list: list[ErrorInfo | ObservationConfigError] = []
     for obs_dict in observation_dicts:
+        if obs_dict["type"] == ObservationType.HISTORY:
+            msg = (
+                "HISTORY_OBSERVATION is deprecated, and must be specified "
+                "as SUMMARY_OBSERVATION. Run"
+                " ert convert_observations <ert_config.ert> to convert your "
+                "observations automatically"
+            )
+            logger.error(msg)
+            error_list.append(ObservationConfigError(msg))
+            continue
         try:
-            result.append(
+            result.extend(
                 _TYPE_TO_CLASS[obs_dict["type"]].from_obs_dict(directory, obs_dict)
             )
         except KeyError as err:
@@ -324,60 +373,9 @@ def make_observations(
     if error_list:
         raise ObservationConfigError.from_collected(error_list)
 
-    _validate_unique_names(result)
     return result
 
 
-def _validate_unique_names(
-    observations: Sequence[Observation],
-) -> None:
-    names_counter = Counter(d.name for d in observations)
-    duplicate_names = [n for n, c in names_counter.items() if c > 1]
-    errors = [
-        ErrorInfo(
-            f"Duplicate observation name {n}",
-        ).set_context(n)
-        for n in duplicate_names
-    ]
-    if errors:
-        raise ObservationConfigError.from_collected(errors)
-
-
-def _validate_segment_dict(name_token: str, inp: dict[str, Any]) -> Segment:
-    start = None
-    stop = None
-    error_mode = ErrorModes.RELMIN
-    error = 0.1
-    error_min = 0.1
-    for key, value in inp.items():
-        match key:
-            case "START":
-                start = validate_int(value, key)
-            case "STOP":
-                stop = validate_int(value, key)
-            case "ERROR":
-                error = validate_positive_float(value, key)
-            case "ERROR_MIN":
-                error_min = validate_positive_float(value, key)
-            case "ERROR_MODE":
-                error_mode = validate_error_mode(value)
-            case _:
-                raise _unknown_key_error(key, name_token)
-
-    if start is None:
-        raise _missing_value_error(name_token, "START")
-    if stop is None:
-        raise _missing_value_error(name_token, "STOP")
-    return Segment(
-        name=name_token,
-        start=start,
-        stop=stop,
-        error_mode=error_mode,
-        error=error,
-        error_min=error_min,
-    )
-
-
 def validate_error_mode(inp: str) -> ErrorModes:
     if inp == "REL":
         return ErrorModes.REL
@@ -415,6 +413,19 @@ def validate_positive_float(val: str, key: str) -> float:
     return v
 
 
+def validate_localization(val: dict[str, Any], obs_name: str) -> None:
+    errors = []
+    if "EAST" not in val:
+        errors.append(_missing_value_error(f"LOCALIZATION for {obs_name}", "EAST"))
+    if "NORTH" not in val:
+        errors.append(_missing_value_error(f"LOCALIZATION for {obs_name}", "NORTH"))
+    for key in val:
+        if key not in {"EAST", "NORTH", "RADIUS"}:
+            errors.append(_unknown_key_error(key, f"LOCALIZATION for {obs_name}"))
+    if errors:
+        raise ObservationConfigError.from_collected(errors)
+
+
 def validate_positive_int(val: str, key: str) -> int:
     try:
         v = int(val)

ert/config/_read_summary.py

@@ -8,7 +8,6 @@ from __future__ import annotations
 
 import fnmatch
 import re
-import warnings
 from collections.abc import Callable, Sequence
 from datetime import datetime, timedelta
 from enum import Enum, auto
@@ -158,10 +157,10 @@ def _read_spec(
             if kw.summary_variable == "TIME":
                 date_index = i
                 date_unit_str = kw.unit
-        except InvalidSummaryKeyError as err:
-            warnings.warn(
-                f"Found {err} in summary specification, key not loaded", stacklevel=2
-            )
+        except InvalidSummaryKeyError:
+            # InvalidSummaryKeyError will happen under normal conditions when
+            # the the number of wells set for WELLDIMS in the .DATA file is
+            # larger than the number of declared wells/groups/etc. These are skipped.
             continue
 
         if should_load_key(key):

ert/config/ert_config.py

@@ -7,7 +7,6 @@ import pprint
 import re
 from collections import Counter, defaultdict
 from collections.abc import Mapping
-from datetime import datetime
 from functools import cached_property
 from os import path
 from pathlib import Path
@@ -23,7 +22,6 @@ from ert.substitutions import Substitutions
 from ._create_observation_dataframes import create_observation_dataframes
 from ._design_matrix_validator import DesignMatrixValidator
 from ._observations import (
-    HistoryObservation,
     Observation,
     RFTObservation,
     SummaryObservation,
@@ -52,7 +50,6 @@ from .parsing import (
     ConfigWarning,
     ErrorInfo,
     ForwardModelStepKeys,
-    HistorySource,
     HookRuntime,
     ObservationConfigError,
     init_forward_model_schema,
@@ -62,7 +59,6 @@ from .parsing import (
 )
 from .parsing.observations_parser import ObservationDict
 from .queue_config import KnownQueueOptions, QueueConfig
-from .refcase import Refcase
 from .rft_config import RFTConfig
 from .workflow import Workflow
 from .workflow_fixtures import fixtures_per_hook
@@ -104,23 +100,6 @@ def _seed_sequence(seed: int | None) -> int:
     return int_seed
 
 
-def _read_time_map(file_contents: str) -> list[datetime]:
-    def str_to_datetime(date_str: str) -> datetime:
-        try:
-            return datetime.fromisoformat(date_str)
-        except ValueError:
-            logger.warning(
-                "DD/MM/YYYY date format is deprecated"
-                ", please use ISO date format YYYY-MM-DD."
-            )
-            return datetime.strptime(date_str, "%d/%m/%Y")
-
-    dates = []
-    for line in file_contents.splitlines():
-        dates.append(str_to_datetime(line.strip()))
-    return dates
-
-
 def create_forward_model_json(
     context: dict[str, str],
     forward_model_steps: list[ForwardModelStep],
@@ -733,9 +712,6 @@ class ErtConfig(BaseModel):
     user_config_file: str = "no_config"
     config_path: str = Field(init=False, default="")
     observation_declarations: list[Observation] = Field(default_factory=list)
-    time_map: list[datetime] | None = None
-    history_source: HistorySource = HistorySource.REFCASE_HISTORY
-    refcase: Refcase | None = None
     _observations: dict[str, pl.DataFrame] | None = PrivateAttr(None)
 
     @property
@@ -755,7 +731,6 @@ class ErtConfig(BaseModel):
                 )
             computed = create_observation_dataframes(
                 self.observation_declarations,
-                self.refcase,
                 cast(
                     GenDataConfig | None,
                     self.ensemble_config.response_configs.get("gen_data", None),
@@ -764,8 +739,6 @@ class ErtConfig(BaseModel):
                     RFTConfig | None,
                     self.ensemble_config.response_configs.get("rft", None),
                 ),
-                self.time_map,
-                self.history_source,
             )
             self._observations = computed
             return computed
@@ -1021,7 +994,7 @@ class ErtConfig(BaseModel):
                 summary_obs = {
                     obs.key
                     for obs in obs_configs
-                    if isinstance(obs, HistoryObservation | SummaryObservation)
+                    if isinstance(obs, SummaryObservation)
                 }
                 if summary_obs:
                     summary_keys = ErtConfig._read_summary_keys(config_dict)
@@ -1029,16 +1002,6 @@ class ErtConfig(BaseModel):
                         [key] for key in summary_obs if key not in summary_keys
                     ]
             ensemble_config = EnsembleConfig.from_dict(config_dict=config_dict)
-            time_map = None
-            if time_map_args := config_dict.get(ConfigKeys.TIME_MAP):
-                time_map_file, time_map_contents = time_map_args
-                try:
-                    time_map = _read_time_map(time_map_contents)
-                except ValueError as err:
-                    raise ConfigValidationError.with_context(
-                        f"Could not read timemap file {time_map_file}: {err}",
-                        time_map_file,
-                    ) from err
         except ConfigValidationError as err:
             errors.append(err)
         except PydanticValidationError as err:
@@ -1091,9 +1054,6 @@ class ErtConfig(BaseModel):
 
         env_vars = {}
         substituter = Substitutions(substitutions)
-        history_source = config_dict.get(
-            ConfigKeys.HISTORY_SOURCE, HistorySource.REFCASE_HISTORY
-        )
 
         # Insert env vars from plugins/site config
         for key, val in cls.ENV_VARS.items():
@@ -1129,7 +1089,6 @@ class ErtConfig(BaseModel):
                 prioritize_private_ip_address = user_prioritize_private_ip_address
 
         try:
-            refcase = Refcase.from_config_dict(config_dict)
             cls_config = cls(
                 substitutions=substitutions,
                 ensemble_config=ensemble_config,
@@ -1151,9 +1110,6 @@ class ErtConfig(BaseModel):
                 runpath_config=model_config,
                 user_config_file=config_file_path,
                 observation_declarations=list(obs_configs),
-                time_map=time_map,
-                history_source=history_source,
-                refcase=refcase,
                 prioritize_private_ip_address=prioritize_private_ip_address,
             )
 
@@ -1171,7 +1127,6 @@ class ErtConfig(BaseModel):
                 )
             cls_config._observations = create_observation_dataframes(
                 obs_configs,
-                refcase,
                 cast(
                     GenDataConfig | None,
                     ensemble_config.response_configs.get("gen_data", None),
@@ -1180,8 +1135,6 @@ class ErtConfig(BaseModel):
                     RFTConfig | None,
                     ensemble_config.response_configs.get("rft", None),
                 ),
-                time_map,
-                history_source,
             )
         except PydanticValidationError as err:
             raise ConfigValidationError.from_pydantic(err) from err