ert 19.0.0rc1__py3-none-any.whl → 19.0.0rc3__py3-none-any.whl

ert/config/_observations.py

@@ -1,14 +1,10 @@
-from __future__ import annotations
-
-import logging
 import os
 from collections.abc import Sequence
 from dataclasses import dataclass
 from enum import StrEnum
+from itertools import starmap
 from typing import Any, Self
 
-import pandas as pd
-
 from .parsing import (
     ErrorInfo,
     ObservationConfigError,
@@ -16,8 +12,6 @@ from .parsing import (
     ObservationType,
 )
 
-logger = logging.getLogger(__name__)
-
 
 class ErrorModes(StrEnum):
     REL = "REL"
@@ -25,52 +19,102 @@ class ErrorModes(StrEnum):
     RELMIN = "RELMIN"
 
 
+@dataclass
+class ObservationError:
+    error_mode: ErrorModes
+    error: float
+    error_min: float
+
+
+@dataclass
+class Segment(ObservationError):
+    name: str
+    start: int
+    stop: int
+
+
+@dataclass
+class HistoryObservation(ObservationError):
+    name: str
+    segments: list[Segment]
+
+    @property
+    def key(self) -> str:
+        """The :term:`summary key` to be fetched from :ref:`refcase`."""
+        # For history observations the key is also the name, ie.
+        # "HISTORY_OBSERVATION FOPR" means to add the values from
+        # the summary vector FOPRH in refcase as observations.
+        return self.name
+
+    @classmethod
+    def from_obs_dict(cls, directory: str, observation_dict: ObservationDict) -> Self:
+        error_mode = ErrorModes.RELMIN
+        error = 0.1
+        error_min = 0.1
+        segments = []
+        for key, value in observation_dict.items():
+            match key:
+                case "type" | "name":
+                    pass
+                case "ERROR":
+                    error = validate_positive_float(value, key)
+                case "ERROR_MIN":
+                    error_min = validate_positive_float(value, key)
+                case "ERROR_MODE":
+                    error_mode = validate_error_mode(value)
+                case "segments":
+                    segments = list(starmap(_validate_segment_dict, value))
+                case _:
+                    raise _unknown_key_error(str(key), observation_dict["name"])
+
+        return cls(
+            name=observation_dict["name"],
+            error_mode=error_mode,
+            error=error,
+            error_min=error_min,
+            segments=segments,
+        )
+
+
+@dataclass
+class ObservationDate:
+    days: float | None = None
+    hours: float | None = None
+    date: str | None = None
+    restart: int | None = None
+
+
 @dataclass
 class _SummaryValues:
     name: str
     value: float
     key: str  #: The :term:`summary key` in the summary response
-    date: str
     location_x: float | None = None
     location_y: float | None = None
     location_range: float | None = None
 
 
 @dataclass
-class ObservationError:
-    error_mode: ErrorModes
-    error: float
-    error_min: float
-
-
-@dataclass
-class SummaryObservation(_SummaryValues, ObservationError):
+class SummaryObservation(ObservationDate, _SummaryValues, ObservationError):
     @classmethod
-    def from_obs_dict(
-        cls, directory: str, observation_dict: ObservationDict
-    ) -> list[Self]:
+    def from_obs_dict(cls, directory: str, observation_dict: ObservationDict) -> Self:
         error_mode = ErrorModes.ABS
         summary_key = None
 
-        date: str | None = None
+        date_dict: ObservationDate = ObservationDate()
         float_values: dict[str, float] = {"ERROR_MIN": 0.1}
-        localization_values: dict[str, float | None] = {}
+        localization_values: dict[str, float] = {}
         for key, value in observation_dict.items():
             match key:
                 case "type" | "name":
                     pass
+                case "RESTART":
+                    date_dict.restart = validate_positive_int(value, key)
                 case "ERROR" | "ERROR_MIN":
                     float_values[str(key)] = validate_positive_float(value, key)
-                case "DAYS" | "HOURS" | "RESTART":
-                    raise ObservationConfigError.with_context(
-                        (
-                            "SUMMARY_OBSERVATION must use DATE to specify "
-                            "date, DAYS | HOURS is no longer allowed. "
-                            "Please run:\n ert convert_observations "
-                            "<your_ert_config.ert>\nto migrate the observation config "
-                            "to use the correct format."
-                        ),
-                        key,
+                case "DAYS" | "HOURS":
+                    setattr(
+                        date_dict, str(key).lower(), validate_positive_float(value, key)
                     )
                 case "VALUE":
                     float_values[str(key)] = validate_float(value, key)
@@ -79,16 +123,13 @@ class SummaryObservation(_SummaryValues, ObservationError):
                 case "KEY":
                     summary_key = value
                 case "DATE":
-                    date = value
-                case "LOCALIZATION":
-                    validate_localization(value, observation_dict["name"])
-                    localization_values["x"] = validate_float(value["EAST"], key)
-                    localization_values["y"] = validate_float(value["NORTH"], key)
-                    localization_values["range"] = (
-                        validate_float(value["RADIUS"], key)
-                        if "RADIUS" in value
-                        else None
-                    )
+                    date_dict.date = value
+                case "LOCATION_X":
+                    localization_values["x"] = validate_float(value, key)
+                case "LOCATION_Y":
+                    localization_values["y"] = validate_float(value, key)
+                case "LOCATION_RANGE":
+                    localization_values["range"] = validate_float(value, key)
                 case _:
                     raise _unknown_key_error(str(key), observation_dict["name"])
         if "VALUE" not in float_values:
@@ -98,21 +139,18 @@ class SummaryObservation(_SummaryValues, ObservationError):
         if "ERROR" not in float_values:
             raise _missing_value_error(observation_dict["name"], "ERROR")
 
-        assert date is not None
-        return [
-            cls(
-                name=observation_dict["name"],
-                error_mode=error_mode,
-                error=float_values["ERROR"],
-                error_min=float_values["ERROR_MIN"],
-                key=summary_key,
-                value=float_values["VALUE"],
-                location_x=localization_values.get("x"),
-                location_y=localization_values.get("y"),
-                location_range=localization_values.get("range"),
-                date=date,
-            )
-        ]
+        return cls(
+            name=observation_dict["name"],
+            error_mode=error_mode,
+            error=float_values["ERROR"],
+            error_min=float_values["ERROR_MIN"],
+            key=summary_key,
+            value=float_values["VALUE"],
+            location_x=localization_values.get("x"),
+            location_y=localization_values.get("y"),
+            location_range=localization_values.get("range"),
+            **date_dict.__dict__,
+        )
 
 
 @dataclass
@@ -124,15 +162,12 @@ class _GeneralObservation:
     index_list: str | None = None
     index_file: str | None = None
     obs_file: str | None = None
-    restart: int | None = None
 
 
 @dataclass
-class GeneralObservation(_GeneralObservation):
+class GeneralObservation(ObservationDate, _GeneralObservation):
     @classmethod
-    def from_obs_dict(
-        cls, directory: str, observation_dict: ObservationDict
-    ) -> list[Self]:
+    def from_obs_dict(cls, directory: str, observation_dict: ObservationDict) -> Self:
         try:
             data = observation_dict["DATA"]
         except KeyError as err:
@@ -147,22 +182,12 @@ class GeneralObservation(_GeneralObservation):
                     output.restart = validate_positive_int(value, key)
                 case "VALUE":
                     output.value = validate_float(value, key)
-                case "ERROR":
+                case "ERROR" | "DAYS" | "HOURS":
                     setattr(
                         output, str(key).lower(), validate_positive_float(value, key)
                     )
-                case "DATE" | "DAYS" | "HOURS":
-                    raise ObservationConfigError.with_context(
-                        (
-                            "GENERAL_OBSERVATION must use RESTART to specify "
-                            "report step. Please run:\n ert convert_observations "
-                            "<your_ert_config.ert>\nto migrate the observation config "
-                            "to use the correct format."
-                        ),
-                        key,
-                    )
-                case "INDEX_LIST":
-                    output.index_list = value
+                case "DATE" | "INDEX_LIST":
+                    setattr(output, str(key).lower(), value)
                 case "OBS_FILE" | "INDEX_FILE":
                     assert not isinstance(key, tuple)
                     filename = value
@@ -185,8 +210,7 @@ class GeneralObservation(_GeneralObservation):
                 f" VALUE = {output.value}, ERROR must also be given.",
                 observation_dict["name"],
             )
-
-        return [output]
+        return output
 
 
 @dataclass
@@ -202,63 +226,7 @@ class RFTObservation:
     tvd: float
 
     @classmethod
-    def from_csv(
-        cls,
-        directory: str,
-        observation_dict: ObservationDict,
-        filename: str,
-        observed_property: str = "PRESSURE",
-    ) -> list[Self]:
-        if not os.path.isabs(filename):
-            filename = os.path.join(directory, filename)
-        if not os.path.exists(filename):
-            raise ObservationConfigError.with_context(
-                f"The CSV file ({filename}) does not exist or is not accessible.",
-                filename,
-            )
-        csv_file = pd.read_csv(
-            filename,
-            encoding="utf-8",
-            on_bad_lines="error",
-        )
-
-        required_columns = {
-            "WELL_NAME",
-            "DATE",
-            observed_property,
-            "ERROR",
-            "NORTH",
-            "EAST",
-            "TVD",
-        }
-        missing_required_columns = required_columns - set(csv_file.keys())
-        if missing_required_columns:
-            raise ObservationConfigError.with_context(
-                f"The rft observations file {filename} is missing required column(s) "
-                f"{', '.join(sorted(missing_required_columns))}.",
-                filename,
-            )
-
-        return [
-            cls(
-                f"{observation_dict['name']}[{row.Index}]",
-                str(row.WELL_NAME),
-                str(row.DATE),
-                observed_property,
-                validate_float(str(getattr(row, observed_property)), observed_property),
-                validate_float(str(row.ERROR), "ERROR"),
-                validate_float(str(row.NORTH), "NORTH"),
-                validate_float(str(row.EAST), "EAST"),
-                validate_float(str(row.TVD), "TVD"),
-            )
-            for row in csv_file.itertuples(index=True)
-        ]
-
-    @classmethod
-    def from_obs_dict(
-        cls, directory: str, observation_dict: ObservationDict
-    ) -> list[Self]:
-        csv_filename = None
+    def from_obs_dict(cls, directory: str, observation_dict: ObservationDict) -> Self:
         well = None
         observed_property = None
         observed_value = None
@@ -287,17 +255,8 @@ class RFTObservation:
                     east = validate_float(value, key)
                 case "TVD":
                     tvd = validate_float(value, key)
-                case "CSV":
-                    csv_filename = value
                 case _:
                     raise _unknown_key_error(str(key), observation_dict["name"])
-        if csv_filename is not None:
-            return cls.from_csv(
-                directory,
-                observation_dict,
-                csv_filename,
-                observed_property or "PRESSURE",
-            )
         if well is None:
             raise _missing_value_error(observation_dict["name"], "WELL")
         if observed_value is None:
@@ -314,24 +273,25 @@ class RFTObservation:
             raise _missing_value_error(observation_dict["name"], "EAST")
         if tvd is None:
             raise _missing_value_error(observation_dict["name"], "TVD")
-        return [
-            cls(
-                observation_dict["name"],
-                well,
-                date,
-                observed_property,
-                observed_value,
-                error,
-                north,
-                east,
-                tvd,
-            )
-        ]
+        return cls(
+            observation_dict["name"],
+            well,
+            date,
+            observed_property,
+            observed_value,
+            error,
+            north,
+            east,
+            tvd,
+        )
 
 
-Observation = SummaryObservation | GeneralObservation | RFTObservation
+Observation = (
+    HistoryObservation | SummaryObservation | GeneralObservation | RFTObservation
+)
 
 _TYPE_TO_CLASS: dict[ObservationType, type[Observation]] = {
+    ObservationType.HISTORY: HistoryObservation,
     ObservationType.SUMMARY: SummaryObservation,
     ObservationType.GENERAL: GeneralObservation,
     ObservationType.RFT: RFTObservation,
@@ -349,20 +309,10 @@ def make_observations(
         inp: The collection of statements to validate.
     """
     result: list[Observation] = []
-    error_list: list[ErrorInfo | ObservationConfigError] = []
+    error_list: list[ErrorInfo] = []
     for obs_dict in observation_dicts:
-        if obs_dict["type"] == ObservationType.HISTORY:
-            msg = (
-                "HISTORY_OBSERVATION is deprecated, and must be specified "
-                "as SUMMARY_OBSERVATION. Run"
-                " ert convert_observations <ert_config.ert> to convert your "
-                "observations automatically"
-            )
-            logger.error(msg)
-            error_list.append(ObservationConfigError(msg))
-            continue
         try:
-            result.extend(
+            result.append(
                 _TYPE_TO_CLASS[obs_dict["type"]].from_obs_dict(directory, obs_dict)
             )
         except KeyError as err:
@@ -376,6 +326,41 @@ def make_observations(
     return result
 
 
+def _validate_segment_dict(name_token: str, inp: dict[str, Any]) -> Segment:
+    start = None
+    stop = None
+    error_mode = ErrorModes.RELMIN
+    error = 0.1
+    error_min = 0.1
+    for key, value in inp.items():
+        match key:
+            case "START":
+                start = validate_int(value, key)
+            case "STOP":
+                stop = validate_int(value, key)
+            case "ERROR":
+                error = validate_positive_float(value, key)
+            case "ERROR_MIN":
+                error_min = validate_positive_float(value, key)
+            case "ERROR_MODE":
+                error_mode = validate_error_mode(value)
+            case _:
+                raise _unknown_key_error(key, name_token)
+
+    if start is None:
+        raise _missing_value_error(name_token, "START")
+    if stop is None:
+        raise _missing_value_error(name_token, "STOP")
+    return Segment(
+        name=name_token,
+        start=start,
+        stop=stop,
+        error_mode=error_mode,
+        error=error,
+        error_min=error_min,
+    )
+
+
 def validate_error_mode(inp: str) -> ErrorModes:
     if inp == "REL":
         return ErrorModes.REL
@@ -413,19 +398,6 @@ def validate_positive_float(val: str, key: str) -> float:
     return v
 
 
-def validate_localization(val: dict[str, Any], obs_name: str) -> None:
-    errors = []
-    if "EAST" not in val:
-        errors.append(_missing_value_error(f"LOCALIZATION for {obs_name}", "EAST"))
-    if "NORTH" not in val:
-        errors.append(_missing_value_error(f"LOCALIZATION for {obs_name}", "NORTH"))
-    for key in val:
-        if key not in {"EAST", "NORTH", "RADIUS"}:
-            errors.append(_unknown_key_error(key, f"LOCALIZATION for {obs_name}"))
-    if errors:
-        raise ObservationConfigError.from_collected(errors)
-
-
 def validate_positive_int(val: str, key: str) -> int:
     try:
         v = int(val)

ert/config/_read_summary.py

@@ -8,6 +8,7 @@ from __future__ import annotations
 
 import fnmatch
 import re
+import warnings
 from collections.abc import Callable, Sequence
 from datetime import datetime, timedelta
 from enum import Enum, auto
@@ -157,10 +158,10 @@ def _read_spec(
             if kw.summary_variable == "TIME":
                 date_index = i
                 date_unit_str = kw.unit
-        except InvalidSummaryKeyError:
-            # InvalidSummaryKeyError will happen under normal conditions when
-            # the the number of wells set for WELLDIMS in the .DATA file is
-            # larger than the number of declared wells/groups/etc. These are skipped.
+        except InvalidSummaryKeyError as err:
+            warnings.warn(
+                f"Found {err} in summary specification, key not loaded", stacklevel=2
+            )
             continue
 
         if should_load_key(key):

ert/config/ert_config.py

@@ -7,6 +7,7 @@ import pprint
 import re
 from collections import Counter, defaultdict
 from collections.abc import Mapping
+from datetime import datetime
 from functools import cached_property
 from os import path
 from pathlib import Path
@@ -22,6 +23,7 @@ from ert.substitutions import Substitutions
 from ._create_observation_dataframes import create_observation_dataframes
 from ._design_matrix_validator import DesignMatrixValidator
 from ._observations import (
+    HistoryObservation,
     Observation,
     RFTObservation,
     SummaryObservation,
@@ -50,6 +52,7 @@ from .parsing import (
     ConfigWarning,
     ErrorInfo,
     ForwardModelStepKeys,
+    HistorySource,
     HookRuntime,
     ObservationConfigError,
     init_forward_model_schema,
@@ -59,6 +62,7 @@ from .parsing import (
 )
 from .parsing.observations_parser import ObservationDict
 from .queue_config import KnownQueueOptions, QueueConfig
+from .refcase import Refcase
 from .rft_config import RFTConfig
 from .workflow import Workflow
 from .workflow_fixtures import fixtures_per_hook
@@ -100,6 +104,23 @@ def _seed_sequence(seed: int | None) -> int:
     return int_seed
 
 
+def _read_time_map(file_contents: str) -> list[datetime]:
+    def str_to_datetime(date_str: str) -> datetime:
+        try:
+            return datetime.fromisoformat(date_str)
+        except ValueError:
+            logger.warning(
+                "DD/MM/YYYY date format is deprecated"
+                ", please use ISO date format YYYY-MM-DD."
+            )
+            return datetime.strptime(date_str, "%d/%m/%Y")
+
+    dates = []
+    for line in file_contents.splitlines():
+        dates.append(str_to_datetime(line.strip()))
+    return dates
+
+
 def create_forward_model_json(
     context: dict[str, str],
     forward_model_steps: list[ForwardModelStep],
@@ -667,6 +688,14 @@ def log_observation_keys(
         if key not in {"name", "type"}
     )
 
+    if "HISTORY_OBSERVATION" in observation_type_counts:
+        msg = (
+            "HISTORY_OBSERVATION is deprecated and will be removed. "
+            "Please use SUMMARY_OBSERVATION instead."
+        )
+        ConfigWarning.warn(msg)
+        logger.warning(msg)
+
     logger.info(
         f"Count of observation types:\n\t{dict(observation_type_counts)}\n"
         f"Count of observation keywords:\n\t{dict(observation_keyword_counts)}"
@@ -712,6 +741,9 @@ class ErtConfig(BaseModel):
     user_config_file: str = "no_config"
     config_path: str = Field(init=False, default="")
     observation_declarations: list[Observation] = Field(default_factory=list)
+    time_map: list[datetime] | None = None
+    history_source: HistorySource = HistorySource.REFCASE_HISTORY
+    refcase: Refcase | None = None
     _observations: dict[str, pl.DataFrame] | None = PrivateAttr(None)
 
     @property
@@ -731,6 +763,7 @@ class ErtConfig(BaseModel):
                 )
             computed = create_observation_dataframes(
                 self.observation_declarations,
+                self.refcase,
                 cast(
                     GenDataConfig | None,
                     self.ensemble_config.response_configs.get("gen_data", None),
@@ -739,6 +772,8 @@ class ErtConfig(BaseModel):
                     RFTConfig | None,
                     self.ensemble_config.response_configs.get("rft", None),
                 ),
+                self.time_map,
+                self.history_source,
             )
             self._observations = computed
             return computed
@@ -994,7 +1029,7 @@ class ErtConfig(BaseModel):
                 summary_obs = {
                     obs.key
                     for obs in obs_configs
-                    if isinstance(obs, SummaryObservation)
+                    if isinstance(obs, HistoryObservation | SummaryObservation)
                 }
                 if summary_obs:
                     summary_keys = ErtConfig._read_summary_keys(config_dict)
@@ -1002,6 +1037,16 @@ class ErtConfig(BaseModel):
                         [key] for key in summary_obs if key not in summary_keys
                     ]
             ensemble_config = EnsembleConfig.from_dict(config_dict=config_dict)
+            time_map = None
+            if time_map_args := config_dict.get(ConfigKeys.TIME_MAP):
+                time_map_file, time_map_contents = time_map_args
+                try:
+                    time_map = _read_time_map(time_map_contents)
+                except ValueError as err:
+                    raise ConfigValidationError.with_context(
+                        f"Could not read timemap file {time_map_file}: {err}",
+                        time_map_file,
+                    ) from err
         except ConfigValidationError as err:
             errors.append(err)
         except PydanticValidationError as err:
@@ -1054,6 +1099,9 @@ class ErtConfig(BaseModel):
 
         env_vars = {}
         substituter = Substitutions(substitutions)
+        history_source = config_dict.get(
+            ConfigKeys.HISTORY_SOURCE, HistorySource.REFCASE_HISTORY
+        )
 
         # Insert env vars from plugins/site config
         for key, val in cls.ENV_VARS.items():
@@ -1089,6 +1137,7 @@ class ErtConfig(BaseModel):
                 prioritize_private_ip_address = user_prioritize_private_ip_address
 
         try:
+            refcase = Refcase.from_config_dict(config_dict)
             cls_config = cls(
                 substitutions=substitutions,
                 ensemble_config=ensemble_config,
@@ -1110,6 +1159,9 @@ class ErtConfig(BaseModel):
                 runpath_config=model_config,
                 user_config_file=config_file_path,
                 observation_declarations=list(obs_configs),
+                time_map=time_map,
+                history_source=history_source,
+                refcase=refcase,
                 prioritize_private_ip_address=prioritize_private_ip_address,
             )
 
@@ -1127,6 +1179,7 @@ class ErtConfig(BaseModel):
                 )
             cls_config._observations = create_observation_dataframes(
                 obs_configs,
+                refcase,
                 cast(
                     GenDataConfig | None,
                     ensemble_config.response_configs.get("gen_data", None),
@@ -1135,6 +1188,8 @@ class ErtConfig(BaseModel):
                     RFTConfig | None,
                     ensemble_config.response_configs.get("rft", None),
                 ),
+                time_map,
+                history_source,
             )
         except PydanticValidationError as err:
             raise ConfigValidationError.from_pydantic(err) from err

ert/config/parsing/observations_parser.py

@@ -138,7 +138,6 @@ observations_parser = Lark(
     PARAMETER_NAME : CHAR+
     object : "{" [(declaration";")*] "}"
     ?declaration: "SEGMENT" STRING object -> segment
-                | "LOCALIZATION" object -> localization
                 | pair
     pair   : PARAMETER_NAME "=" value
 
@@ -194,11 +193,6 @@ class TreeToObservations(Transformer[FileContextToken, list[ObservationDict]]):
     def segment(tree):
         return (("SEGMENT", tree[0]), tree[1])
 
-    @staticmethod
-    @no_type_check
-    def localization(tree):
-        return ("LOCALIZATION", tree[0])
-
     @staticmethod
     @no_type_check
     def object(tree):

ert/config/rft_config.py

@@ -194,7 +194,7 @@ class RFTConfig(ResponseConfig):
                     .explode("location")
                     for (well, time), inner_dict in fetched.items()
                     for prop, vals in inner_dict.items()
-                    if prop != "DEPTH" and len(vals) > 0
+                    if prop != "DEPTH"
                 ]
             )
         except KeyError as err: