ert 19.0.1__py3-none-any.whl → 20.0.0b1__py3-none-any.whl

ert/config/_read_summary.py

@@ -8,7 +8,6 @@ from __future__ import annotations
 
 import fnmatch
 import re
-import warnings
 from collections.abc import Callable, Sequence
 from datetime import datetime, timedelta
 from enum import Enum, auto
@@ -158,10 +157,10 @@ def _read_spec(
             if kw.summary_variable == "TIME":
                 date_index = i
                 date_unit_str = kw.unit
-        except InvalidSummaryKeyError as err:
-            warnings.warn(
-                f"Found {err} in summary specification, key not loaded", stacklevel=2
-            )
+        except InvalidSummaryKeyError:
+            # InvalidSummaryKeyError will happen under normal conditions when
+            # the the number of wells set for WELLDIMS in the .DATA file is
+            # larger than the number of declared wells/groups/etc. These are skipped.
             continue
 
         if should_load_key(key):

ert/config/ert_config.py

@@ -7,23 +7,21 @@ import pprint
 import re
 from collections import Counter, defaultdict
 from collections.abc import Mapping
-from datetime import datetime
 from functools import cached_property
 from os import path
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, ClassVar, Self, cast, overload
 
-import polars as pl
 from numpy.random import SeedSequence
-from pydantic import BaseModel, Field, PrivateAttr, model_validator
+from pydantic import BaseModel, Field, model_validator
 from pydantic import ValidationError as PydanticValidationError
 
+from ert.config._create_observation_dataframes import create_observation_dataframes
 from ert.substitutions import Substitutions
 
-from ._create_observation_dataframes import create_observation_dataframes
 from ._design_matrix_validator import DesignMatrixValidator
 from ._observations import (
-    HistoryObservation,
+    GeneralObservation,
     Observation,
     RFTObservation,
     SummaryObservation,
@@ -52,7 +50,6 @@ from .parsing import (
     ConfigWarning,
     ErrorInfo,
     ForwardModelStepKeys,
-    HistorySource,
     HookRuntime,
     ObservationConfigError,
     init_forward_model_schema,
@@ -62,7 +59,6 @@ from .parsing import (
 )
 from .parsing.observations_parser import ObservationDict
 from .queue_config import KnownQueueOptions, QueueConfig
-from .refcase import Refcase
 from .rft_config import RFTConfig
 from .workflow import Workflow
 from .workflow_fixtures import fixtures_per_hook
@@ -104,23 +100,6 @@ def _seed_sequence(seed: int | None) -> int:
     return int_seed
 
 
-def _read_time_map(file_contents: str) -> list[datetime]:
-    def str_to_datetime(date_str: str) -> datetime:
-        try:
-            return datetime.fromisoformat(date_str)
-        except ValueError:
-            logger.warning(
-                "DD/MM/YYYY date format is deprecated"
-                ", please use ISO date format YYYY-MM-DD."
-            )
-            return datetime.strptime(date_str, "%d/%m/%Y")
-
-    dates = []
-    for line in file_contents.splitlines():
-        dates.append(str_to_datetime(line.strip()))
-    return dates
-
-
 def create_forward_model_json(
     context: dict[str, str],
     forward_model_steps: list[ForwardModelStep],
@@ -688,14 +667,6 @@ def log_observation_keys(
         if key not in {"name", "type"}
     )
 
-    if "HISTORY_OBSERVATION" in observation_type_counts:
-        msg = (
-            "HISTORY_OBSERVATION is deprecated and will be removed. "
-            "Please use SUMMARY_OBSERVATION instead."
-        )
-        ConfigWarning.warn(msg)
-        logger.warning(msg)
-
     logger.info(
         f"Count of observation types:\n\t{dict(observation_type_counts)}\n"
         f"Count of observation keywords:\n\t{dict(observation_keyword_counts)}"
@@ -739,43 +710,7 @@ class ErtConfig(BaseModel):
     user_config_file: str = "no_config"
     config_path: str = Field(init=False, default="")
     observation_declarations: list[Observation] = Field(default_factory=list)
-    time_map: list[datetime] | None = None
-    history_source: HistorySource = HistorySource.REFCASE_HISTORY
-    refcase: Refcase | None = None
-    _observations: dict[str, pl.DataFrame] | None = PrivateAttr(None)
-
-    @property
-    def observations(self) -> dict[str, pl.DataFrame]:
-        if self._observations is None:
-            has_rft_observations = any(
-                isinstance(o, RFTObservation) for o in self.observation_declarations
-            )
-            if (
-                has_rft_observations
-                and "rft" not in self.ensemble_config.response_configs
-            ):
-                self.ensemble_config.response_configs["rft"] = RFTConfig(
-                    input_files=[self.runpath_config.eclbase_format_string],
-                    data_to_read={},
-                    locations=[],
-                )
-            computed = create_observation_dataframes(
-                self.observation_declarations,
-                self.refcase,
-                cast(
-                    GenDataConfig | None,
-                    self.ensemble_config.response_configs.get("gen_data", None),
-                ),
-                cast(
-                    RFTConfig | None,
-                    self.ensemble_config.response_configs.get("rft", None),
-                ),
-                self.time_map,
-                self.history_source,
-            )
-            self._observations = computed
-            return computed
-        return self._observations
+    zonemap: dict[int, list[str]] = Field(default_factory=dict)
 
     @model_validator(mode="after")
     def set_fields(self) -> Self:
@@ -830,27 +765,45 @@ class ErtConfig(BaseModel):
         )
         return self
 
-    def __eq__(self, other: object) -> bool:
-        if not isinstance(other, ErtConfig):
-            return False
-
-        for attr in vars(self):
-            if attr == "observations":
-                if self.observations.keys() != other.observations.keys():
-                    return False
-
-                if not all(
-                    self.observations[k].equals(other.observations[k])
-                    for k in self.observations
-                ):
-                    return False
+    @model_validator(mode="after")
+    def validate_observations_against_responses(self) -> Self:
+        gen_data_config = cast(
+            GenDataConfig | None,
+            self.ensemble_config.response_configs.get("gen_data", None),
+        )
 
-                continue
+        errors: list[ErrorInfo] = []
+        for obs in self.observation_declarations:
+            if isinstance(obs, GeneralObservation):
+                response_key = obs.data
+                if gen_data_config is None or response_key not in gen_data_config.keys:
+                    errors.append(
+                        ErrorInfo(
+                            message=(
+                                f"Problem with GENERAL_OBSERVATION {obs.name}:"
+                                f" No GEN_DATA with name {response_key!r} found"
+                            )
+                        ).set_context(response_key)
+                    )
+                    continue
+                assert isinstance(gen_data_config, GenDataConfig)
+                _, report_steps = gen_data_config.get_args_for_key(response_key)
+                response_report_steps = [] if report_steps is None else report_steps
+                if response_report_steps and obs.restart not in response_report_steps:
+                    errors.append(
+                        ErrorInfo(
+                            message=(
+                                f"The GEN_DATA node:{response_key} is not configured "
+                                f"to load from report step:{obs.restart} for the "
+                                f"observation:{obs.name}"
+                            )
+                        ).set_context(response_key)
+                    )
 
-            if getattr(self, attr) != getattr(other, attr):
-                return False
+        if errors:
+            raise ConfigValidationError.from_collected(errors)
 
-        return True
+        return self
 
     @staticmethod
     def with_plugins(runtime_plugins: ErtRuntimePlugins) -> type[ErtConfig]:
@@ -1024,7 +977,7 @@ class ErtConfig(BaseModel):
                 summary_obs = {
                     obs.key
                     for obs in obs_configs
-                    if isinstance(obs, HistoryObservation | SummaryObservation)
+                    if isinstance(obs, SummaryObservation)
                 }
                 if summary_obs:
                     summary_keys = ErtConfig._read_summary_keys(config_dict)
@@ -1032,16 +985,6 @@ class ErtConfig(BaseModel):
                         [key] for key in summary_obs if key not in summary_keys
                     ]
             ensemble_config = EnsembleConfig.from_dict(config_dict=config_dict)
-            time_map = None
-            if time_map_args := config_dict.get(ConfigKeys.TIME_MAP):
-                time_map_file, time_map_contents = time_map_args
-                try:
-                    time_map = _read_time_map(time_map_contents)
-                except ValueError as err:
-                    raise ConfigValidationError.with_context(
-                        f"Could not read timemap file {time_map_file}: {err}",
-                        time_map_file,
-                    ) from err
         except ConfigValidationError as err:
             errors.append(err)
         except PydanticValidationError as err:
@@ -1094,9 +1037,6 @@ class ErtConfig(BaseModel):
 
         env_vars = {}
         substituter = Substitutions(substitutions)
-        history_source = config_dict.get(
-            ConfigKeys.HISTORY_SOURCE, HistorySource.REFCASE_HISTORY
-        )
 
         # Insert env vars from plugins/site config
         for key, val in cls.ENV_VARS.items():
@@ -1118,8 +1058,10 @@ class ErtConfig(BaseModel):
             user_configured_.add(key)
             env_vars[key] = substituter.substitute(val)
 
+        if errors:
+            raise ObservationConfigError.from_collected(errors)
+
         try:
-            refcase = Refcase.from_config_dict(config_dict)
             cls_config = cls(
                 substitutions=substitutions,
                 ensemble_config=ensemble_config,
@@ -1141,9 +1083,7 @@ class ErtConfig(BaseModel):
                 runpath_config=model_config,
                 user_config_file=config_file_path,
                 observation_declarations=list(obs_configs),
-                time_map=time_map,
-                history_source=history_source,
-                refcase=refcase,
+                zonemap=config_dict.get(ConfigKeys.ZONEMAP, ("", {}))[1],
             )
 
             # The observations are created here because create_observation_dataframes
@@ -1157,20 +1097,15 @@ class ErtConfig(BaseModel):
                     input_files=[eclbase],
                     data_to_read={},
                     locations=[],
+                    zonemap=cls_config.zonemap,
                 )
-            cls_config._observations = create_observation_dataframes(
+
+            # PS:
+            # This mutates the rft config and is necessary for the moment
+            # Consider changing this pattern
+            _ = create_observation_dataframes(
                 obs_configs,
-                refcase,
-                cast(
-                    GenDataConfig | None,
-                    ensemble_config.response_configs.get("gen_data", None),
-                ),
-                cast(
-                    RFTConfig | None,
-                    ensemble_config.response_configs.get("rft", None),
-                ),
-                time_map,
-                history_source,
+                cast(RFTConfig | None, ensemble_config.response_configs.get("rft")),
             )
         except PydanticValidationError as err:
             raise ConfigValidationError.from_pydantic(err) from err

ert/config/everest_control.py

@@ -11,13 +11,14 @@ from typing import TYPE_CHECKING, Any, Literal, Self
 
 import networkx as nx
 import numpy as np
+import polars as pl
 import xarray as xr
 from pydantic import BaseModel, ConfigDict, Field, model_validator
 from ropt.workflow import find_sampler_plugin
 
 from ert.substitutions import substitute_runpath_name
 
-from .parameter_config import ParameterConfig
+from .parameter_config import ParameterCardinality, ParameterConfig
 
 if TYPE_CHECKING:
     import numpy.typing as npt
@@ -175,31 +176,47 @@ class EverestControl(ParameterConfig):
     def parameter_keys(self) -> list[str]:
         return self.input_keys
 
+    @property
+    def cardinality(self) -> ParameterCardinality:
+        return ParameterCardinality.multiple_configs_per_ensemble_dataset
+
     def read_from_runpath(
         self, run_path: Path, real_nr: int, iteration: int
     ) -> xr.Dataset:
         raise NotImplementedError
 
+    def load_parameters(
+        self, ensemble: Ensemble, realizations: npt.NDArray[np.int_]
+    ) -> npt.NDArray[np.float64]:
+        raise NotImplementedError
+
+    def load_parameter_graph(self) -> nx.Graph[int]:
+        raise NotImplementedError
+
+    def __len__(self) -> int:
+        return len(self.input_keys)
+
     def write_to_runpath(
         self, run_path: Path, real_nr: int, ensemble: Ensemble
     ) -> None:
-        file_path = run_path / substitute_runpath_name(
+        file_path: Path = run_path / substitute_runpath_name(
             self.output_file, real_nr, ensemble.iteration
         )
         Path.mkdir(file_path.parent, exist_ok=True, parents=True)
 
-        data: MutableDataType = {}
-        for da in ensemble.load_parameters(self.name, real_nr)["values"]:
-            assert isinstance(da, xr.DataArray)
-            name = str(da.names.values)
-            try:
-                outer, inner = name.split("\0")
-
-                if outer not in data:
-                    data[outer] = {}
-                data[outer][inner] = float(da)  # type: ignore
-            except ValueError:
-                data[name] = float(da)
+        data: dict[str, Any] = {}
+        df = ensemble.load_parameters(self.name, real_nr)
+        assert isinstance(df, pl.DataFrame)
+        df = df.drop("realization")
+        df = df.rename({col: col.replace(f"{self.name}.", "", 1) for col in df.columns})
+        for c in df.columns:
+            if "." in c:
+                top_key, sub_key = c.split(".", 1)
+                if top_key not in data:
+                    data[top_key] = {}
+                data[top_key][sub_key] = df[c].item()
+            else:
+                data[c] = df[c].item()
 
         file_path.write_text(json.dumps(data), encoding="utf-8")
 
@@ -207,28 +224,12 @@ class EverestControl(ParameterConfig):
         self,
         from_data: npt.NDArray[np.float64],
         iens_active_index: npt.NDArray[np.int_],
-    ) -> Iterator[tuple[int, xr.Dataset]]:
-        for i, realization in enumerate(iens_active_index):
-            yield (
-                int(realization),
-                xr.Dataset(
-                    {
-                        "values": ("names", from_data[:, i]),
-                        "names": [
-                            x.split(f"{self.name}.")[1].replace(".", "\0")
-                            for x in self.parameter_keys
-                        ],
-                    }
-                ),
-            )
-
-    def load_parameters(
-        self, ensemble: Ensemble, realizations: npt.NDArray[np.int_]
-    ) -> npt.NDArray[np.float64]:
-        raise NotImplementedError
-
-    def load_parameter_graph(self) -> nx.Graph[int]:
-        raise NotImplementedError
-
-    def __len__(self) -> int:
-        return len(self.input_keys)
+    ) -> Iterator[tuple[None, pl.DataFrame]]:
+        df = pl.DataFrame(
+            {
+                "realization": iens_active_index,
+                **{k: from_data[:, i] for i, k in enumerate(self.parameter_keys)},
+            },
+            strict=False,
+        )
+        yield None, df

ert/config/everest_response.py

@@ -7,7 +7,7 @@ import polars as pl
 from ert.substitutions import substitute_runpath_name
 
 from .parsing import ConfigDict
-from .response_config import InvalidResponseFile, ResponseConfig, ResponseMetadata
+from .response_config import InvalidResponseFile, ResponseConfig
 from .responses_index import responses_index
 
 
@@ -21,18 +21,6 @@ class EverestResponse(ResponseConfig):
     def primary_key(self) -> list[str]:
         return []
 
-    @property
-    def metadata(self) -> list[ResponseMetadata]:
-        return [
-            ResponseMetadata(
-                response_type=self.type,
-                response_key=response_key,
-                finalized=self.has_finalized_keys,
-                filter_on=None,
-            )
-            for response_key in self.keys
-        ]
-
     @property
     def expected_input_files(self) -> list[str]:
         return self.input_files
@@ -91,8 +79,8 @@ class EverestResponse(ResponseConfig):
 class EverestConstraintsConfig(EverestResponse):
     type: Literal["everest_constraints"] = "everest_constraints"
     targets: list[float | None]
-    upper_bounds: list[float]
-    lower_bounds: list[float]
+    upper_bounds: list[float | None]
+    lower_bounds: list[float | None]
 
 
 responses_index.add_response_type(EverestConstraintsConfig)

ert/config/field.py

@@ -17,13 +17,10 @@ from ert.field_utils import (
     ErtboxParameters,
     FieldFileFormat,
     Shape,
-    calc_rho_for_2d_grid_layer,
     calculate_ertbox_parameters,
     get_shape,
     read_field,
     save_field,
-    transform_local_ellipse_angle_to_local_coords,
-    transform_positions_to_local_field_coordinates,
 )
 from ert.substitutions import substitute_runpath_name
 from ert.utils import log_duration
@@ -73,10 +70,10 @@ class Field(ParameterConfig):
     dimensionality: Literal[3] = 3
     ertbox_params: ErtboxParameters
     file_format: FieldFileFormat
-    output_transformation: str | None
-    input_transformation: str | None
-    truncation_min: float | None
-    truncation_max: float | None
+    output_transformation: str | None = None
+    input_transformation: str | None = None
+    truncation_min: float | None = None
+    truncation_max: float | None = None
     forward_init_file: str
     output_file: Path
     grid_file: str
@@ -318,75 +315,6 @@ class Field(ParameterConfig):
     def nz(self) -> int:
         return self.ertbox_params.nz
 
-    def calc_rho_for_2d_grid_layer(
-        self,
-        obs_xpos: npt.NDArray[np.float64],
-        obs_ypos: npt.NDArray[np.float64],
-        obs_main_range: npt.NDArray[np.float64],
-        obs_perp_range: npt.NDArray[np.float64],
-        obs_anisotropy_angle: npt.NDArray[np.float64],
-        right_handed_grid_indexing: bool = True,
-    ) -> npt.NDArray[np.float64]:
-        """Function to calculate scaling values to be used in the RHO matrix
-        for distance-based localization.
-
-        Args:
-            obs_xpos: x-coordinates in global coordinates of observations
-            obs_ypos: y-coordinates in global coordinates of observations
-            obs_main_range: Size of influence ellipse main principal direction.
-            obs_perp_range: Size of influence ellipse second principal direction.
-            obs_anisotropy_angle: Rotation angle anticlock wise of main principal
-              direction of influence ellipse relative to global coordinate
-              system's x-axis.
-            right_handed_grid_indexing: When this is True the field parameters
-              grid index order counts J-index down from ny-1 to 0.
-              If the value is False, the grid index order is to count J index
-              from 0 to ny-1. As standard for 3D field parameters,
-              the grid index order follows the right_handed grid indexing.
-
-        Returns:
-            Scaling values (elements of the RHO matrix) as a numpy array
-              of shape=(nx,ny,nobservations)
-
-        """
-        # Can only be used if ertbox coordinate system is defined
-        assert self.ertbox_params.xinc is not None, (
-            "Parameter for grid resolution must be defined"
-        )
-        assert self.ertbox_params.yinc is not None, (
-            "Parameter for grid resolution must be defined"
-        )
-        assert self.ertbox_params.origin is not None, (
-            "Parameter for grid origin must be defined"
-        )
-        assert self.ertbox_params.rotation_angle is not None, (
-            "Parameter for grid rotation must be defined"
-        )
-        # Transform positions of observations into local coordinates
-        xpos, ypos = transform_positions_to_local_field_coordinates(
-            self.ertbox_params.origin,
-            self.ertbox_params.rotation_angle,
-            obs_xpos,
-            obs_ypos,
-        )
-        # Transform localization ellipse orientation to local coordinates
-        ellipse_rotation = transform_local_ellipse_angle_to_local_coords(
-            self.ertbox_params.rotation_angle, obs_anisotropy_angle
-        )
-
-        return calc_rho_for_2d_grid_layer(
-            self.ertbox_params.nx,
-            self.ertbox_params.ny,
-            self.ertbox_params.xinc,
-            self.ertbox_params.yinc,
-            xpos,
-            ypos,
-            obs_main_range,
-            obs_perp_range,
-            ellipse_rotation,
-            right_handed_grid_indexing=right_handed_grid_indexing,
-        )
-
 
 TRANSFORM_FUNCTIONS: Final[dict[str, Callable[[Any], Any]]] = {
     "LN": np.log,

ert/config/forward_model_step.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import inspect
 import logging
 from typing import (
     TYPE_CHECKING,
@@ -95,9 +96,24 @@ class ForwardModelStepOptions(TypedDict, total=False):
     required_keywords: NotRequired[list[str]]
 
 
+def _get_source_package() -> str:
+    """Return the top-level package name of the calling forward model step.
+
+    Finds the documentation() call (stack[2]) under the forward model step class
+    and return its parent module
+    """
+    stack = inspect.stack()
+    if len(stack) > 2:
+        caller_frame = stack[2]
+        caller_module = inspect.getmodule(caller_frame.frame)
+        if caller_module:
+            return caller_module.__name__.split(".")[0]
+    return "not found"
+
+
 class ForwardModelStepDocumentation(BaseModel):
     config_file: str | None = Field(default=None)
-    source_package: str = Field(default="ert")
+    source_package: str = Field(default_factory=_get_source_package)
     source_function_name: str = Field(default="ert")
     description: str = Field(default="No description")
     examples: str = Field(default="No examples")

ert/config/gen_data_config.py

@@ -14,7 +14,6 @@ from .parsing import ConfigDict, ConfigValidationError, ConfigWarning, ErrorInfo
 from .response_config import (
     InvalidResponseFile,
     ResponseConfig,
-    ResponseMetadata,
 )
 from .responses_index import responses_index
 
@@ -24,22 +23,6 @@ class GenDataConfig(ResponseConfig):
     report_steps_list: list[list[int] | None] = Field(default_factory=list)
     has_finalized_keys: bool = True
 
-    @property
-    def metadata(self) -> list[ResponseMetadata]:
-        return [
-            ResponseMetadata(
-                response_type=self.type,
-                response_key=response_key,
-                finalized=self.has_finalized_keys,
-                filter_on={"report_step": report_steps}
-                if report_steps is not None
-                else {"report_step": [0]},
-            )
-            for response_key, report_steps in zip(
-                self.keys, self.report_steps_list, strict=False
-            )
-        ]
-
     def model_post_init(self, ctx: Any) -> None:
         if len(self.report_steps_list) == 0:
             self.report_steps_list = [[0] for _ in self.keys]
@@ -48,6 +31,20 @@ class GenDataConfig(ResponseConfig):
                 if report_steps is not None:
                     report_steps.sort()
 
+    @property
+    def filter_on(self) -> dict[str, dict[str, list[int]]]:
+        """Filters for this response.
+
+        For ``GEN_DATA`` this is always supported: return
+        ``{response_key: {"report_step": [allowed_steps...]}}``.
+        """
+        return {
+            response_key: {"report_step": report_steps or [0]}
+            for response_key, report_steps in zip(
+                self.keys, self.report_steps_list, strict=False
+            )
+        }
+
     @property
     def expected_input_files(self) -> list[str]:
         expected_files = []