ert 17.0.0__py3-none-any.whl → 19.0.0rc2__py3-none-any.whl

ert/config/field.py

@@ -3,32 +3,33 @@ from __future__ import annotations
 import itertools
 import logging
 import os
-from collections.abc import Iterator
-from functools import cached_property
+from collections.abc import Callable, Iterator
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Literal, Self, cast, overload
+from typing import TYPE_CHECKING, Any, Final, Literal, Self, cast, overload
 
 import networkx as nx
 import numpy as np
 import xarray as xr
-import xtgeo  # type: ignore
+import xtgeo
 from pydantic import field_serializer
 
 from ert.field_utils import (
     ErtboxParameters,
     FieldFileFormat,
     Shape,
+    calc_rho_for_2d_grid_layer,
     calculate_ertbox_parameters,
     get_shape,
     read_field,
-    read_mask,
     save_field,
+    transform_local_ellipse_angle_to_local_coords,
+    transform_positions_to_local_field_coordinates,
 )
 from ert.substitutions import substitute_runpath_name
 from ert.utils import log_duration
 
 from ._str_to_bool import str_to_bool
-from .parameter_config import ParameterConfig, ParameterMetadata
+from .parameter_config import ParameterConfig
 from .parsing import ConfigValidationError, ConfigWarning
 
 if TYPE_CHECKING:
@@ -67,32 +68,9 @@ def create_flattened_cube_graph(px: int, py: int, pz: int) -> nx.Graph[int]:
     return G
 
 
-def adjust_graph_for_masking(
-    G: nx.Graph[int], mask: npt.NDArray[np.bool_]
-) -> nx.Graph[int]:
-    """
-    Adjust the graph G according to the masking indices.
-    Removes nodes specified by the mask and relabels the remaining nodes
-    to have consecutive labels from 0 to G.number_of_nodes - 1.
-    Parameters:
-    - G: The graph to adjust
-    - mask: Boolean mask flattened array
-    Returns:
-    - The adjusted graph
-    """
-    # Step 1: Remove nodes specified by mask_indices
-    mask_indices = np.where(mask)[0]
-    G.remove_nodes_from(mask_indices)
-
-    # Step 2: Relabel remaining nodes to 0, 1, 2, ..., G.number_of_nodes - 1
-    new_labels = {old_label: new_label for new_label, old_label in enumerate(G.nodes())}
-    G = nx.relabel_nodes(G, new_labels, copy=True)
-
-    return G
-
-
 class Field(ParameterConfig):
     type: Literal["field"] = "field"
+    dimensionality: Literal[3] = 3
     ertbox_params: ErtboxParameters
     file_format: FieldFileFormat
     output_transformation: str | None
@@ -102,31 +80,15 @@ class Field(ParameterConfig):
     forward_init_file: str
     output_file: Path
     grid_file: str
-    mask_file: Path | None = None
 
     @field_serializer("output_file")
     def serialize_output_file(self, path: Path) -> str:
         return str(path)
 
-    @field_serializer("mask_file")
-    def serialize_mask_file(self, path: Path | None) -> str | None:
-        return str(path) if path is not None else None
-
     @property
     def parameter_keys(self) -> list[str]:
         return []
 
-    @property
-    def metadata(self) -> list[ParameterMetadata]:
-        return [
-            ParameterMetadata(
-                key=self.name,
-                transformation=self.output_transformation,
-                dimensionality=3,
-                userdata={"data_origin": "FIELD", "ertbox_params": self.ertbox_params},
-            )
-        ]
-
     @classmethod
     def from_config_list(
         cls,
@@ -239,11 +201,7 @@ class Field(ParameterConfig):
         )
 
     def __len__(self) -> int:
-        if self.mask_file is None:
-            return self.ertbox_params.nx * self.ertbox_params.ny * self.ertbox_params.nz
-
-        # Uses int() to convert to standard python int for mypy
-        return int(np.size(self.mask) - np.count_nonzero(self.mask))
+        return self.ertbox_params.nx * self.ertbox_params.ny * self.ertbox_params.nz
 
     @log_duration(_logger, custom_name="load_field")
     def read_from_runpath(
@@ -258,7 +216,6 @@ class Field(ParameterConfig):
                         read_field(
                             run_path / file_name,
                             self.name,
-                            self.mask,
                             Shape(
                                 self.ertbox_params.nx,
                                 self.ertbox_params.ny,
@@ -294,16 +251,15 @@ class Field(ParameterConfig):
         from_data: npt.NDArray[np.float64],
         iens_active_index: npt.NDArray[np.int_],
     ) -> Iterator[tuple[int, xr.Dataset]]:
+        dim_nx, dim_ny, dim_nz = (
+            self.ertbox_params.nx,
+            self.ertbox_params.ny,
+            self.ertbox_params.nz,
+        )
+
         for i, realization in enumerate(iens_active_index):
-            ma = np.ma.MaskedArray(  # type: ignore
-                data=np.zeros(self.mask.size),
-                mask=self.mask,
-                fill_value=np.nan,
-                dtype=from_data.dtype,
-            )
-            ma[~ma.mask] = from_data[:, i]
-            ma = ma.reshape(self.mask.shape)  # type: ignore
-            ds = xr.Dataset({"values": (["x", "y", "z"], ma.filled())})
+            values = from_data[:, i].reshape((dim_nx, dim_ny, dim_nz))
+            ds = xr.Dataset({"values": (["x", "y", "z"], values)})
             yield int(realization), ds
 
     def load_parameters(
@@ -314,7 +270,7 @@ class Field(ParameterConfig):
         ensemble_size = len(ds.realizations)
         da = xr.DataArray(
             [
-                np.ma.MaskedArray(data=d, mask=self.mask).compressed()  # type: ignore
+                np.ma.MaskedArray(data=d).compressed()
                 for d in ds["values"].values.reshape(ensemble_size, -1)
             ]
         )
@@ -328,7 +284,7 @@ class Field(ParameterConfig):
     def _transform_data(
         self, data_array: xr.DataArray
     ) -> np.ma.MaskedArray[Any, np.dtype[np.float32]]:
-        return np.ma.MaskedArray(  # type: ignore
+        return np.ma.MaskedArray(
             _field_truncate(
                 field_transform(
                     data_array,
@@ -337,31 +293,18 @@ class Field(ParameterConfig):
                 self.truncation_min,
                 self.truncation_max,
             ),
-            self.mask,
             fill_value=np.nan,
         )
 
-    def save_experiment_data(self, experiment_path: Path) -> None:
-        mask_path = experiment_path / "grid_mask.npy"
-        if not mask_path.exists():
-            mask, _ = read_mask(self.grid_file)
-            np.save(mask_path, mask)
-        self.mask_file = mask_path
-
-    @cached_property
-    def mask(self) -> Any:
-        if self.mask_file is None:
-            raise ValueError(
-                "In order to get Field.mask, Field.save_experiment_data has"
-                " to be called first"
-            )
-        return np.load(self.mask_file)
-
-    def load_parameter_graph(self) -> nx.Graph:  # type: ignore
+    def load_parameter_graph(self) -> nx.Graph[int]:
         parameter_graph = create_flattened_cube_graph(
             px=self.ertbox_params.nx, py=self.ertbox_params.ny, pz=self.ertbox_params.nz
         )
-        return adjust_graph_for_masking(G=parameter_graph, mask=self.mask.flatten())
+        new_labels = {
+            old_label: new_label
+            for new_label, old_label in enumerate(parameter_graph.nodes())
+        }
+        return nx.relabel_nodes(parameter_graph, new_labels, copy=True)
 
     @property
     def nx(self) -> int:
@@ -375,8 +318,77 @@ class Field(ParameterConfig):
     def nz(self) -> int:
         return self.ertbox_params.nz
 
+    def calc_rho_for_2d_grid_layer(
+        self,
+        obs_xpos: npt.NDArray[np.float64],
+        obs_ypos: npt.NDArray[np.float64],
+        obs_main_range: npt.NDArray[np.float64],
+        obs_perp_range: npt.NDArray[np.float64],
+        obs_anisotropy_angle: npt.NDArray[np.float64],
+        right_handed_grid_indexing: bool = True,
+    ) -> npt.NDArray[np.float64]:
+        """Function to calculate scaling values to be used in the RHO matrix
+        for distance-based localization.
+
+        Args:
+            obs_xpos: x-coordinates in global coordinates of observations
+            obs_ypos: y-coordinates in global coordinates of observations
+            obs_main_range: Size of influence ellipse main principal direction.
+            obs_perp_range: Size of influence ellipse second principal direction.
+            obs_anisotropy_angle: Rotation angle anticlock wise of main principal
+              direction of influence ellipse relative to global coordinate
+              system's x-axis.
+            right_handed_grid_indexing: When this is True the field parameters
+              grid index order counts J-index down from ny-1 to 0.
+              If the value is False, the grid index order is to count J index
+              from 0 to ny-1. As standard for 3D field parameters,
+              the grid index order follows the right_handed grid indexing.
+
+        Returns:
+            Scaling values (elements of the RHO matrix) as a numpy array
+              of shape=(nx,ny,nobservations)
+
+        """
+        # Can only be used if ertbox coordinate system is defined
+        assert self.ertbox_params.xinc is not None, (
+            "Parameter for grid resolution must be defined"
+        )
+        assert self.ertbox_params.yinc is not None, (
+            "Parameter for grid resolution must be defined"
+        )
+        assert self.ertbox_params.origin is not None, (
+            "Parameter for grid origin must be defined"
+        )
+        assert self.ertbox_params.rotation_angle is not None, (
+            "Parameter for grid rotation must be defined"
+        )
+        # Transform positions of observations into local coordinates
+        xpos, ypos = transform_positions_to_local_field_coordinates(
+            self.ertbox_params.origin,
+            self.ertbox_params.rotation_angle,
+            obs_xpos,
+            obs_ypos,
+        )
+        # Transform localization ellipse orientation to local coordinates
+        ellipse_rotation = transform_local_ellipse_angle_to_local_coords(
+            self.ertbox_params.rotation_angle, obs_anisotropy_angle
+        )
+
+        return calc_rho_for_2d_grid_layer(
+            self.ertbox_params.nx,
+            self.ertbox_params.ny,
+            self.ertbox_params.xinc,
+            self.ertbox_params.yinc,
+            xpos,
+            ypos,
+            obs_main_range,
+            obs_perp_range,
+            ellipse_rotation,
+            right_handed_grid_indexing=right_handed_grid_indexing,
+        )
+
 
-TRANSFORM_FUNCTIONS = {
+TRANSFORM_FUNCTIONS: Final[dict[str, Callable[[Any], Any]]] = {
     "LN": np.log,
     "LOG": np.log,
     "LN0": lambda v: np.log(v + 0.000001),
@@ -407,7 +419,7 @@ def field_transform(
 ) -> npt.NDArray[np.float32] | xr.DataArray:
     if transform_name is None:
         return data
-    return TRANSFORM_FUNCTIONS[transform_name](data)  # type: ignore
+    return TRANSFORM_FUNCTIONS[transform_name](data)
 
 
 def _field_truncate(data: npt.ArrayLike, min_: float | None, max_: float | None) -> Any:

ert/config/forward_model_step.py

@@ -1,13 +1,33 @@
 from __future__ import annotations
 
 import logging
-from typing import ClassVar, Literal, NotRequired, Self
-
-from pydantic import BaseModel, Field, field_validator, model_validator
+from typing import (
+    TYPE_CHECKING,
+    Annotated,
+    Any,
+    ClassVar,
+    Literal,
+    NotRequired,
+    Self,
+    cast,
+)
+
+from pydantic import (
+    BaseModel,
+    Field,
+    field_validator,
+    model_serializer,
+    model_validator,
+)
+from pydantic_core.core_schema import ValidationInfo
 from typing_extensions import TypedDict, Unpack
 
+from ..base_model_context import BaseModelWithContextSupport
 from .parsing import ConfigValidationError, ConfigWarning, SchemaItemType
 
+if TYPE_CHECKING:
+    from ert.plugins import ErtRuntimePlugins
+
 logger = logging.getLogger(__name__)
 
 
@@ -72,6 +92,7 @@ class ForwardModelStepOptions(TypedDict, total=False):
     max_running_minutes: NotRequired[int]
     environment: NotRequired[dict[str, str]]
     default_mapping: NotRequired[dict[str, str]]
+    required_keywords: NotRequired[list[str]]
 
 
 class ForwardModelStepDocumentation(BaseModel):
@@ -80,18 +101,21 @@ class ForwardModelStepDocumentation(BaseModel):
     source_function_name: str = Field(default="ert")
     description: str = Field(default="No description")
     examples: str = Field(default="No examples")
-    category: (
-        Literal[
-            "utility.file_system",
-            "simulators.reservoir",
-            "modelling.reservoir",
-            "utility.templating",
-        ]
-        | str
-    ) = Field(default="Uncategorized")
-
-
-class ForwardModelStep(BaseModel):
+    category: Annotated[
+        str,
+        Field(
+            default="Uncategorized",
+            examples=[
+                "utility.file_system",
+                "simulators.reservoir",
+                "modelling.reservoir",
+                "utility.templating",
+            ],
+        ),
+    ]
+
+
+class ForwardModelStep(BaseModelWithContextSupport):
     """
     Holds information to execute one step of a forward model
 
@@ -202,7 +226,87 @@ class ForwardModelStep(BaseModel):
         return None if v == "null" else v
 
 
-class ForwardModelStepPlugin(ForwardModelStep):
+class UserInstalledForwardModelStep(ForwardModelStep):
+    """
+    Represents a forward model step installed by a user via the ERT Config
+    forward model step format provided via the INSTALL_JOB keyword.
+    User-installed forward model steps serialize with their full configuration,
+    unlike site-installed steps which only serialize as references.
+    """
+
+    type: Literal["user_installed"] = "user_installed"
+
+
+class _SerializedSiteInstalledForwardModelStep(TypedDict):
+    type: Literal["site_installed"]
+    name: str
+    private_args: dict[str, str]
+
+
+class SiteInstalledForwardModelStep(ForwardModelStep):
+    """
+    Represents a forward model step installed via external plugins.
+    Instances of this class serialize only as references to the plugin by name, and
+    the user-provided private_args, allowing them to dynamically update when plugins
+    change, rather than being locked to a specific executable at serialization time.
+    """
+
+    type: Literal["site_installed"] = "site_installed"
+
+    @model_serializer(mode="plain")
+    def serialize_model(self) -> _SerializedSiteInstalledForwardModelStep:
+        return {
+            "type": "site_installed",
+            "name": self.name,
+            "private_args": self.private_args,
+        }
+
+    @model_validator(mode="before")
+    @classmethod
+    def deserialize_model(
+        cls, values: dict[str, Any], info: ValidationInfo
+    ) -> dict[str, Any]:
+        runtime_plugins = cast("ErtRuntimePlugins", info.context)
+        name = values["name"]
+
+        if runtime_plugins is None:
+            if values.get("type") == "site_installed":
+                msg = (
+                    f"Trying to find site-installed forward model step {name} "
+                    f"without site plugins. This forward model must be loaded "
+                    f"with ERT site plugins available."
+                )
+                raise KeyError(msg)
+            return values
+
+        if name not in runtime_plugins.installed_forward_model_steps:
+            msg = (
+                f"Expected forward model step {name} to be installed "
+                f"via plugins, but it was not found. Please check that "
+                f"your python environment has it installed."
+            )
+            raise KeyError(msg)
+        site_installed_fm = runtime_plugins.installed_forward_model_steps[name]
+
+        # Intent: copy the site installed forward model to this instance.
+        # bypassing the model_serializer
+        site_fm_instance = {
+            k: getattr(site_installed_fm, k)
+            for k in SiteInstalledForwardModelStep.model_fields
+        }
+
+        return site_fm_instance | (
+            {"private_args": values["private_args"]} if "private_args" in values else {}
+        )
+
+
+SiteOrUserForwardModelStep = Annotated[
+    (UserInstalledForwardModelStep | SiteInstalledForwardModelStep),
+    Field(discriminator="type"),
+]
+
+
+class ForwardModelStepPlugin(SiteInstalledForwardModelStep):
     def __init__(
         self, name: str, command: list[str], **kwargs: Unpack[ForwardModelStepOptions]
     ) -> None:
@@ -221,6 +325,7 @@ class ForwardModelStepPlugin(ForwardModelStep):
         max_running_minutes = kwargs.get("max_running_minutes")
         environment = kwargs.get("environment", {}) or {}
         default_mapping = kwargs.get("default_mapping", {}) or {}
+        required_keywords = kwargs.get("required_keywords", []) or []
 
         super().__init__(
             name=name,
@@ -235,7 +340,7 @@ class ForwardModelStepPlugin(ForwardModelStep):
             max_running_minutes=max_running_minutes,
             min_arg=0,
             max_arg=0,
-            required_keywords=[],
+            required_keywords=required_keywords,
             arg_types=[],
             environment=environment,
             default_mapping=default_mapping,

ert/config/gen_data_config.py

@@ -21,7 +21,6 @@ from .responses_index import responses_index
 
 class GenDataConfig(ResponseConfig):
     type: Literal["gen_data"] = "gen_data"
-    name: str = "gen_data"
     report_steps_list: list[list[int] | None] = Field(default_factory=list)
     has_finalized_keys: bool = True
 
@@ -29,7 +28,7 @@ class GenDataConfig(ResponseConfig):
     def metadata(self) -> list[ResponseMetadata]:
         return [
             ResponseMetadata(
-                response_type=self.name,
+                response_type=self.type,
                 response_key=response_key,
                 finalized=self.has_finalized_keys,
                 filter_on={"report_step": report_steps}
@@ -198,28 +197,24 @@ class GenDataConfig(ResponseConfig):
             if all(isinstance(err, FileNotFoundError) for err in errors):
                 raise FileNotFoundError(
                     "Could not find one or more files/directories while reading "
-                    f"GEN_DATA {self.name}: {','.join([str(err) for err in errors])}"
+                    f"GEN_DATA: {','.join([str(err) for err in errors])}"
                 )
             else:
                 raise InvalidResponseFile(
                     "Error reading GEN_DATA "
-                    f"{self.name}, errors: {','.join([str(err) for err in errors])}"
+                    f"{self.type}, errors: {','.join([str(err) for err in errors])}"
                 )
 
         combined = pl.concat(datasets_per_name)
         return combined
 
     def get_args_for_key(self, key: str) -> tuple[str | None, list[int] | None]:
-        for i, _key in enumerate(self.keys):
-            if key == _key:
+        for i, key_ in enumerate(self.keys):
+            if key == key_:
                 return self.input_files[i], self.report_steps_list[i]
 
         return None, None
 
-    @property
-    def response_type(self) -> str:
-        return "gen_data"
-
     @property
     def primary_key(self) -> list[str]:
         return ["report_step", "index"]

ert/config/gen_kw_config.py

@@ -15,7 +15,7 @@ from typing_extensions import TypedDict
 
 from ._str_to_bool import str_to_bool
 from .distribution import DISTRIBUTION_CLASSES, DistributionSettings, get_distribution
-from .parameter_config import ParameterCardinality, ParameterConfig, ParameterMetadata
+from .parameter_config import ParameterCardinality, ParameterConfig
 from .parsing import ConfigValidationError, ConfigWarning
 
 if TYPE_CHECKING:
@@ -53,6 +53,7 @@ class DataSource(StrEnum):
 
 class GenKwConfig(ParameterConfig):
     type: Literal["gen_kw"] = "gen_kw"
+    dimensionality: Literal[1] = 1
     distribution: DistributionSettings
     forward_init: bool = False
     update: bool = True
@@ -73,17 +74,6 @@ class GenKwConfig(ParameterConfig):
     def cardinality(self) -> ParameterCardinality:
         return ParameterCardinality.multiple_configs_per_ensemble_dataset
 
-    @property
-    def metadata(self) -> list[ParameterMetadata]:
-        return [
-            ParameterMetadata(
-                key=f"{self.group}:{self.name}",
-                transformation=self.distribution.name.upper(),
-                dimensionality=1,
-                userdata={"data_origin": "GEN_KW"},
-            )
-        ]
-
     @classmethod
     def templates_from_config(
         cls, gen_kw: list[str | dict[str, str]]
@@ -121,7 +111,7 @@ class GenKwConfig(ParameterConfig):
         gen_kw_key = cast(str, config_list[0])
 
         options = cast(dict[str, str], config_list[-1])
-        positional_args = cast(list[str], config_list[:-1])
+        positional_args = cast(list[str | list[str]], config_list[:-1])
         errors = []
         update_parameter = str_to_bool(options.get("UPDATE", "TRUE"))
         if _get_abs_path(options.get("INIT_FILES")):
@@ -186,7 +176,7 @@ class GenKwConfig(ParameterConfig):
                         params[0], params[1], params[2:]
                     ),
                     forward_init=False,
-                    update=update_parameter,
+                    update="CONST" not in params and update_parameter,
                 )
                 for params in distributions_spec
             ]
@@ -217,18 +207,7 @@ class GenKwConfig(ParameterConfig):
         real_nr: int,
         ensemble: Ensemble,
     ) -> dict[str, dict[str, float | str]]:
-        df = ensemble.load_parameters(self.name, real_nr, transformed=True).drop(
-            "realization"
-        )
-
-        assert isinstance(df, pl.DataFrame)
-        if not df.width == 1:
-            raise ValueError(
-                f"GEN_KW {self.group_name}:{self.name} should be a single parameter!"
-            )
-
-        data = df.to_dicts()[0]
-        return {self.group_name: data}
+        raise NotImplementedError
 
     def load_parameters(
         self, ensemble: Ensemble, realizations: npt.NDArray[np.int_]
@@ -259,15 +238,6 @@ class GenKwConfig(ParameterConfig):
     def group_name(self) -> str:
         return self.group
 
-    def copy_parameters(
-        self,
-        source_ensemble: Ensemble,
-        target_ensemble: Ensemble,
-        realizations: npt.NDArray[np.int_],
-    ) -> None:
-        df = source_ensemble.load_parameters(self.name, realizations)
-        target_ensemble.save_parameters(dataset=df)
-
     def get_priors(self) -> list[PriorDict]:
         dist_json = self.distribution.model_dump(exclude={"name"})
         return [

ert/config/known_response_types.py

@@ -0,0 +1,14 @@
+from .everest_response import EverestConstraintsConfig, EverestObjectivesConfig
+from .gen_data_config import GenDataConfig
+from .rft_config import RFTConfig
+from .summary_config import SummaryConfig
+
+KnownErtResponseTypes = SummaryConfig | GenDataConfig | RFTConfig
+KNOWN_ERT_RESPONSE_TYPES = (
+    SummaryConfig,
+    GenDataConfig,
+    RFTConfig,
+)
+KnownResponseTypes = (
+    KnownErtResponseTypes | EverestConstraintsConfig | EverestObjectivesConfig
+)

ert/config/parameter_config.py

@@ -5,7 +5,7 @@ from collections.abc import Callable, Iterator
 from enum import StrEnum, auto
 from hashlib import sha256
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Literal
+from typing import TYPE_CHECKING
 
 import networkx as nx
 import numpy as np
@@ -39,13 +39,6 @@ class ParameterCardinality(StrEnum):
     one_config_per_realization_dataset = auto()
 
 
-class ParameterMetadata(BaseModel):
-    key: str
-    transformation: str | None
-    dimensionality: Literal[1, 2, 3] = 1
-    userdata: dict[str, Any]
-
-
 class ParameterConfig(BaseModel):
     type: str
     name: str
@@ -59,14 +52,6 @@ class ParameterConfig(BaseModel):
         Returns a list of parameter keys within this parameter group
         """
 
-    @property
-    @abstractmethod
-    def metadata(self) -> list[ParameterMetadata]:
-        """
-        Returns metadata describing this parameter
-
-        """
-
     @abstractmethod
     def __len__(self) -> int:
         """Number of parameters"""
@@ -108,23 +93,6 @@ class ParameterConfig(BaseModel):
         or polars DataFrame from the numpy data
         """
 
-    def copy_parameters(
-        self,
-        source_ensemble: Ensemble,
-        target_ensemble: Ensemble,
-        realizations: npt.NDArray[np.int_],
-    ) -> None:
-        """
-        Copy parameters from one ensemble to another.
-        If realizations is None, copy all realizations.
-        If realizations is given, copy only those realizations.
-        """
-        for realization in realizations:
-            # Converts to standard python scalar due to mypy
-            realization_int = int(realization)
-            ds = source_ensemble.load_parameters(self.name, realization_int)
-            target_ensemble.save_parameters(ds, self.name, realization_int)
-
     @abstractmethod
     def load_parameters(
         self, ensemble: Ensemble, realizations: npt.NDArray[np.int_]