ert 16.0.9__py3-none-any.whl → 19.0.0rc2__py3-none-any.whl

ert/config/everest_control.py

@@ -0,0 +1,234 @@
+from __future__ import annotations
+
+import importlib
+import json
+import logging
+from collections.abc import Iterator, Mapping, MutableMapping
+from dataclasses import field
+from pathlib import Path
+from textwrap import dedent
+from typing import TYPE_CHECKING, Any, Literal, Self
+
+import networkx as nx
+import numpy as np
+import xarray as xr
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+from ropt.workflow import find_sampler_plugin
+
+from ert.substitutions import substitute_runpath_name
+
+from .parameter_config import ParameterConfig
+
+if TYPE_CHECKING:
+    import numpy.typing as npt
+
+    from ert.storage import Ensemble
+
+    Number = int | float
+    DataType = Mapping[str, Number | Mapping[str, Number]]
+    MutableDataType = MutableMapping[str, Number | MutableMapping[str, Number]]
+
+
+class SamplerConfig(BaseModel):
+    backend: str | None = Field(
+        default=None,
+        description=dedent(
+            """
+            [Deprecated]
+
+            The correct backend will be inferred by the method. If several backends
+            have a method named `A`, pick a specific backend `B` by putting `B/A` in
+            the `method` field.
+            """
+        ),
+    )
+    method: str = Field(
+        default="norm",
+        description=dedent(
+            """
+            The sampling method or distribution used by the sampler backend.
+
+            The set of available methods depends on the sampler backend used. By
+            default a plugin based on `scipy.stats` is used, implementing the
+            following methods:
+
+            - From Probability Distributions:
+                - `norm`: Samples from a standard normal distribution (mean 0,
+                  standard deviation 1).
+                - `truncnorm`: Samples from a truncated normal distribution
+                  (mean 0, std. dev. 1), truncated to the range `[-1, 1]`.
+                - `uniform`: Samples from a uniform distribution in the range
+                  `[-1, 1]`.
+
+            - From Quasi-Monte Carlo Sequences:
+                - `sobol`: Uses Sobol' sequences.
+                - `halton`: Uses Halton sequences.
+                - `lhs`: Uses Latin Hypercube Sampling.
+
+                Note: QMC samples are generated in the unit hypercube `[0, 1]^d`
+                and then scaled to the hypercube `[-1, 1]^d`.
+            """
+        ),
+    )
+    options: dict[str, Any] | None = Field(
+        default=None,
+        description=dedent(
+            """
+            Specifies a dict of optional parameters for the sampler backend.
+
+            This dict of values is passed unchanged to the selected method in
+            the backend.
+            """
+        ),
+    )
+    shared: bool | None = Field(
+        default=None,
+        description=dedent(
+            """
+            Whether to share perturbations between realizations.
+            """
+        ),
+    )
+    model_config = ConfigDict(extra="forbid")
+
+    @model_validator(mode="after")
+    def validate_backend_and_method(self) -> Self:
+        if self.backend is not None:
+            message = (
+                "sampler.backend is deprecated. "
+                "The correct backend will be inferred by the method. "
+                "If several backends have a method named A, you need to pick "
+                "a specific backend B by putting B/A in sampler.method."
+            )
+            print(message)
+            # Note: Importing EVEREST.everest
+            # leads to circular import, but we still wish to log
+            # from "everest" here as per old behavior.
+            # Can consider logging this as if from ERT,
+            # which is valid if we store SamplerConfig as part of
+            # EverestControl configs.
+            logging.getLogger("everest").warning(message)
+
+        # Update the default for backends that are not scipy:
+        if (
+            self.backend not in {None, "scipy"}
+            and "method" not in self.model_fields_set
+        ):
+            self.method = "default"
+
+        if self.backend is not None:
+            self.method = f"{self.backend}/{self.method}"
+
+        try:
+            plugin = find_sampler_plugin(f"{self.method}")
+        except ValueError:
+            raise
+        except Exception as exc:
+            ert_version = importlib.metadata.version("ert")
+            ropt_version = importlib.metadata.version("ropt")
+            msg = (
+                f"Error while initializing ropt:\n\n{exc}.\n\n"
+                "There may a be version mismatch between "
+                f"ERT ({ert_version}) and ropt ({ropt_version})\n"
+                "If the installation is correct, please report this as a bug."
+            )
+            raise RuntimeError(msg) from exc
+
+        if plugin is None:
+            raise ValueError(f"Sampler method '{self.method}' not found")
+
+        self.backend = None
+
+        return self
+
+
+class EverestControl(ParameterConfig):
+    """Create an EverestControl for @key with the given @input_keys
+
+    @input_keys can be either a list of keys as strings or a dict with
+    keys as strings and a list of suffixes for each key.
+    If a list of strings is given, the order is preserved.
+    """
+
+    type: Literal["everest_parameters"] = "everest_parameters"
+    input_keys: list[str] = field(default_factory=list)
+    forward_init: bool = False
+    output_file: str = ""
+    forward_init_file: str = ""
+    update: bool = False
+    types: list[Literal["well_control", "generic_control"]]
+    initial_guesses: list[float]
+    control_types: list[Literal["real", "integer"]]
+    enabled: list[bool]
+    min: list[float]
+    max: list[float]
+    perturbation_types: list[Literal["absolute", "relative"]]
+    perturbation_magnitudes: list[float]
+    scaled_ranges: list[tuple[float, float]]
+    samplers: list[SamplerConfig | None]
+
+    # Set up for deprecation, but has to live here until support for the
+    # "dotdash" notation is removed for everest controls via everest config.
+    input_keys_dotdash: list[str] = field(default_factory=list)
+
+    @property
+    def parameter_keys(self) -> list[str]:
+        return self.input_keys
+
+    def read_from_runpath(
+        self, run_path: Path, real_nr: int, iteration: int
+    ) -> xr.Dataset:
+        raise NotImplementedError
+
+    def write_to_runpath(
+        self, run_path: Path, real_nr: int, ensemble: Ensemble
+    ) -> None:
+        file_path = run_path / substitute_runpath_name(
+            self.output_file, real_nr, ensemble.iteration
+        )
+        Path.mkdir(file_path.parent, exist_ok=True, parents=True)
+
+        data: MutableDataType = {}
+        for da in ensemble.load_parameters(self.name, real_nr)["values"]:
+            assert isinstance(da, xr.DataArray)
+            name = str(da.names.values)
+            try:
+                outer, inner = name.split("\0")
+
+                if outer not in data:
+                    data[outer] = {}
+                data[outer][inner] = float(da)  # type: ignore
+            except ValueError:
+                data[name] = float(da)
+
+        file_path.write_text(json.dumps(data), encoding="utf-8")
+
+    def create_storage_datasets(
+        self,
+        from_data: npt.NDArray[np.float64],
+        iens_active_index: npt.NDArray[np.int_],
+    ) -> Iterator[tuple[int, xr.Dataset]]:
+        for i, realization in enumerate(iens_active_index):
+            yield (
+                int(realization),
+                xr.Dataset(
+                    {
+                        "values": ("names", from_data[:, i]),
+                        "names": [
+                            x.split(f"{self.name}.")[1].replace(".", "\0")
+                            for x in self.parameter_keys
+                        ],
+                    }
+                ),
+            )
+
+    def load_parameters(
+        self, ensemble: Ensemble, realizations: npt.NDArray[np.int_]
+    ) -> npt.NDArray[np.float64]:
+        raise NotImplementedError
+
+    def load_parameter_graph(self) -> nx.Graph[int]:
+        raise NotImplementedError
+
+    def __len__(self) -> int:
+        return len(self.input_keys)

ert/config/{everest_constraints_config.py → everest_response.py}

@@ -11,12 +11,21 @@ from .response_config import InvalidResponseFile, ResponseConfig, ResponseMetada
 from .responses_index import responses_index
 
 
-class EverestConstraintsConfig(ResponseConfig):
+class EverestResponse(ResponseConfig):
+    """Base class for Everest response configurations."""
+
+    has_finalized_keys: bool = True
+    scales: list[float | None]
+
+    @property
+    def primary_key(self) -> list[str]:
+        return []
+
     @property
     def metadata(self) -> list[ResponseMetadata]:
         return [
             ResponseMetadata(
-                response_type=self.name,
+                response_type=self.type,
                 response_key=response_key,
                 finalized=self.has_finalized_keys,
                 filter_on=None,
@@ -24,10 +33,6 @@ class EverestConstraintsConfig(ResponseConfig):
             for response_key in self.keys
         ]
 
-    type: Literal["everest_constraints"] = "everest_constraints"
-    name: str = "everest_constraints"
-    has_finalized_keys: bool = True
-
     @property
     def expected_input_files(self) -> list[str]:
         return self.input_files
@@ -49,7 +54,6 @@ class EverestConstraintsConfig(ResponseConfig):
             )
 
         errors = []
-
         run_path_ = Path(run_path)
         datasets_per_name = []
 
@@ -72,24 +76,32 @@ class EverestConstraintsConfig(ResponseConfig):
             if all(isinstance(err, FileNotFoundError) for err in errors):
                 raise FileNotFoundError(
                     "Could not find one or more files/directories while reading "
-                    f"{self.name}: {','.join([str(err) for err in errors])}"
+                    f"{self.type}: {','.join([str(err) for err in errors])}"
                 )
             else:
                 raise InvalidResponseFile(
                     "Error reading "
-                    f"{self.name}, errors: {','.join([str(err) for err in errors])}"
+                    f"{self.type}, errors: {','.join([str(err) for err in errors])}"
                 )
 
         combined = pl.concat(datasets_per_name)
         return combined
 
-    @property
-    def response_type(self) -> str:
-        return "everest_constraints"
 
-    @property
-    def primary_key(self) -> list[str]:
-        return []
+class EverestConstraintsConfig(EverestResponse):
+    type: Literal["everest_constraints"] = "everest_constraints"
+    targets: list[float | None]
+    upper_bounds: list[float]
+    lower_bounds: list[float]
 
 
 responses_index.add_response_type(EverestConstraintsConfig)
+
+
+class EverestObjectivesConfig(EverestResponse):
+    type: Literal["everest_objectives"] = "everest_objectives"
+    weights: list[float | None]
+    objective_types: list[Literal["mean", "stddev"]]
+
+
+responses_index.add_response_type(EverestObjectivesConfig)

ert/config/field.py

@@ -3,32 +3,33 @@ from __future__ import annotations
 import itertools
 import logging
 import os
-from collections.abc import Iterator
-from functools import cached_property
+from collections.abc import Callable, Iterator
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Literal, Self, cast, overload
+from typing import TYPE_CHECKING, Any, Final, Literal, Self, cast, overload
 
 import networkx as nx
 import numpy as np
 import xarray as xr
-import xtgeo  # type: ignore
+import xtgeo
 from pydantic import field_serializer
 
 from ert.field_utils import (
     ErtboxParameters,
     FieldFileFormat,
     Shape,
+    calc_rho_for_2d_grid_layer,
     calculate_ertbox_parameters,
     get_shape,
     read_field,
-    read_mask,
     save_field,
+    transform_local_ellipse_angle_to_local_coords,
+    transform_positions_to_local_field_coordinates,
 )
 from ert.substitutions import substitute_runpath_name
 from ert.utils import log_duration
 
 from ._str_to_bool import str_to_bool
-from .parameter_config import ParameterConfig, ParameterMetadata
+from .parameter_config import ParameterConfig
 from .parsing import ConfigValidationError, ConfigWarning
 
 if TYPE_CHECKING:
@@ -67,32 +68,9 @@ def create_flattened_cube_graph(px: int, py: int, pz: int) -> nx.Graph[int]:
     return G
 
 
-def adjust_graph_for_masking(
-    G: nx.Graph[int], mask: npt.NDArray[np.bool_]
-) -> nx.Graph[int]:
-    """
-    Adjust the graph G according to the masking indices.
-    Removes nodes specified by the mask and relabels the remaining nodes
-    to have consecutive labels from 0 to G.number_of_nodes - 1.
-    Parameters:
-    - G: The graph to adjust
-    - mask: Boolean mask flattened array
-    Returns:
-    - The adjusted graph
-    """
-    # Step 1: Remove nodes specified by mask_indices
-    mask_indices = np.where(mask)[0]
-    G.remove_nodes_from(mask_indices)
-
-    # Step 2: Relabel remaining nodes to 0, 1, 2, ..., G.number_of_nodes - 1
-    new_labels = {old_label: new_label for new_label, old_label in enumerate(G.nodes())}
-    G = nx.relabel_nodes(G, new_labels, copy=True)
-
-    return G
-
-
 class Field(ParameterConfig):
     type: Literal["field"] = "field"
+    dimensionality: Literal[3] = 3
     ertbox_params: ErtboxParameters
     file_format: FieldFileFormat
     output_transformation: str | None
@@ -102,31 +80,15 @@ class Field(ParameterConfig):
     forward_init_file: str
     output_file: Path
     grid_file: str
-    mask_file: Path | None = None
 
     @field_serializer("output_file")
     def serialize_output_file(self, path: Path) -> str:
         return str(path)
 
-    @field_serializer("mask_file")
-    def serialize_mask_file(self, path: Path | None) -> str | None:
-        return str(path) if path is not None else None
-
     @property
     def parameter_keys(self) -> list[str]:
         return []
 
-    @property
-    def metadata(self) -> list[ParameterMetadata]:
-        return [
-            ParameterMetadata(
-                key=self.name,
-                transformation=self.output_transformation,
-                dimensionality=3,
-                userdata={"data_origin": "FIELD", "ertbox_params": self.ertbox_params},
-            )
-        ]
-
     @classmethod
     def from_config_list(
         cls,
@@ -239,11 +201,7 @@ class Field(ParameterConfig):
         )
 
     def __len__(self) -> int:
-        if self.mask_file is None:
-            return self.ertbox_params.nx * self.ertbox_params.ny * self.ertbox_params.nz
-
-        # Uses int() to convert to standard python int for mypy
-        return int(np.size(self.mask) - np.count_nonzero(self.mask))
+        return self.ertbox_params.nx * self.ertbox_params.ny * self.ertbox_params.nz
 
     @log_duration(_logger, custom_name="load_field")
     def read_from_runpath(
@@ -258,7 +216,6 @@ class Field(ParameterConfig):
                         read_field(
                             run_path / file_name,
                             self.name,
-                            self.mask,
                             Shape(
                                 self.ertbox_params.nx,
                                 self.ertbox_params.ny,
@@ -294,16 +251,15 @@ class Field(ParameterConfig):
         from_data: npt.NDArray[np.float64],
         iens_active_index: npt.NDArray[np.int_],
     ) -> Iterator[tuple[int, xr.Dataset]]:
+        dim_nx, dim_ny, dim_nz = (
+            self.ertbox_params.nx,
+            self.ertbox_params.ny,
+            self.ertbox_params.nz,
+        )
+
         for i, realization in enumerate(iens_active_index):
-            ma = np.ma.MaskedArray(  # type: ignore
-                data=np.zeros(self.mask.size),
-                mask=self.mask,
-                fill_value=np.nan,
-                dtype=from_data.dtype,
-            )
-            ma[~ma.mask] = from_data[:, i]
-            ma = ma.reshape(self.mask.shape)  # type: ignore
-            ds = xr.Dataset({"values": (["x", "y", "z"], ma.filled())})
+            values = from_data[:, i].reshape((dim_nx, dim_ny, dim_nz))
+            ds = xr.Dataset({"values": (["x", "y", "z"], values)})
             yield int(realization), ds
 
     def load_parameters(
@@ -314,7 +270,7 @@ class Field(ParameterConfig):
         ensemble_size = len(ds.realizations)
         da = xr.DataArray(
             [
-                np.ma.MaskedArray(data=d, mask=self.mask).compressed()  # type: ignore
+                np.ma.MaskedArray(data=d).compressed()
                 for d in ds["values"].values.reshape(ensemble_size, -1)
             ]
         )
@@ -328,7 +284,7 @@ class Field(ParameterConfig):
     def _transform_data(
         self, data_array: xr.DataArray
     ) -> np.ma.MaskedArray[Any, np.dtype[np.float32]]:
-        return np.ma.MaskedArray(  # type: ignore
+        return np.ma.MaskedArray(
             _field_truncate(
                 field_transform(
                     data_array,
@@ -337,31 +293,18 @@ class Field(ParameterConfig):
                 self.truncation_min,
                 self.truncation_max,
             ),
-            self.mask,
             fill_value=np.nan,
         )
 
-    def save_experiment_data(self, experiment_path: Path) -> None:
-        mask_path = experiment_path / "grid_mask.npy"
-        if not mask_path.exists():
-            mask, _ = read_mask(self.grid_file)
-            np.save(mask_path, mask)
-        self.mask_file = mask_path
-
-    @cached_property
-    def mask(self) -> Any:
-        if self.mask_file is None:
-            raise ValueError(
-                "In order to get Field.mask, Field.save_experiment_data has"
-                " to be called first"
-            )
-        return np.load(self.mask_file)
-
-    def load_parameter_graph(self) -> nx.Graph:  # type: ignore
+    def load_parameter_graph(self) -> nx.Graph[int]:
         parameter_graph = create_flattened_cube_graph(
             px=self.ertbox_params.nx, py=self.ertbox_params.ny, pz=self.ertbox_params.nz
         )
-        return adjust_graph_for_masking(G=parameter_graph, mask=self.mask.flatten())
+        new_labels = {
+            old_label: new_label
+            for new_label, old_label in enumerate(parameter_graph.nodes())
+        }
+        return nx.relabel_nodes(parameter_graph, new_labels, copy=True)
 
     @property
     def nx(self) -> int:
@@ -375,8 +318,77 @@ class Field(ParameterConfig):
     def nz(self) -> int:
         return self.ertbox_params.nz
 
+    def calc_rho_for_2d_grid_layer(
+        self,
+        obs_xpos: npt.NDArray[np.float64],
+        obs_ypos: npt.NDArray[np.float64],
+        obs_main_range: npt.NDArray[np.float64],
+        obs_perp_range: npt.NDArray[np.float64],
+        obs_anisotropy_angle: npt.NDArray[np.float64],
+        right_handed_grid_indexing: bool = True,
+    ) -> npt.NDArray[np.float64]:
+        """Function to calculate scaling values to be used in the RHO matrix
+        for distance-based localization.
+
+        Args:
+            obs_xpos: x-coordinates in global coordinates of observations
+            obs_ypos: y-coordinates in global coordinates of observations
+            obs_main_range: Size of influence ellipse main principal direction.
+            obs_perp_range: Size of influence ellipse second principal direction.
+            obs_anisotropy_angle: Rotation angle anticlock wise of main principal
+              direction of influence ellipse relative to global coordinate
+              system's x-axis.
+            right_handed_grid_indexing: When this is True the field parameters
+              grid index order counts J-index down from ny-1 to 0.
+              If the value is False, the grid index order is to count J index
+              from 0 to ny-1. As standard for 3D field parameters,
+              the grid index order follows the right_handed grid indexing.
+
+        Returns:
+            Scaling values (elements of the RHO matrix) as a numpy array
+              of shape=(nx,ny,nobservations)
+
+        """
+        # Can only be used if ertbox coordinate system is defined
+        assert self.ertbox_params.xinc is not None, (
+            "Parameter for grid resolution must be defined"
+        )
+        assert self.ertbox_params.yinc is not None, (
+            "Parameter for grid resolution must be defined"
+        )
+        assert self.ertbox_params.origin is not None, (
+            "Parameter for grid origin must be defined"
+        )
+        assert self.ertbox_params.rotation_angle is not None, (
+            "Parameter for grid rotation must be defined"
+        )
+        # Transform positions of observations into local coordinates
+        xpos, ypos = transform_positions_to_local_field_coordinates(
+            self.ertbox_params.origin,
+            self.ertbox_params.rotation_angle,
+            obs_xpos,
+            obs_ypos,
+        )
+        # Transform localization ellipse orientation to local coordinates
+        ellipse_rotation = transform_local_ellipse_angle_to_local_coords(
+            self.ertbox_params.rotation_angle, obs_anisotropy_angle
+        )
+
+        return calc_rho_for_2d_grid_layer(
+            self.ertbox_params.nx,
+            self.ertbox_params.ny,
+            self.ertbox_params.xinc,
+            self.ertbox_params.yinc,
+            xpos,
+            ypos,
+            obs_main_range,
+            obs_perp_range,
+            ellipse_rotation,
+            right_handed_grid_indexing=right_handed_grid_indexing,
+        )
+
 
-TRANSFORM_FUNCTIONS = {
+TRANSFORM_FUNCTIONS: Final[dict[str, Callable[[Any], Any]]] = {
     "LN": np.log,
     "LOG": np.log,
     "LN0": lambda v: np.log(v + 0.000001),
@@ -407,17 +419,14 @@ def field_transform(
 ) -> npt.NDArray[np.float32] | xr.DataArray:
     if transform_name is None:
         return data
-    return TRANSFORM_FUNCTIONS[transform_name](data)  # type: ignore
+    return TRANSFORM_FUNCTIONS[transform_name](data)
 
 
 def _field_truncate(data: npt.ArrayLike, min_: float | None, max_: float | None) -> Any:
     if min_ is not None and max_ is not None:
-        vfunc = np.vectorize(lambda x: max(min(x, max_), min_))
-        return vfunc(data)
+        return np.clip(data, min_, max_)
     elif min_ is not None:
-        vfunc = np.vectorize(lambda x: max(x, min_))
-        return vfunc(data)
+        return np.maximum(data, min_)
     elif max_ is not None:
-        vfunc = np.vectorize(lambda x: min(x, max_))
-        return vfunc(data)
+        return np.minimum(data, max_)
     return data