ert 17.1.9__py3-none-any.whl → 18.0.1__py3-none-any.whl

ert/config/ext_param_config.py

@@ -1,14 +1,19 @@
 from __future__ import annotations
 
+import importlib
 import json
+import logging
 from collections.abc import Iterator, Mapping, MutableMapping
 from dataclasses import field
 from pathlib import Path
-from typing import TYPE_CHECKING, Literal
+from textwrap import dedent
+from typing import TYPE_CHECKING, Any, Literal, Self
 
 import networkx as nx
 import numpy as np
 import xarray as xr
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+from ropt.workflow import find_sampler_plugin
 
 from ert.substitutions import substitute_runpath_name
 
@@ -24,6 +29,119 @@ if TYPE_CHECKING:
     MutableDataType = MutableMapping[str, Number | MutableMapping[str, Number]]
 
 
+class SamplerConfig(BaseModel):
+    backend: str | None = Field(
+        default=None,
+        description=dedent(
+            """
+            [Deprecated]
+
+            The correct backend will be inferred by the method. If several backends
+            have a method named `A`, pick a specific backend `B` by putting `B/A` in
+            the `method` field.
+            """
+        ),
+    )
+    method: str = Field(
+        default="norm",
+        description=dedent(
+            """
+            The sampling method or distribution used by the sampler backend.
+
+            The set of available methods depends on the sampler backend used. By
+            default a plugin based on `scipy.stats` is used, implementing the
+            following methods:
+
+            - From Probability Distributions:
+                - `norm`: Samples from a standard normal distribution (mean 0,
+                  standard deviation 1).
+                - `truncnorm`: Samples from a truncated normal distribution
+                  (mean 0, std. dev. 1), truncated to the range `[-1, 1]`.
+                - `uniform`: Samples from a uniform distribution in the range
+                  `[-1, 1]`.
+
+            - From Quasi-Monte Carlo Sequences:
+                - `sobol`: Uses Sobol' sequences.
+                - `halton`: Uses Halton sequences.
+                - `lhs`: Uses Latin Hypercube Sampling.
+
+                Note: QMC samples are generated in the unit hypercube `[0, 1]^d`
+                and then scaled to the hypercube `[-1, 1]^d`.
+            """
+        ),
+    )
+    options: dict[str, Any] | None = Field(
+        default=None,
+        description=dedent(
+            """
+            Specifies a dict of optional parameters for the sampler backend.
+
+            This dict of values is passed unchanged to the selected method in
+            the backend.
+            """
+        ),
+    )
+    shared: bool | None = Field(
+        default=None,
+        description=dedent(
+            """
+            Whether to share perturbations between realizations.
+            """
+        ),
+    )
+    model_config = ConfigDict(extra="forbid")
+
+    @model_validator(mode="after")
+    def validate_backend_and_method(self) -> Self:
+        if self.backend is not None:
+            message = (
+                "sampler.backend is deprecated. "
+                "The correct backend will be inferred by the method. "
+                "If several backends have a method named A, you need to pick "
+                "a specific backend B by putting B/A in sampler.method."
+            )
+            print(message)
+            # Note: Importing EVEREST.everest
+            # leads to circular import, but we still wish to log
+            # from "everest" here as per old behavior.
+            # Can consider logging this as if from ERT,
+            # which is valid if we store SamplerConfig as part of
+            # ExtParam configs.
+            logging.getLogger("everest").warning(message)
+
+        # Update the default for backends that are not scipy:
+        if (
+            self.backend not in {None, "scipy"}
+            and "method" not in self.model_fields_set
+        ):
+            self.method = "default"
+
+        if self.backend is not None:
+            self.method = f"{self.backend}/{self.method}"
+
+        try:
+            plugin = find_sampler_plugin(f"{self.method}")
+        except ValueError:
+            raise
+        except Exception as exc:
+            ert_version = importlib.metadata.version("ert")
+            ropt_version = importlib.metadata.version("ropt")
+            msg = (
+                f"Error while initializing ropt:\n\n{exc}.\n\n"
+                "There may a be version mismatch between "
+                f"ERT ({ert_version}) and ropt ({ropt_version})\n"
+                "If the installation is correct, please report this as a bug."
+            )
+            raise RuntimeError(msg) from exc
+
+        if plugin is None:
+            raise ValueError(f"Sampler method '{self.method}' not found")
+
+        self.backend = None
+
+        return self
+
+
 class ExtParamConfig(ParameterConfig):
     """Create an ExtParamConfig for @key with the given @input_keys
 
@@ -46,6 +164,20 @@ class ExtParamConfig(ParameterConfig):
     output_file: str = ""
     forward_init_file: str = ""
     update: bool = False
+    types: list[Literal["well_control", "generic_control"]]
+    initial_guesses: list[float]
+    control_types: list[Literal["real", "integer"]]
+    enabled: list[bool]
+    min: list[float]
+    max: list[float]
+    perturbation_types: list[Literal["absolute", "relative"]]
+    perturbation_magnitudes: list[float]
+    scaled_ranges: list[tuple[float, float]]
+    samplers: list[SamplerConfig | None]
+
+    # Set up for deprecation, but has to live here until support for the
+    # "dotdash" notation is removed for everest controls via everest config.
+    input_keys_dotdash: list[str] = field(default_factory=list)
 
     def read_from_runpath(
         self, run_path: Path, real_nr: int, iteration: int

ert/config/field.py

@@ -3,14 +3,14 @@ from __future__ import annotations
 import itertools
 import logging
 import os
-from collections.abc import Iterator
+from collections.abc import Callable, Iterator
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Literal, Self, cast, overload
+from typing import TYPE_CHECKING, Any, Final, Literal, Self, cast, overload
 
 import networkx as nx
 import numpy as np
 import xarray as xr
-import xtgeo  # type: ignore
+import xtgeo
 from pydantic import field_serializer
 
 from ert.field_utils import (
@@ -258,10 +258,14 @@ class Field(ParameterConfig):
         from_data: npt.NDArray[np.float64],
         iens_active_index: npt.NDArray[np.int_],
     ) -> Iterator[tuple[int, xr.Dataset]]:
-        nx, ny, nz = self.ertbox_params.nx, self.ertbox_params.ny, self.ertbox_params.nz
+        dim_nx, dim_ny, dim_nz = (
+            self.ertbox_params.nx,
+            self.ertbox_params.ny,
+            self.ertbox_params.nz,
+        )
 
         for i, realization in enumerate(iens_active_index):
-            values = from_data[:, i].reshape((nx, ny, nz))
+            values = from_data[:, i].reshape((dim_nx, dim_ny, dim_nz))
             ds = xr.Dataset({"values": (["x", "y", "z"], values)})
             yield int(realization), ds
 
@@ -299,7 +303,7 @@ class Field(ParameterConfig):
             fill_value=np.nan,
         )
 
-    def load_parameter_graph(self) -> nx.Graph:  # type: ignore
+    def load_parameter_graph(self) -> nx.Graph[int]:
         parameter_graph = create_flattened_cube_graph(
             px=self.ertbox_params.nx, py=self.ertbox_params.ny, pz=self.ertbox_params.nz
         )
@@ -322,7 +326,7 @@ class Field(ParameterConfig):
         return self.ertbox_params.nz
 
 
-TRANSFORM_FUNCTIONS = {
+TRANSFORM_FUNCTIONS: Final[dict[str, Callable[[Any], Any]]] = {
     "LN": np.log,
     "LOG": np.log,
     "LN0": lambda v: np.log(v + 0.000001),
@@ -353,7 +357,7 @@ def field_transform(
 ) -> npt.NDArray[np.float32] | xr.DataArray:
     if transform_name is None:
         return data
-    return TRANSFORM_FUNCTIONS[transform_name](data)  # type: ignore
+    return TRANSFORM_FUNCTIONS[transform_name](data)
 
 
 def _field_truncate(data: npt.ArrayLike, min_: float | None, max_: float | None) -> Any:

ert/config/forward_model_step.py

@@ -1,13 +1,33 @@
 from __future__ import annotations
 
 import logging
-from typing import ClassVar, Literal, NotRequired, Self
-
-from pydantic import BaseModel, Field, field_validator, model_validator
+from typing import (
+    TYPE_CHECKING,
+    Annotated,
+    Any,
+    ClassVar,
+    Literal,
+    NotRequired,
+    Self,
+    cast,
+)
+
+from pydantic import (
+    BaseModel,
+    Field,
+    field_validator,
+    model_serializer,
+    model_validator,
+)
+from pydantic_core.core_schema import ValidationInfo
 from typing_extensions import TypedDict, Unpack
 
+from ..base_model_context import BaseModelWithContextSupport
 from .parsing import ConfigValidationError, ConfigWarning, SchemaItemType
 
+if TYPE_CHECKING:
+    from ert.plugins import ErtRuntimePlugins
+
 logger = logging.getLogger(__name__)
 
 
@@ -72,6 +92,7 @@ class ForwardModelStepOptions(TypedDict, total=False):
     max_running_minutes: NotRequired[int]
     environment: NotRequired[dict[str, str]]
     default_mapping: NotRequired[dict[str, str]]
+    required_keywords: NotRequired[list[str]]
 
 
 class ForwardModelStepDocumentation(BaseModel):
@@ -91,7 +112,7 @@ class ForwardModelStepDocumentation(BaseModel):
     ) = Field(default="Uncategorized")
 
 
-class ForwardModelStep(BaseModel):
+class ForwardModelStep(BaseModelWithContextSupport):
     """
     Holds information to execute one step of a forward model
 
@@ -202,7 +223,87 @@ class ForwardModelStep(BaseModel):
         return None if v == "null" else v
 
 
-class ForwardModelStepPlugin(ForwardModelStep):
+class UserInstalledForwardModelStep(ForwardModelStep):
+    """
+    Represents a forward model step installed by a user via the ERT Config
+    forward model step format provided via the INSTALL_JOB keyword.
+    User-installed forward model steps serialize with their full configuration,
+    unlike site-installed steps which only serialize as references.
+    """
+
+    type: Literal["user_installed"] = "user_installed"
+
+
+class _SerializedSiteInstalledForwardModelStep(TypedDict):
+    type: Literal["site_installed"]
+    name: str
+    private_args: dict[str, str]
+
+
+class SiteInstalledForwardModelStep(ForwardModelStep):
+    """
+    Represents a forward model step installed via external plugins.
+    Instances of this class serialize only as references to the plugin by name, and
+    the user-provided private_args, allowing them to dynamically update when plugins
+    change, rather than being locked to a specific executable at serialization time.
+    """
+
+    type: Literal["site_installed"] = "site_installed"
+
+    @model_serializer(mode="plain")
+    def serialize_model(self) -> _SerializedSiteInstalledForwardModelStep:
+        return {
+            "type": "site_installed",
+            "name": self.name,
+            "private_args": self.private_args,
+        }
+
+    @model_validator(mode="before")
+    @classmethod
+    def deserialize_model(
+        cls, values: dict[str, Any], info: ValidationInfo
+    ) -> dict[str, Any]:
+        runtime_plugins = cast("ErtRuntimePlugins", info.context)
+        name = values["name"]
+
+        if runtime_plugins is None:
+            if values.get("type") == "site_installed":
+                msg = (
+                    f"Trying to find site-installed forward model step {name} "
+                    f"without site plugins. This forward model must be loaded "
+                    f"with ERT site plugins available."
+                )
+                raise KeyError(msg)
+            return values
+
+        if name not in runtime_plugins.installed_forward_model_steps:
+            msg = (
+                f"Expected forward model step {name} to be installed "
+                f"via plugins, but it was not found. Please check that "
+                f"your python environment has it installed."
+            )
+            raise KeyError(msg)
+        site_installed_fm = runtime_plugins.installed_forward_model_steps[name]
+
+        # Intent: copy the site installed forward model to this instance.
+        # bypassing the model_serializer
+        site_fm_instance = {
+            k: getattr(site_installed_fm, k)
+            for k in SiteInstalledForwardModelStep.model_fields
+        }
+
+        return site_fm_instance | (
+            {"private_args": values["private_args"]} if "private_args" in values else {}
+        )
+
+
+SiteOrUserForwardModelStep = Annotated[
+    (UserInstalledForwardModelStep | SiteInstalledForwardModelStep),
+    Field(discriminator="type"),
+]
+
+
+class ForwardModelStepPlugin(SiteInstalledForwardModelStep):
     def __init__(
         self, name: str, command: list[str], **kwargs: Unpack[ForwardModelStepOptions]
     ) -> None:
@@ -221,6 +322,7 @@ class ForwardModelStepPlugin(ForwardModelStep):
         max_running_minutes = kwargs.get("max_running_minutes")
         environment = kwargs.get("environment", {}) or {}
         default_mapping = kwargs.get("default_mapping", {}) or {}
+        required_keywords = kwargs.get("required_keywords", []) or []
 
         super().__init__(
             name=name,
@@ -235,7 +337,7 @@ class ForwardModelStepPlugin(ForwardModelStep):
             max_running_minutes=max_running_minutes,
             min_arg=0,
             max_arg=0,
-            required_keywords=[],
+            required_keywords=required_keywords,
             arg_types=[],
             environment=environment,
             default_mapping=default_mapping,

ert/config/gen_data_config.py

@@ -210,16 +210,12 @@ class GenDataConfig(ResponseConfig):
         return combined
 
     def get_args_for_key(self, key: str) -> tuple[str | None, list[int] | None]:
-        for i, _key in enumerate(self.keys):
-            if key == _key:
+        for i, key_ in enumerate(self.keys):
+            if key == key_:
                 return self.input_files[i], self.report_steps_list[i]
 
         return None, None
 
-    @property
-    def response_type(self) -> str:
-        return "gen_data"
-
     @property
     def primary_key(self) -> list[str]:
         return ["report_step", "index"]

ert/config/gen_kw_config.py

@@ -248,15 +248,6 @@ class GenKwConfig(ParameterConfig):
     def group_name(self) -> str:
         return self.group
 
-    def copy_parameters(
-        self,
-        source_ensemble: Ensemble,
-        target_ensemble: Ensemble,
-        realizations: npt.NDArray[np.int_],
-    ) -> None:
-        df = source_ensemble.load_parameters(self.name, realizations)
-        target_ensemble.save_parameters(dataset=df)
-
     def get_priors(self) -> list[PriorDict]:
         dist_json = self.distribution.model_dump(exclude={"name"})
         return [

ert/config/known_response_types.py

@@ -0,0 +1,14 @@
+from .everest_response import EverestConstraintsConfig, EverestObjectivesConfig
+from .gen_data_config import GenDataConfig
+from .rft_config import RFTConfig
+from .summary_config import SummaryConfig
+
+KnownErtResponseTypes = SummaryConfig | GenDataConfig | RFTConfig
+KNOWN_ERT_RESPONSE_TYPES = (
+    SummaryConfig,
+    GenDataConfig,
+    RFTConfig,
+)
+KnownResponseTypes = (
+    KnownErtResponseTypes | EverestConstraintsConfig | EverestObjectivesConfig
+)

ert/config/parameter_config.py

@@ -108,23 +108,6 @@ class ParameterConfig(BaseModel):
         or polars DataFrame from the numpy data
         """
 
-    def copy_parameters(
-        self,
-        source_ensemble: Ensemble,
-        target_ensemble: Ensemble,
-        realizations: npt.NDArray[np.int_],
-    ) -> None:
-        """
-        Copy parameters from one ensemble to another.
-        If realizations is None, copy all realizations.
-        If realizations is given, copy only those realizations.
-        """
-        for realization in realizations:
-            # Converts to standard python scalar due to mypy
-            realization_int = int(realization)
-            ds = source_ensemble.load_parameters(self.name, realization_int)
-            target_ensemble.save_parameters(ds, self.name, realization_int)
-
     @abstractmethod
     def load_parameters(
         self, ensemble: Ensemble, realizations: npt.NDArray[np.int_]

ert/config/parsing/config_keywords.py

@@ -42,6 +42,7 @@ class ConfigKeys(StrEnum):
     SETENV = "SETENV"
     STD_CUTOFF = "STD_CUTOFF"
     SUMMARY = "SUMMARY"
+    RFT = "RFT"
     SURFACE = "SURFACE"
     UPDATE_LOG_PATH = "UPDATE_LOG_PATH"
     RANDOM_SEED = "RANDOM_SEED"

ert/config/parsing/config_schema.py

@@ -63,6 +63,17 @@ def data_kw_keyword() -> SchemaItem:
     )
 
 
+def rft_keyword() -> SchemaItem:
+    return SchemaItem(
+        kw=ConfigKeys.RFT,
+        required_set=False,
+        multi_occurrence=True,
+        options_after=0,
+        argc_min=1,
+        argc_max=1,
+    )
+
+
 def define_keyword() -> SchemaItem:
     return SchemaItem(
         kw=ConfigKeys.DEFINE,
@@ -303,6 +314,7 @@ def init_user_config_schema() -> ConfigSchemaDict:
         gen_kw_keyword(),
         gen_data_keyword(),
         summary_keyword(),
+        rft_keyword(),
         surface_keyword(),
         field_keyword(),
         single_arg_keyword(ConfigKeys.ECLBASE),

ert/config/parsing/config_schema_deprecations.py

@@ -217,4 +217,15 @@ deprecated_keywords_list = [
         ),
         check=lambda line: line[0] == "DESIGN2PARAMS",
     ),
+    DeprecationInfo(
+        keyword="FORWARD_MODEL",
+        message=(
+            "FORWARD_MODEL DESIGN_KW will be replaced with RUN_TEMPLATE. "
+            "DESIGN2PARAMS has been replaced by DESIGN_MATRIX, so the "
+            "parameters are already available for magic string replacement "
+            "with the RUN_TEMPLATE keyword. Please use this format: "
+            "'RUN_TEMPLATE my_text_file_template.txt my_text_output_file.txt'"
+        ),
+        check=lambda line: line[0] == "DESIGN_KW",
+    ),
 ]

ert/config/parsing/config_schema_item.py

@@ -48,7 +48,7 @@ class SchemaItem:
     # if positive, arguments after this count will be concatenated with a " " between
     join_after: PositiveInt | None = None
     # if positive, arguments after this count will be interpreted as options
-    options_after: PositiveInt | Varies | None = None
+    options_after: NonNegativeInt | Varies | None = None
     # if true, will accumulate many values set for key, otherwise each entry will
     # overwrite any previous value set
     multi_occurrence: bool = False

ert/config/queue_config.py

@@ -478,11 +478,11 @@ class QueueConfig(BaseModelWithContextSupport):
 
         # validate all queue options for the unselected queues
         # and show a warning
-        for _queue_system in QueueSystem:
-            if _queue_system != selected_queue_system:
+        for queue_system in QueueSystem:
+            if queue_system != selected_queue_system:
                 _ = QueueOptions.create_queue_options(
-                    _queue_system,
-                    grouped_queue_options[_queue_system],
+                    queue_system,
+                    grouped_queue_options[queue_system],
                     False,
                 )
 

ert/config/response_config.py

@@ -60,13 +60,6 @@ class ResponseConfig(BaseModel):
     def expected_input_files(self) -> list[str]:
         """Returns a list of filenames expected to be produced by the forward model"""
 
-    @property
-    @abstractmethod
-    def response_type(self) -> str:
-        """Label to identify what kind of response it is.
-        Must not overlap with that of other response configs."""
-        ...
-
     @property
     @abstractmethod
     def primary_key(self) -> list[str]: