ert 17.0.0__py3-none-any.whl → 19.0.0rc2__py3-none-any.whl

ert/config/ert_config.py

@@ -25,6 +25,7 @@ from ._design_matrix_validator import DesignMatrixValidator
 from ._observations import (
     HistoryObservation,
     Observation,
+    RFTObservation,
     SummaryObservation,
     make_observations,
 )
@@ -36,9 +37,12 @@ from .forward_model_step import (
     ForwardModelStepJSON,
     ForwardModelStepValidationError,
     ForwardModelStepWarning,
+    SiteInstalledForwardModelStep,
+    SiteOrUserForwardModelStep,
+    UserInstalledForwardModelStep,
 )
 from .gen_data_config import GenDataConfig
-from .gen_kw_config import GenKwConfig
+from .gen_kw_config import DataSource, GenKwConfig
 from .model_config import ModelConfig
 from .parse_arg_types_list import parse_arg_types_list
 from .parsing import (
@@ -59,6 +63,7 @@ from .parsing import (
 from .parsing.observations_parser import ObservationDict
 from .queue_config import KnownQueueOptions, QueueConfig
 from .refcase import Refcase
+from .rft_config import RFTConfig
 from .workflow import Workflow
 from .workflow_fixtures import fixtures_per_hook
 from .workflow_job import (
@@ -133,6 +138,7 @@ def create_forward_model_json(
         env_pr_fm_step = {}
 
     context_substitutions = Substitutions(context)
+    real_iter_substituter = context_substitutions.real_iter_substituter(iens, itr)
 
     class Substituter:
         def __init__(self, fm_step: ForwardModelStep) -> None:
@@ -146,7 +152,7 @@ def create_forward_model_json(
             )
             self.copy_private_args = Substitutions(
                 {
-                    key: context_substitutions.substitute_real_iter(val, iens, itr)
+                    key: real_iter_substituter.substitute(val)
                     for key, val in fm_step.private_args.items()
                 }
             )
@@ -163,7 +169,7 @@ def create_forward_model_json(
             string = self.copy_private_args.substitute(
                 string, self.substitution_context_hint, 1, warn_max_iter=False
             )
-            return context_substitutions.substitute_real_iter(string, iens, itr)
+            return real_iter_substituter.substitute(string)
 
         def filter_env_dict(self, env_dict: dict[str, str]) -> dict[str, str] | None:
             substituted_dict = {}
@@ -518,18 +524,16 @@ def workflows_from_dict(
 
 def installed_forward_model_steps_from_dict(
     config_dict: ConfigDict,
-) -> dict[str, ForwardModelStep]:
+) -> dict[str, UserInstalledForwardModelStep]:
     errors: list[ErrorInfo | ConfigValidationError] = []
-    fm_steps: dict[str, ForwardModelStep] = {}
+    fm_steps: dict[str, UserInstalledForwardModelStep] = {}
     for name, (fm_step_config_file, config_contents) in config_dict.get(
         ConfigKeys.INSTALL_JOB, []
     ):
         fm_step_config_file = path.abspath(fm_step_config_file)
         try:
             new_fm_step = forward_model_step_from_config_contents(
-                config_contents,
-                name=name,
-                config_file=fm_step_config_file,
+                config_contents, name=name, config_file=fm_step_config_file
             )
         except ConfigValidationError as e:
             errors.append(e)
@@ -692,6 +696,8 @@ def log_observation_keys(
 
 RESERVED_KEYWORDS = ["realization", "IENS", "ITER"]
 
+USER_CONFIG_SCHEMA = init_user_config_schema()
+
 
 class ErtConfig(BaseModel):
     DEFAULT_ENSPATH: ClassVar[str] = "storage"
@@ -703,6 +709,7 @@ class ErtConfig(BaseModel):
     QUEUE_OPTIONS: ClassVar[KnownQueueOptions | None] = None
     RESERVED_KEYWORDS: ClassVar[list[str]] = RESERVED_KEYWORDS
     ENV_VARS: ClassVar[dict[str, str]] = {}
+    PRIORITIZE_PRIVATE_IP_ADDRESS: ClassVar[bool] = False
 
     substitutions: dict[str, str] = Field(default_factory=dict)
     ensemble_config: EnsembleConfig = Field(default_factory=EnsembleConfig)
@@ -717,10 +724,11 @@ class ErtConfig(BaseModel):
         default_factory=lambda: defaultdict(lambda: cast(list[Workflow], []))
     )
     runpath_file: Path = Path(DEFAULT_RUNPATH_FILE)
+    prioritize_private_ip_address: bool = False
 
     ert_templates: list[tuple[str, str]] = Field(default_factory=list)
 
-    forward_model_steps: list[ForwardModelStep] = Field(default_factory=list)
+    forward_model_steps: list[SiteOrUserForwardModelStep] = Field(default_factory=list)
     runpath_config: ModelConfig = Field(default_factory=ModelConfig)
     user_config_file: str = "no_config"
     config_path: str = Field(init=False, default="")
@@ -733,6 +741,18 @@ class ErtConfig(BaseModel):
     @property
     def observations(self) -> dict[str, pl.DataFrame]:
         if self._observations is None:
+            has_rft_observations = any(
+                isinstance(o, RFTObservation) for o in self.observation_declarations
+            )
+            if (
+                has_rft_observations
+                and "rft" not in self.ensemble_config.response_configs
+            ):
+                self.ensemble_config.response_configs["rft"] = RFTConfig(
+                    input_files=[self.runpath_config.eclbase_format_string],
+                    data_to_read={},
+                    locations=[],
+                )
             computed = create_observation_dataframes(
                 self.observation_declarations,
                 self.refcase,
@@ -740,6 +760,10 @@ class ErtConfig(BaseModel):
                     GenDataConfig | None,
                     self.ensemble_config.response_configs.get("gen_data", None),
                 ),
+                cast(
+                    RFTConfig | None,
+                    self.ensemble_config.response_configs.get("rft", None),
+                ),
                 self.time_map,
                 self.history_source,
             )
@@ -791,6 +815,15 @@ class ErtConfig(BaseModel):
             )
         return self
 
+    @model_validator(mode="after")
+    def log_ensemble_config_contents(self) -> Self:
+        all_parameters = self.parameter_configurations_with_design_matrix
+        parameter_type_count = Counter(parameter.type for parameter in all_parameters)
+        logger.info(
+            f"EnsembleConfig contains parameters of type {dict(parameter_type_count)}"
+        )
+        return self
+
     def __eq__(self, other: object) -> bool:
         if not isinstance(other, ErtConfig):
             return False
@@ -817,7 +850,7 @@ class ErtConfig(BaseModel):
     def with_plugins(runtime_plugins: ErtRuntimePlugins) -> type[ErtConfig]:
         class ErtConfigWithPlugins(ErtConfig):
             PREINSTALLED_FORWARD_MODEL_STEPS: ClassVar[
-                Mapping[str, ForwardModelStep]
+                Mapping[str, SiteInstalledForwardModelStep]
             ] = runtime_plugins.installed_forward_model_steps
             PREINSTALLED_WORKFLOWS = dict(runtime_plugins.installed_workflow_jobs)
             ENV_PR_FM_STEP: ClassVar[dict[str, dict[str, Any]]] = (
@@ -827,6 +860,9 @@ class ErtConfig(BaseModel):
             )
             ENV_VARS = dict(runtime_plugins.environment_variables)
             QUEUE_OPTIONS = runtime_plugins.queue_options
+            PRIORITIZE_PRIVATE_IP_ADDRESS = (
+                runtime_plugins.prioritize_private_ip_address
+            )
 
         ErtConfigWithPlugins.model_rebuild()
         assert issubclass(ErtConfigWithPlugins, ErtConfig)
@@ -1025,11 +1061,30 @@ class ErtConfig(BaseModel):
                 if isinstance(cfg, GenKwConfig) and cfg.name in dm_params
             ]
             if overwrite_params:
-                ConfigWarning.warn(
-                    f"Parameters {overwrite_params} "
-                    "will be overridden by design matrix. This will cause "
-                    "updates to be turned off for these parameters."
-                )
+                param_sampled = [
+                    k
+                    for k in overwrite_params
+                    if analysis_config.design_matrix.parameter_priority[k]
+                    == DataSource.SAMPLED
+                ]
+                param_design = [
+                    k
+                    for k in overwrite_params
+                    if analysis_config.design_matrix.parameter_priority[k]
+                    == DataSource.DESIGN_MATRIX
+                ]
+                if param_sampled:
+                    ConfigWarning.warn(
+                        f"Parameters {param_sampled} "
+                        "are also defined in design matrix, but due to the sampled"
+                        " priority they will remain as such."
+                    )
+                if param_design:
+                    ConfigWarning.warn(
+                        f"Parameters {param_design} "
+                        "will be overridden by design matrix. This will cause "
+                        "updates to be turned off for these parameters."
+                    )
 
             if dm_errors:
                 raise ConfigValidationError.from_collected(dm_errors)
@@ -1060,6 +1115,19 @@ class ErtConfig(BaseModel):
             user_configured_.add(key)
             env_vars[key] = substituter.substitute(val)
 
+        prioritize_private_ip_address: bool = cls.PRIORITIZE_PRIVATE_IP_ADDRESS
+        if ConfigKeys.PRIORITIZE_PRIVATE_IP_ADDRESS in config_dict:
+            user_prioritize_private_ip_address = bool(
+                config_dict[ConfigKeys.PRIORITIZE_PRIVATE_IP_ADDRESS]
+            )
+            if prioritize_private_ip_address != user_prioritize_private_ip_address:
+                logger.warning(
+                    "PRIORITIZE_PRIVATE_IP_ADDRESS was overwritten by user: "
+                    f"{prioritize_private_ip_address} -> "
+                    f"{user_prioritize_private_ip_address}"
+                )
+                prioritize_private_ip_address = user_prioritize_private_ip_address
+
         try:
             refcase = Refcase.from_config_dict(config_dict)
             cls_config = cls(
@@ -1086,11 +1154,21 @@ class ErtConfig(BaseModel):
                 time_map=time_map,
                 history_source=history_source,
                 refcase=refcase,
+                prioritize_private_ip_address=prioritize_private_ip_address,
             )
 
             # The observations are created here because create_observation_dataframes
             # will perform additonal validation which needs the context in
             # obs_configs which is stripped by pydantic
+            has_rft_observations = any(
+                isinstance(o, RFTObservation) for o in obs_configs
+            )
+            if has_rft_observations and "rft" not in ensemble_config.response_configs:
+                ensemble_config.response_configs["rft"] = RFTConfig(
+                    input_files=[eclbase],
+                    data_to_read={},
+                    locations=[],
+                )
             cls_config._observations = create_observation_dataframes(
                 obs_configs,
                 refcase,
@@ -1098,6 +1176,10 @@ class ErtConfig(BaseModel):
                     GenDataConfig | None,
                     ensemble_config.response_configs.get("gen_data", None),
                 ),
+                cast(
+                    RFTConfig | None,
+                    ensemble_config.response_configs.get("rft", None),
+                ),
                 time_map,
                 history_source,
             )
@@ -1275,7 +1357,7 @@ class ErtConfig(BaseModel):
     @classmethod
     def _read_user_config_contents(cls, user_config: str, file_name: str) -> ConfigDict:
         return parse_contents(
-            user_config, file_name=file_name, schema=init_user_config_schema()
+            user_config, file_name=file_name, schema=USER_CONFIG_SCHEMA
         )
 
     @classmethod
@@ -1365,8 +1447,10 @@ def uppercase_subkeys_and_stringify_subvalues(
 
 
 def forward_model_step_from_config_contents(
-    config_contents: str, config_file: str, name: str | None = None
-) -> ForwardModelStep:
+    config_contents: str,
+    config_file: str,
+    name: str | None = None,
+) -> UserInstalledForwardModelStep:
     if name is None:
         name = os.path.basename(config_file)
 
@@ -1390,7 +1474,7 @@ def forward_model_step_from_config_contents(
     environment = {k: v for [k, v] in content_dict.get("ENV", [])}
     default_mapping = {k: v for [k, v] in content_dict.get("DEFAULT", [])}
 
-    return ForwardModelStep(
+    return UserInstalledForwardModelStep(
         name=name,
         executable=content_dict["EXECUTABLE"],
         stdin_file=content_dict.get("STDIN"),

ert/config/everest_control.py

@@ -0,0 +1,234 @@
+from __future__ import annotations
+
+import importlib
+import json
+import logging
+from collections.abc import Iterator, Mapping, MutableMapping
+from dataclasses import field
+from pathlib import Path
+from textwrap import dedent
+from typing import TYPE_CHECKING, Any, Literal, Self
+
+import networkx as nx
+import numpy as np
+import xarray as xr
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+from ropt.workflow import find_sampler_plugin
+
+from ert.substitutions import substitute_runpath_name
+
+from .parameter_config import ParameterConfig
+
+if TYPE_CHECKING:
+    import numpy.typing as npt
+
+    from ert.storage import Ensemble
+
+    Number = int | float
+    DataType = Mapping[str, Number | Mapping[str, Number]]
+    MutableDataType = MutableMapping[str, Number | MutableMapping[str, Number]]
+
+
+class SamplerConfig(BaseModel):
+    backend: str | None = Field(
+        default=None,
+        description=dedent(
+            """
+            [Deprecated]
+
+            The correct backend will be inferred by the method. If several backends
+            have a method named `A`, pick a specific backend `B` by putting `B/A` in
+            the `method` field.
+            """
+        ),
+    )
+    method: str = Field(
+        default="norm",
+        description=dedent(
+            """
+            The sampling method or distribution used by the sampler backend.
+
+            The set of available methods depends on the sampler backend used. By
+            default a plugin based on `scipy.stats` is used, implementing the
+            following methods:
+
+            - From Probability Distributions:
+                - `norm`: Samples from a standard normal distribution (mean 0,
+                  standard deviation 1).
+                - `truncnorm`: Samples from a truncated normal distribution
+                  (mean 0, std. dev. 1), truncated to the range `[-1, 1]`.
+                - `uniform`: Samples from a uniform distribution in the range
+                  `[-1, 1]`.
+
+            - From Quasi-Monte Carlo Sequences:
+                - `sobol`: Uses Sobol' sequences.
+                - `halton`: Uses Halton sequences.
+                - `lhs`: Uses Latin Hypercube Sampling.
+
+                Note: QMC samples are generated in the unit hypercube `[0, 1]^d`
+                and then scaled to the hypercube `[-1, 1]^d`.
+            """
+        ),
+    )
+    options: dict[str, Any] | None = Field(
+        default=None,
+        description=dedent(
+            """
+            Specifies a dict of optional parameters for the sampler backend.
+
+            This dict of values is passed unchanged to the selected method in
+            the backend.
+            """
+        ),
+    )
+    shared: bool | None = Field(
+        default=None,
+        description=dedent(
+            """
+            Whether to share perturbations between realizations.
+            """
+        ),
+    )
+    model_config = ConfigDict(extra="forbid")
+
+    @model_validator(mode="after")
+    def validate_backend_and_method(self) -> Self:
+        if self.backend is not None:
+            message = (
+                "sampler.backend is deprecated. "
+                "The correct backend will be inferred by the method. "
+                "If several backends have a method named A, you need to pick "
+                "a specific backend B by putting B/A in sampler.method."
+            )
+            print(message)
+            # Note: Importing EVEREST.everest
+            # leads to circular import, but we still wish to log
+            # from "everest" here as per old behavior.
+            # Can consider logging this as if from ERT,
+            # which is valid if we store SamplerConfig as part of
+            # EverestControl configs.
+            logging.getLogger("everest").warning(message)
+
+        # Update the default for backends that are not scipy:
+        if (
+            self.backend not in {None, "scipy"}
+            and "method" not in self.model_fields_set
+        ):
+            self.method = "default"
+
+        if self.backend is not None:
+            self.method = f"{self.backend}/{self.method}"
+
+        try:
+            plugin = find_sampler_plugin(f"{self.method}")
+        except ValueError:
+            raise
+        except Exception as exc:
+            ert_version = importlib.metadata.version("ert")
+            ropt_version = importlib.metadata.version("ropt")
+            msg = (
+                f"Error while initializing ropt:\n\n{exc}.\n\n"
+                "There may a be version mismatch between "
+                f"ERT ({ert_version}) and ropt ({ropt_version})\n"
+                "If the installation is correct, please report this as a bug."
+            )
+            raise RuntimeError(msg) from exc
+
+        if plugin is None:
+            raise ValueError(f"Sampler method '{self.method}' not found")
+
+        self.backend = None
+
+        return self
+
+
+class EverestControl(ParameterConfig):
+    """Create an EverestControl for @key with the given @input_keys
+
+    @input_keys can be either a list of keys as strings or a dict with
+    keys as strings and a list of suffixes for each key.
+    If a list of strings is given, the order is preserved.
+    """
+
+    type: Literal["everest_parameters"] = "everest_parameters"
+    input_keys: list[str] = field(default_factory=list)
+    forward_init: bool = False
+    output_file: str = ""
+    forward_init_file: str = ""
+    update: bool = False
+    types: list[Literal["well_control", "generic_control"]]
+    initial_guesses: list[float]
+    control_types: list[Literal["real", "integer"]]
+    enabled: list[bool]
+    min: list[float]
+    max: list[float]
+    perturbation_types: list[Literal["absolute", "relative"]]
+    perturbation_magnitudes: list[float]
+    scaled_ranges: list[tuple[float, float]]
+    samplers: list[SamplerConfig | None]
+
+    # Set up for deprecation, but has to live here until support for the
+    # "dotdash" notation is removed for everest controls via everest config.
+    input_keys_dotdash: list[str] = field(default_factory=list)
+
+    @property
+    def parameter_keys(self) -> list[str]:
+        return self.input_keys
+
+    def read_from_runpath(
+        self, run_path: Path, real_nr: int, iteration: int
+    ) -> xr.Dataset:
+        raise NotImplementedError
+
+    def write_to_runpath(
+        self, run_path: Path, real_nr: int, ensemble: Ensemble
+    ) -> None:
+        file_path = run_path / substitute_runpath_name(
+            self.output_file, real_nr, ensemble.iteration
+        )
+        Path.mkdir(file_path.parent, exist_ok=True, parents=True)
+
+        data: MutableDataType = {}
+        for da in ensemble.load_parameters(self.name, real_nr)["values"]:
+            assert isinstance(da, xr.DataArray)
+            name = str(da.names.values)
+            try:
+                outer, inner = name.split("\0")
+
+                if outer not in data:
+                    data[outer] = {}
+                data[outer][inner] = float(da)  # type: ignore
+            except ValueError:
+                data[name] = float(da)
+
+        file_path.write_text(json.dumps(data), encoding="utf-8")
+
+    def create_storage_datasets(
+        self,
+        from_data: npt.NDArray[np.float64],
+        iens_active_index: npt.NDArray[np.int_],
+    ) -> Iterator[tuple[int, xr.Dataset]]:
+        for i, realization in enumerate(iens_active_index):
+            yield (
+                int(realization),
+                xr.Dataset(
+                    {
+                        "values": ("names", from_data[:, i]),
+                        "names": [
+                            x.split(f"{self.name}.")[1].replace(".", "\0")
+                            for x in self.parameter_keys
+                        ],
+                    }
+                ),
+            )
+
+    def load_parameters(
+        self, ensemble: Ensemble, realizations: npt.NDArray[np.int_]
+    ) -> npt.NDArray[np.float64]:
+        raise NotImplementedError
+
+    def load_parameter_graph(self) -> nx.Graph[int]:
+        raise NotImplementedError
+
+    def __len__(self) -> int:
+        return len(self.input_keys)

ert/config/{everest_objective_config.py → everest_response.py}

@@ -11,19 +11,21 @@ from .response_config import InvalidResponseFile, ResponseConfig, ResponseMetada
 from .responses_index import responses_index
 
 
-class EverestObjectivesConfig(ResponseConfig):
-    type: Literal["everest_objectives"] = "everest_objectives"
-    name: str = "everest_objectives"
+class EverestResponse(ResponseConfig):
+    """Base class for Everest response configurations."""
+
     has_finalized_keys: bool = True
-    weights: list[float | None]
     scales: list[float | None]
-    objective_types: list[Literal["mean", "stddev"]]
+
+    @property
+    def primary_key(self) -> list[str]:
+        return []
 
     @property
     def metadata(self) -> list[ResponseMetadata]:
         return [
             ResponseMetadata(
-                response_type=self.name,
+                response_type=self.type,
                 response_key=response_key,
                 finalized=self.has_finalized_keys,
                 filter_on=None,
@@ -52,7 +54,6 @@ class EverestObjectivesConfig(ResponseConfig):
             )
 
         errors = []
-
         run_path_ = Path(run_path)
         datasets_per_name = []
 
@@ -75,24 +76,32 @@ class EverestObjectivesConfig(ResponseConfig):
             if all(isinstance(err, FileNotFoundError) for err in errors):
                 raise FileNotFoundError(
                     "Could not find one or more files/directories while reading "
-                    f"{self.name}: {','.join([str(err) for err in errors])}"
+                    f"{self.type}: {','.join([str(err) for err in errors])}"
                 )
             else:
                 raise InvalidResponseFile(
                     "Error reading "
-                    f"{self.name}, errors: {','.join([str(err) for err in errors])}"
+                    f"{self.type}, errors: {','.join([str(err) for err in errors])}"
                 )
 
         combined = pl.concat(datasets_per_name)
         return combined
 
-    @property
-    def response_type(self) -> str:
-        return "everest_objectives"
 
-    @property
-    def primary_key(self) -> list[str]:
-        return []
+class EverestConstraintsConfig(EverestResponse):
+    type: Literal["everest_constraints"] = "everest_constraints"
+    targets: list[float | None]
+    upper_bounds: list[float]
+    lower_bounds: list[float]
+
+
+responses_index.add_response_type(EverestConstraintsConfig)
+
+
+class EverestObjectivesConfig(EverestResponse):
+    type: Literal["everest_objectives"] = "everest_objectives"
+    weights: list[float | None]
+    objective_types: list[Literal["mean", "stddev"]]
 
 
 responses_index.add_response_type(EverestObjectivesConfig)