ert 19.0.0rc4__py3-none-any.whl → 20.0.0b0__py3-none-any.whl

ert/config/parameter_config.py

@@ -126,35 +126,28 @@ class ParameterConfig(BaseModel):
     def transform_data(self) -> Callable[[float], float]:
         return lambda x: x
 
-    def sample_value(
-        self,
-        global_seed: str,
-        realization: int,
+    def sample_values(
+        self, global_seed: str, active_realizations: list[int], num_realizations: int
     ) -> npt.NDArray[np.double]:
         """
-        Generate a sample value for each key in a parameter group.
+        Generate reproducible standard-normal samples for active realizations.
 
-        The sampling is reproducible and dependent on a global seed combined
-        with the parameter group name and individual key names. The 'realization'
-        parameter determines the specific sample point from the distribution for each
-        parameter.
+        For this parameter (identified by self.group_name and self.name), a random
+        sampling of size `num_realizations` is constructed using an RNG
+        seeded from `global_seed` and the parameter name. The entries at the
+        indices specified by `active_realizations` are then returned.
 
         Parameters:
-        - global_seed (str): A global seed string used for RNG seed generation to ensure
-        reproducibility across runs.
-        - realization (int): An integer used to advance the RNG to a specific point in
-        its sequence, effectively selecting the 'realization'-th sample from the
-        distribution.
+        - global_seed (str): A global seed string used for RNG seed generation to
+        ensure reproducibility across runs.
+        - active_realizations (list[int]): indices of the realizations
+        to select from the sampling vector; each must satisfy 0 <= i < num_realizations.
+        - num_realizations (int): Total number of realizations. Assures stable sampling
+        for a given global_seed regardless of currently active realizations.
 
         Returns:
-        - npt.NDArray[np.double]: An array of sample values, one for each key in the
-        provided list.
-
-        Note:
-        The method uses SHA-256 for hash generation and numpy's default random number
-        generator for sampling. The RNG state is advanced to the 'realization' point
-        before generating a single sample, enhancing efficiency by avoiding the
-        generation of large, unused sample sets.
+        - npt.NDArray[np.double]: Array of shape (len(active_realizations),
+        containing sample values, one for each `active_realization`.
         """
         key_hash = sha256(
             global_seed.encode("utf-8") + f"{self.group_name}:{self.name}".encode()
@@ -162,9 +155,6 @@ class ParameterConfig(BaseModel):
         seed = np.frombuffer(key_hash.digest(), dtype="uint32")
         rng = np.random.default_rng(seed)
 
-        # Advance the RNG state to the realization point
-        rng.standard_normal(realization)
-
-        # Generate a single sample
-        value = rng.standard_normal(1)
-        return np.array([value[0]])
+        # Generate samples for all active realizations
+        all_values = rng.standard_normal(num_realizations)
+        return all_values[active_realizations]

ert/config/parsing/__init__.py

@@ -36,7 +36,6 @@ __all__ = [
     "ObservationConfigError",
     "ObservationDict",
     "ObservationType",
-    "ObservationType",
     "QueueSystem",
     "SchemaItemType",
     "WarningInfo",

ert/config/parsing/_parse_zonemap.py

@@ -0,0 +1,45 @@
+from .config_errors import ConfigValidationError
+
+
+def parse_zonemap(filename: str, content: str) -> dict[int, list[str]]:
+    """A zonemap is a map from a simulation grids layers to a list of zone names
+    that the k-layers belongs to.
+
+    Note that the map uses 1-indexing of layers.
+    """
+    zones_at_k_value: dict[int, list[str]] = {}
+
+    base_err_msg = "On Line {line_number} in the ZONEMAP file: "
+    for idx, line in enumerate(content.splitlines()):
+        line_number = idx + 1
+        zonemap_line = _strip_comments(line).split()
+
+        if not zonemap_line:
+            continue
+
+        if len(zonemap_line) < 2:
+            raise ConfigValidationError.with_context(
+                "Number of zonenames must be 1 or more.",
+                filename,
+            )
+        try:
+            k = int(zonemap_line[0])
+        except ValueError as err:
+            raise ConfigValidationError.with_context(
+                base_err_msg.format(line_number=line_number)
+                + f"k must be an integer, was {zonemap_line[0]}.",
+                filename,
+            ) from err
+        if k <= 0:
+            raise ConfigValidationError.with_context(
+                base_err_msg.format(line_number=line_number)
+                + "k must be at least 1. Layers are 1-indexed.",
+                filename,
+            )
+        zones_at_k_value[k] = [zone.strip() for zone in zonemap_line[1:]]
+
+    return zones_at_k_value
+
+
+def _strip_comments(line: str) -> str:
+    return line.partition("--")[0].rstrip()

ert/config/parsing/config_keywords.py

@@ -31,6 +31,7 @@ class ConfigKeys(StrEnum):
     NUM_REALIZATIONS = "NUM_REALIZATIONS"
     MIN_REALIZATIONS = "MIN_REALIZATIONS"
     OBS_CONFIG = "OBS_CONFIG"
+    ZONEMAP = "ZONEMAP"
     QUEUE_SYSTEM = "QUEUE_SYSTEM"
     QUEUE_OPTION = "QUEUE_OPTION"
     HOOK_WORKFLOW = "HOOK_WORKFLOW"
@@ -56,7 +57,6 @@ class ConfigKeys(StrEnum):
     REALIZATION_MEMORY = "REALIZATION_MEMORY"
     SUBMIT_SLEEP = "SUBMIT_SLEEP"
     MAX_RUNNING = "MAX_RUNNING"
-    PRIORITIZE_PRIVATE_IP_ADDRESS = "PRIORITIZE_PRIVATE_IP_ADDRESS"
 
     def __repr__(self) -> str:
         return f"{self.value!r}"

ert/config/parsing/config_schema.py

@@ -1,3 +1,4 @@
+from ._parse_zonemap import parse_zonemap
 from .config_dict import ConfigDict
 from .config_keywords import ConfigKeys
 from .config_schema_deprecations import deprecated_keywords_list
@@ -126,13 +127,6 @@ def hook_workflow_keyword() -> SchemaItem:
     )
 
 
-def prioritize_private_ip_address_keyword() -> SchemaItem:
-    return SchemaItem(
-        kw=ConfigKeys.PRIORITIZE_PRIVATE_IP_ADDRESS,
-        type_map=[SchemaItemType.BOOL],
-    )
-
-
 def set_env_keyword() -> SchemaItem:
     # You can set environment variables which will be applied to the run-time
     # environment.
@@ -340,6 +334,7 @@ def init_user_config_schema() -> ConfigSchemaDict:
         existing_path_inline_keyword(
             ConfigKeys.OBS_CONFIG, parser=lambda f, c: parse_observations(c, f)
         ),
+        existing_path_inline_keyword(ConfigKeys.ZONEMAP, parser=parse_zonemap),
         existing_path_inline_keyword(ConfigKeys.TIME_MAP),
         single_arg_keyword(ConfigKeys.GEN_KW_EXPORT_NAME),
         history_source_keyword(),
@@ -356,7 +351,6 @@ def init_user_config_schema() -> ConfigSchemaDict:
         install_job_keyword(),
         install_job_directory_keyword(),
         hook_workflow_keyword(),
-        prioritize_private_ip_address_keyword(),
     ]:
         schema[item.kw] = item
         if item.kw in ConfigAliases:

ert/config/parsing/observations_parser.py

@@ -19,6 +19,7 @@ class ObservationType(StrEnum):
     SUMMARY = "SUMMARY_OBSERVATION"
     GENERAL = "GENERAL_OBSERVATION"
     RFT = "RFT_OBSERVATION"
+    BREAKTHROUGH = "BREAKTHROUGH_OBSERVATION"
 
 
 ObservationDict = dict[str, Any]
@@ -127,6 +128,7 @@ observations_parser = Lark(
       | "SUMMARY_OBSERVATION"
       | "GENERAL_OBSERVATION"
       | "RFT_OBSERVATION"
+      | "BREAKTHROUGH_OBSERVATION"
     type: TYPE
     ?value: object
           | STRING

ert/config/response_config.py

@@ -14,35 +14,12 @@ class InvalidResponseFile(Exception):
     """
 
 
-class ResponseMetadata(BaseModel):
-    response_type: str
-    response_key: str
-    finalized: bool
-    filter_on: dict[str, list[Any]] | None = Field(
-        default=None,
-        description="""
-    Holds information about which columns the response can be filtered on.
-    For example, for gen data, { "report_step": [0, 199, 299] } indicates
-    that we can filter the response by report step with the potential values
-    [0, 199, 299].
-    """,
-    )
-
-
 class ResponseConfig(BaseModel):
     type: str
     input_files: list[str] = Field(default_factory=list)
     keys: list[str] = Field(default_factory=list)
     has_finalized_keys: bool = False
 
-    @property
-    @abstractmethod
-    def metadata(self) -> list[ResponseMetadata]:
-        """
-        Returns metadata describing this response
-
-        """
-
     @abstractmethod
     def read_from_file(self, run_path: str, iens: int, iter_: int) -> pl.DataFrame:
         """Reads the data for the response from run_path.
@@ -76,3 +53,8 @@ class ResponseConfig(BaseModel):
     def display_column(cls, value: Any, column_name: str) -> str:
         """Formats a value to a user-friendly displayable format."""
         return str(value)
+
+    @property
+    def filter_on(self) -> dict[str, dict[str, list[int]]] | None:
+        """Optional filters for this response."""
+        return None

ert/config/rft_config.py

@@ -5,8 +5,9 @@ import fnmatch
 import logging
 import os
 import re
+import warnings
 from collections import defaultdict
-from typing import IO, Any, Literal
+from typing import IO, Any, Literal, TypeAlias
 
 import numpy as np
 import numpy.typing as npt
@@ -15,32 +16,27 @@ from pydantic import Field
 from resfo_utilities import CornerpointGrid, InvalidRFTError, RFTReader
 
 from ert.substitutions import substitute_runpath_name
+from ert.warnings import PostSimulationWarning
 
 from .parsing import ConfigDict, ConfigKeys, ConfigValidationError, ConfigWarning
-from .response_config import InvalidResponseFile, ResponseConfig, ResponseMetadata
+from .response_config import InvalidResponseFile, ResponseConfig
 from .responses_index import responses_index
 
 logger = logging.getLogger(__name__)
 
 
+Point: TypeAlias = tuple[float, float, float]
+GridIndex: TypeAlias = tuple[int, int, int]
+ZoneName: TypeAlias = str
+
+
 class RFTConfig(ResponseConfig):
     type: Literal["rft"] = "rft"
     name: str = "rft"
     has_finalized_keys: bool = False
     data_to_read: dict[str, dict[str, list[str]]] = Field(default_factory=dict)
-    locations: list[tuple[float, float, float]] = Field(default_factory=list)
-
-    @property
-    def metadata(self) -> list[ResponseMetadata]:
-        return [
-            ResponseMetadata(
-                response_type=self.name,
-                response_key=response_key,
-                filter_on=None,
-                finalized=self.has_finalized_keys,
-            )
-            for response_key in self.keys
-        ]
+    locations: list[Point | tuple[Point, ZoneName]] = Field(default_factory=list)
+    zonemap: dict[int, list[ZoneName]] = Field(default_factory=dict)
 
     @property
     def expected_input_files(self) -> list[str]:
@@ -55,11 +51,11 @@ class RFTConfig(ResponseConfig):
 
     def _find_indices(
         self, egrid_file: str | os.PathLike[str] | IO[Any]
-    ) -> dict[tuple[int, int, int] | None, set[tuple[float, float, float]]]:
+    ) -> dict[GridIndex | None, set[Point | tuple[Point, ZoneName]]]:
         indices = defaultdict(set)
         for a, b in zip(
             CornerpointGrid.read_egrid(egrid_file).find_cell_containing_point(
-                self.locations
+                [loc[0] if isinstance(loc[1], str) else loc for loc in self.locations]
             ),
             self.locations,
             strict=True,
@@ -67,6 +63,34 @@ class RFTConfig(ResponseConfig):
             indices[a].add(b)
         return indices
 
+    def _filter_zones(
+        self,
+        indices: dict[GridIndex | None, set[Point | tuple[Point, ZoneName]]],
+        iens: int,
+        iter_: int,
+    ) -> dict[GridIndex | None, set[Point]]:
+        for idx, locs in indices.items():
+            if idx is not None:
+                for loc in list(locs):
+                    if isinstance(loc[1], str):
+                        zone = loc[1]
+                        # zonemap is 1-indexed so +1
+                        if zone not in self.zonemap.get(idx[-1] + 1, []):
+                            warnings.warn(
+                                PostSimulationWarning(
+                                    f"An RFT observation with location {loc[0]}, "
+                                    f"in iteration {iter_}, realization {iens} did "
+                                    f"not match expected zone {zone}. The observation "
+                                    "was deactivated",
+                                ),
+                                stacklevel=2,
+                            )
+                            locs.remove(loc)
+        return {
+            k: {v[0] if isinstance(v[1], str) else v for v in vs}
+            for k, vs in indices.items()
+        }
+
     def read_from_file(self, run_path: str, iens: int, iter_: int) -> pl.DataFrame:
         filename = substitute_runpath_name(self.input_files[0], iens, iter_)
         if filename.upper().endswith(".DATA"):
@@ -83,7 +107,7 @@ class RFTConfig(ResponseConfig):
         )
         indices = {}
         if self.locations:
-            indices = self._find_indices(grid_filename)
+            indices = self._filter_zones(self._find_indices(grid_filename), iens, iter_)
         if None in indices:
             raise InvalidResponseFile(
                 f"Did not find grid coordinate for location(s) {indices[None]}"
@@ -194,7 +218,7 @@ class RFTConfig(ResponseConfig):
                     .explode("location")
                     for (well, time), inner_dict in fetched.items()
                     for prop, vals in inner_dict.items()
-                    if prop != "DEPTH"
+                    if prop != "DEPTH" and len(vals) > 0
                 ]
             )
         except KeyError as err:
@@ -267,6 +291,7 @@ class RFTConfig(ResponseConfig):
                 input_files=[eclbase.replace("%d", "<IENS>")],
                 keys=keys,
                 data_to_read=data_to_read,
+                zonemap=config_dict.get(ConfigKeys.ZONEMAP, ("", {}))[1],
             )
 
         return None

ert/config/summary_config.py

@@ -11,7 +11,7 @@ from ert.substitutions import substitute_runpath_name
 from ._read_summary import read_summary
 from .parsing import ConfigDict, ConfigKeys
 from .parsing.config_errors import ConfigValidationError, ConfigWarning
-from .response_config import InvalidResponseFile, ResponseConfig, ResponseMetadata
+from .response_config import InvalidResponseFile, ResponseConfig
 from .responses_index import responses_index
 
 logger = logging.getLogger(__name__)
@@ -21,18 +21,6 @@ class SummaryConfig(ResponseConfig):
     type: Literal["summary"] = "summary"
     has_finalized_keys: bool = False
 
-    @property
-    def metadata(self) -> list[ResponseMetadata]:
-        return [
-            ResponseMetadata(
-                response_type=self.type,
-                response_key=response_key,
-                filter_on=None,
-                finalized=self.has_finalized_keys,
-            )
-            for response_key in self.keys
-        ]
-
     @property
     def expected_input_files(self) -> list[str]:
         base = self.input_files[0]

ert/config/surface_config.py

@@ -10,11 +10,6 @@ import xarray as xr
 from pydantic import field_serializer
 from surfio import IrapHeader, IrapSurface
 
-from ert.field_utils import (
-    calc_rho_for_2d_grid_layer,
-    transform_local_ellipse_angle_to_local_coords,
-    transform_positions_to_local_field_coordinates,
-)
 from ert.substitutions import substitute_runpath_name
 
 from ._str_to_bool import str_to_bool
@@ -239,55 +234,3 @@ class SurfaceConfig(ParameterConfig):
         this flattening process"""
 
         return create_flattened_cube_graph(px=self.ncol, py=self.nrow, pz=1)
-
-    def calc_rho_for_2d_grid_layer(
-        self,
-        obs_xpos: npt.NDArray[np.float64],
-        obs_ypos: npt.NDArray[np.float64],
-        obs_main_range: npt.NDArray[np.float64],
-        obs_perp_range: npt.NDArray[np.float64],
-        obs_anisotropy_angle: npt.NDArray[np.float64],
-    ) -> npt.NDArray[np.float64]:
-        """Function to calculate scaling values to be used in the RHO matrix
-        for distance-based localization.
-
-        Args:
-            obs_xpos: x-coordinates in global coordinates of observations
-            obs_ypos: y-coordinates in global coordinates of observations
-            obs_main_range: Size of influence ellipse main principal direction.
-            obs_perp_range: Size of influence ellipse second principal direction.
-            obs_anisotropy_angle: Rotation angle anticlock wise of main principal
-              direction of influence ellipse relative to global coordinate
-              system's x-axis.
-
-        Returns:
-            Scaling values (elements of the RHO matrix) as a numpy array
-              of shape=(nx,ny,nobservations)
-
-        """
-        # Transform observation positions to local surface coordinates
-        xpos, ypos = transform_positions_to_local_field_coordinates(
-            (self.xori, self.yori), self.rotation, obs_xpos, obs_ypos
-        )
-        # Transform ellipse orientation to local surface coordinates
-        rotation_angle_of_localization_ellipse = (
-            transform_local_ellipse_angle_to_local_coords(
-                self.rotation, obs_anisotropy_angle
-            )
-        )
-
-        # Assume the coordinate system is not flipped.
-        # This means the right_handed_grid_indexing is False
-        assert self.yflip == 1
-        return calc_rho_for_2d_grid_layer(
-            self.ncol,
-            self.nrow,
-            self.xinc,
-            self.yinc,
-            xpos,
-            ypos,
-            obs_main_range,
-            obs_perp_range,
-            rotation_angle_of_localization_ellipse,
-            right_handed_grid_indexing=False,
-        )

ert/dark_storage/compute/misfits.py

@@ -1,42 +0,0 @@
-from collections.abc import Mapping
-
-import numpy as np
-import numpy.typing as npt
-import pandas as pd
-
-
-def _calculate_signed_chi_squared_misfit(
-    obs_value: npt.NDArray[np.float64],
-    response_value: npt.NDArray[np.float64],
-    obs_std: npt.NDArray[np.float64],
-) -> list[float]:
-    """The signed version is intended for visualization. For data assimiliation one
-    would normally use the normal chi-square"""
-    residual = response_value - obs_value
-    return (np.sign(residual) * residual * residual / (obs_std * obs_std)).tolist()
-
-
-def calculate_signed_chi_squared_misfits(
-    reponses_dict: Mapping[int, pd.DataFrame],
-    observation: pd.DataFrame,
-    summary_misfits: bool = False,
-) -> pd.DataFrame:
-    """
-    Compute misfits from reponses_dict (real_id, values in dataframe)
-    and observation
-    """
-    misfits_dict = {}
-    for realization_index in reponses_dict:
-        misfits_dict[realization_index] = _calculate_signed_chi_squared_misfit(
-            observation["values"],
-            reponses_dict[realization_index]
-            .loc[:, observation.index]
-            .to_numpy()
-            .flatten(),
-            observation["errors"],
-        )
-
-    df = pd.DataFrame(data=misfits_dict, index=observation.index)
-    if summary_misfits:
-        df = pd.DataFrame([df.abs().sum(axis=0)], columns=df.columns, index=[0])
-    return df.T

ert/dark_storage/endpoints/__init__.py

@@ -1,6 +1,5 @@
 from fastapi import APIRouter
 
-from .compute.misfits import router as misfits_router
 from .ensembles import router as ensembles_router
 from .experiment_server import router as experiment_server_router
 from .experiments import router as experiments_router
@@ -15,7 +14,6 @@ router.include_router(experiments_router)
 router.include_router(ensembles_router)
 router.include_router(observations_router)
 router.include_router(updates_router)
-router.include_router(misfits_router)
 router.include_router(parameters_router)
 router.include_router(responses_router)
 router.include_router(experiment_server_router)

ert/dark_storage/endpoints/experiments.py

@@ -6,7 +6,6 @@ from fastapi import APIRouter, Body, Depends, HTTPException
 from ert.config import SurfaceConfig
 from ert.dark_storage import json_schema as js
 from ert.dark_storage.common import get_storage
-from ert.shared.storage.extraction import create_priors
 from ert.storage import Storage
 
 router = APIRouter(tags=["experiment"])
@@ -26,7 +25,6 @@ def get_experiments(
             id=experiment.id,
             name=experiment.name,
             ensemble_ids=[ens.id for ens in experiment.ensembles],
-            priors=create_priors(experiment),
             userdata={},
             parameters={
                 group: config.model_dump()
@@ -34,7 +32,7 @@ def get_experiments(
                 if not isinstance(config, SurfaceConfig)
             },
             responses={
-                response_type: [m.model_dump() for m in config.metadata]
+                response_type: config.model_dump()
                 for response_type, config in experiment.response_configuration.items()
             },
             observations=experiment.response_key_to_observation_key,
@@ -62,14 +60,13 @@ def get_experiment_by_id(
         name=experiment.name,
         id=experiment.id,
         ensemble_ids=[ens.id for ens in experiment.ensembles],
-        priors=create_priors(experiment),
         userdata={},
         parameters={
             group: config.model_dump()
             for group, config in experiment.parameter_configuration.items()
         },
         responses={
-            response_type: [m.model_dump() for m in config.metadata]
+            response_type: config.model_dump()
             for response_type, config in experiment.response_configuration.items()
         },
         observations=experiment.response_key_to_observation_key,

ert/dark_storage/json_schema/experiment.py

@@ -22,8 +22,7 @@ class ExperimentIn(_Experiment):
 class ExperimentOut(_Experiment):
     id: UUID
     ensemble_ids: list[UUID]
-    priors: Mapping[str, dict[str, Any]]
     userdata: Mapping[str, Any]
     parameters: Mapping[str, dict[str, Any]]
-    responses: Mapping[str, list[dict[str, Any]]]
+    responses: Mapping[str, dict[str, Any]]
     observations: Mapping[str, dict[str, list[str]]]

ert/field_utils/__init__.py

@@ -4,7 +4,6 @@ from .field_file_format import FieldFileFormat
 from .field_utils import (
     ErtboxParameters,
     Shape,
-    calc_rho_for_2d_grid_layer,
     calculate_ertbox_parameters,
     get_shape,
     localization_scaling_function,
@@ -19,7 +18,6 @@ __all__ = [
     "ErtboxParameters",
     "FieldFileFormat",
     "Shape",
-    "calc_rho_for_2d_grid_layer",
     "calculate_ertbox_parameters",
     "get_shape",
     "localization_scaling_function",