ert 18.0.8__py3-none-any.whl → 19.0.0__py3-none-any.whl

ert/config/parsing/observations_parser.py

@@ -18,6 +18,7 @@ class ObservationType(StrEnum):
     HISTORY = "HISTORY_OBSERVATION"
     SUMMARY = "SUMMARY_OBSERVATION"
     GENERAL = "GENERAL_OBSERVATION"
+    RFT = "RFT_OBSERVATION"
 
 
 ObservationDict = dict[str, Any]
@@ -92,7 +93,7 @@ def parse_observations(content: str, filename: str) -> list[ObservationDict]:
             ), ["TYPE"]:
                 message = (
                     f"Unknown observation type '{unexpected_token}', "
-                    f"expected either 'GENERAL_OBSERVATION', "
+                    f"expected either 'RFT_OBSERVATION', 'GENERAL_OBSERVATION', "
                     f"'SUMMARY_OBSERVATION' or 'HISTORY_OBSERVATION'."
                 )
             case UnexpectedToken(token=unexpected_char, expected=allowed_chars), _:
@@ -122,7 +123,10 @@ observations_parser = Lark(
     r"""
     start: observation*
     ?observation: type OBSERVATION_NAME object? ";"
-    TYPE: "HISTORY_OBSERVATION" | "SUMMARY_OBSERVATION" | "GENERAL_OBSERVATION"
+    TYPE: "HISTORY_OBSERVATION"
+      | "SUMMARY_OBSERVATION"
+      | "GENERAL_OBSERVATION"
+      | "RFT_OBSERVATION"
     type: TYPE
     ?value: object
           | STRING
@@ -134,6 +138,7 @@ observations_parser = Lark(
     PARAMETER_NAME : CHAR+
     object : "{" [(declaration";")*] "}"
     ?declaration: "SEGMENT" STRING object -> segment
+                | "LOCALIZATION" object -> localization
                 | pair
     pair   : PARAMETER_NAME "=" value
 
@@ -189,6 +194,11 @@ class TreeToObservations(Transformer[FileContextToken, list[ObservationDict]]):
     def segment(tree):
         return (("SEGMENT", tree[0]), tree[1])
 
+    @staticmethod
+    @no_type_check
+    def localization(tree):
+        return ("LOCALIZATION", tree[0])
+
     @staticmethod
     @no_type_check
     def object(tree):

ert/config/parsing/queue_system.py

@@ -23,3 +23,12 @@ class QueueSystem(StrEnum):
     @staticmethod
     def ert_config_case() -> str:
         return "upper"
+
+    @property
+    def formatted_name(self) -> str:
+        return {
+            self.LSF: "LSF",
+            self.LOCAL: "Local",
+            self.TORQUE: "Torque/OpenPBS",
+            self.SLURM: "Slurm",
+        }[self]

ert/config/queue_config.py

@@ -51,7 +51,6 @@ class QueueOptions(
     BaseModelWithContextSupport,
     validate_assignment=True,
     extra="forbid",
-    use_enum_values=True,
     validate_default=True,
 ):
     name: QueueSystem

ert/config/response_config.py

@@ -31,7 +31,6 @@ class ResponseMetadata(BaseModel):
 
 class ResponseConfig(BaseModel):
     type: str
-    name: str
     input_files: list[str] = Field(default_factory=list)
     keys: list[str] = Field(default_factory=list)
     has_finalized_keys: bool = False

ert/config/rft_config.py

@@ -3,15 +3,16 @@ from __future__ import annotations
 import datetime
 import fnmatch
 import logging
+import os
 import re
 from collections import defaultdict
-from typing import Literal
+from typing import IO, Any, Literal
 
 import numpy as np
 import numpy.typing as npt
 import polars as pl
 from pydantic import Field
-from resfo_utilities import InvalidRFTError, RFTReader
+from resfo_utilities import CornerpointGrid, InvalidRFTError, RFTReader
 
 from ert.substitutions import substitute_runpath_name
 
@@ -27,6 +28,7 @@ class RFTConfig(ResponseConfig):
     name: str = "rft"
     has_finalized_keys: bool = False
     data_to_read: dict[str, dict[str, list[str]]] = Field(default_factory=dict)
+    locations: list[tuple[float, float, float]] = Field(default_factory=list)
 
     @property
     def metadata(self) -> list[ResponseMetadata]:
@@ -51,6 +53,20 @@ class RFTConfig(ResponseConfig):
 
         return [f"{base}.RFT"]
 
+    def _find_indices(
+        self, egrid_file: str | os.PathLike[str] | IO[Any]
+    ) -> dict[tuple[int, int, int] | None, set[tuple[float, float, float]]]:
+        indices = defaultdict(set)
+        for a, b in zip(
+            CornerpointGrid.read_egrid(egrid_file).find_cell_containing_point(
+                self.locations
+            ),
+            self.locations,
+            strict=True,
+        ):
+            indices[a].add(b)
+        return indices
+
     def read_from_file(self, run_path: str, iens: int, iter_: int) -> pl.DataFrame:
         filename = substitute_runpath_name(self.input_files[0], iens, iter_)
         if filename.upper().endswith(".DATA"):
@@ -58,9 +74,20 @@ class RFTConfig(ResponseConfig):
             # allowed to give REFCASE and ECLBASE both
             # with and without .DATA extensions
             filename = filename[:-5]
+        grid_filename = f"{run_path}/{filename}"
+        if grid_filename.upper().endswith(".RFT"):
+            grid_filename = grid_filename[:-4]
+        grid_filename += ".EGRID"
         fetched: dict[tuple[str, datetime.date], dict[str, npt.NDArray[np.float32]]] = (
             defaultdict(dict)
         )
+        indices = {}
+        if self.locations:
+            indices = self._find_indices(grid_filename)
+        if None in indices:
+            raise InvalidResponseFile(
+                f"Did not find grid coordinate for location(s) {indices[None]}"
+            )
         # This is a somewhat complicated optimization in order to
         # support wildcards in well names, dates and properties
         # A python for loop is too slow so we use a compiled regex
@@ -72,6 +99,7 @@ class RFTConfig(ResponseConfig):
                     "time": [],
                     "depth": [],
                     "values": [],
+                    "location": [],
                 }
             )
 
@@ -104,6 +132,7 @@ class RFTConfig(ResponseConfig):
                 for time, props in inner_dict.items()
             )
         )
+        locations = {}
         try:
             with RFTReader.open(f"{run_path}/{filename}") as rft:
                 for entry in rft:
@@ -113,6 +142,15 @@ class RFTConfig(ResponseConfig):
                         key = f"{well}{sep}{date}{sep}{rft_property}"
                         if matcher.fullmatch(key) is not None:
                             values = entry[rft_property]
+                            locations[well, date] = [
+                                list(
+                                    indices.get(
+                                        (c[0] - 1, c[1] - 1, c[2] - 1),
+                                        [(None, None, None)],
+                                    )
+                                )
+                                for c in entry.connections
+                            ]
                             if np.isdtype(values.dtype, np.float32):
                                 fetched[well, date][rft_property] = values
         except (FileNotFoundError, InvalidRFTError) as err:
@@ -127,41 +165,48 @@ class RFTConfig(ResponseConfig):
                     "time": [],
                     "depth": [],
                     "values": [],
+                    "location": [],
                 }
             )
 
-        dfs = []
-
-        for (well, time), inner_dict in fetched.items():
-            wide = pl.DataFrame(
-                {k: pl.Series(v.astype("<f4")) for k, v in inner_dict.items()}
-            )
-
-            if wide.columns == ["DEPTH"]:
-                continue
-
-            if "DEPTH" not in wide.columns:
-                raise InvalidResponseFile(f"Could not find DEPTH in RFTFile {filename}")
-
-            # Unpivot all columns except DEPTH
-            long = wide.unpivot(
-                index="DEPTH",  # keep depth as column
-                # turn other prop values into response_key col
-                variable_name="response_key",
-                value_name="values",  # put values in own column
-            ).rename({"DEPTH": "depth"})
-
-            # Add wellname prefix to response_keys
-            long = long.with_columns(
-                (pl.lit(f"{well}:{time.isoformat()}:") + pl.col("response_key")).alias(
-                    "response_key"
-                ),
-                pl.lit(time).alias("time"),
+        try:
+            df = pl.concat(
+                [
+                    pl.DataFrame(
+                        {
+                            "response_key": [f"{well}:{time.isoformat()}:{prop}"],
+                            "time": [time],
+                            "depth": [fetched[well, time]["DEPTH"]],
+                            "values": [vals],
+                            "location": pl.Series(
+                                [
+                                    locations.get(
+                                        (well, time), [(None, None, None)] * len(vals)
+                                    )
+                                ],
+                                dtype=pl.Array(
+                                    pl.List(pl.Array(pl.Float32, 3)), len(vals)
+                                ),
+                            ),
+                        }
+                    )
+                    .explode("depth", "values", "location")
+                    .explode("location")
+                    for (well, time), inner_dict in fetched.items()
+                    for prop, vals in inner_dict.items()
+                    if prop != "DEPTH"
+                ]
             )
+        except KeyError as err:
+            raise InvalidResponseFile(
+                f"Could not find {err.args[0]} in RFTFile {filename}"
+            ) from err
 
-            dfs.append(long.select("response_key", "time", "depth", "values"))
-
-        return pl.concat(dfs)
+        return df.with_columns(
+            east=pl.col("location").arr.get(0),
+            north=pl.col("location").arr.get(1),
+            tvd=pl.col("location").arr.get(2),
+        ).drop("location")
 
     @property
     def response_type(self) -> str:
@@ -169,7 +214,7 @@ class RFTConfig(ResponseConfig):
 
     @property
     def primary_key(self) -> list[str]:
-        return []
+        return ["east", "north", "tvd"]
 
     @classmethod
     def from_config_dict(cls, config_dict: ConfigDict) -> RFTConfig | None:

ert/config/summary_config.py

@@ -19,14 +19,13 @@ logger = logging.getLogger(__name__)
 
 class SummaryConfig(ResponseConfig):
     type: Literal["summary"] = "summary"
-    name: str = "summary"
     has_finalized_keys: bool = False
 
     @property
     def metadata(self) -> list[ResponseMetadata]:
         return [
             ResponseMetadata(
-                response_type=self.name,
+                response_type=self.type,
                 response_key=response_key,
                 filter_on=None,
                 finalized=self.has_finalized_keys,

ert/config/surface_config.py

@@ -10,11 +10,16 @@ import xarray as xr
 from pydantic import field_serializer
 from surfio import IrapHeader, IrapSurface
 
+from ert.field_utils import (
+    calc_rho_for_2d_grid_layer,
+    transform_local_ellipse_angle_to_local_coords,
+    transform_positions_to_local_field_coordinates,
+)
 from ert.substitutions import substitute_runpath_name
 
 from ._str_to_bool import str_to_bool
 from .field import create_flattened_cube_graph
-from .parameter_config import InvalidParameterFile, ParameterConfig, ParameterMetadata
+from .parameter_config import InvalidParameterFile, ParameterConfig
 from .parsing import ConfigValidationError, ErrorInfo
 
 if TYPE_CHECKING:
@@ -46,6 +51,7 @@ class SurfaceMismatchError(InvalidParameterFile):
 
 class SurfaceConfig(ParameterConfig):
     type: Literal["surface"] = "surface"
+    dimensionality: Literal[2] = 2
     ncol: int
     nrow: int
     xori: float
@@ -70,21 +76,6 @@ class SurfaceConfig(ParameterConfig):
     def parameter_keys(self) -> list[str]:
         return []
 
-    @property
-    def metadata(self) -> list[ParameterMetadata]:
-        return [
-            ParameterMetadata(
-                key=self.name,
-                dimensionality=2,
-                transformation=None,
-                userdata={
-                    "data_origin": "SURFACE",
-                    "nx": self.ncol,
-                    "ny": self.nrow,
-                },
-            )
-        ]
-
     @classmethod
     def from_config_list(cls, config_list: list[str | dict[str, str]]) -> Self:
         name = cast(str, config_list[0])
@@ -248,3 +239,55 @@ class SurfaceConfig(ParameterConfig):
         this flattening process"""
 
         return create_flattened_cube_graph(px=self.ncol, py=self.nrow, pz=1)
+
+    def calc_rho_for_2d_grid_layer(
+        self,
+        obs_xpos: npt.NDArray[np.float64],
+        obs_ypos: npt.NDArray[np.float64],
+        obs_main_range: npt.NDArray[np.float64],
+        obs_perp_range: npt.NDArray[np.float64],
+        obs_anisotropy_angle: npt.NDArray[np.float64],
+    ) -> npt.NDArray[np.float64]:
+        """Function to calculate scaling values to be used in the RHO matrix
+        for distance-based localization.
+
+        Args:
+            obs_xpos: x-coordinates in global coordinates of observations
+            obs_ypos: y-coordinates in global coordinates of observations
+            obs_main_range: Size of influence ellipse main principal direction.
+            obs_perp_range: Size of influence ellipse second principal direction.
+            obs_anisotropy_angle: Rotation angle anticlock wise of main principal
+              direction of influence ellipse relative to global coordinate
+              system's x-axis.
+
+        Returns:
+            Scaling values (elements of the RHO matrix) as a numpy array
+              of shape=(nx,ny,nobservations)
+
+        """
+        # Transform observation positions to local surface coordinates
+        xpos, ypos = transform_positions_to_local_field_coordinates(
+            (self.xori, self.yori), self.rotation, obs_xpos, obs_ypos
+        )
+        # Transform ellipse orientation to local surface coordinates
+        rotation_angle_of_localization_ellipse = (
+            transform_local_ellipse_angle_to_local_coords(
+                self.rotation, obs_anisotropy_angle
+            )
+        )
+
+        # Assume the coordinate system is not flipped.
+        # This means the right_handed_grid_indexing is False
+        assert self.yflip == 1
+        return calc_rho_for_2d_grid_layer(
+            self.ncol,
+            self.nrow,
+            self.xinc,
+            self.yinc,
+            xpos,
+            ypos,
+            obs_main_range,
+            obs_perp_range,
+            rotation_angle_of_localization_ellipse,
+            right_handed_grid_indexing=False,
+        )

ert/dark_storage/common.py

@@ -28,7 +28,7 @@ def get_storage() -> Storage:
         except ErtStorageException as err:
             logger.exception(f"Error accessing storage: {err!s}")
             raise InternalServerError("Error accessing storage") from None
-    _storage.refresh()
+    _storage.reload()
     return _storage
 
 

ert/dark_storage/compute/misfits.py

@@ -29,7 +29,10 @@ def calculate_signed_chi_squared_misfits(
     for realization_index in reponses_dict:
         misfits_dict[realization_index] = _calculate_signed_chi_squared_misfit(
             observation["values"],
-            reponses_dict[realization_index].loc[:, observation.index].values.flatten(),
+            reponses_dict[realization_index]
+            .loc[:, observation.index]
+            .to_numpy()
+            .flatten(),
             observation["errors"],
         )
 

ert/dark_storage/endpoints/compute/misfits.py

@@ -1,6 +1,6 @@
 import json
 from datetime import datetime
-from typing import Any
+from typing import Annotated, Any
 from uuid import UUID
 
 import pandas as pd
@@ -36,7 +36,9 @@ async def get_response_misfits(
     response_name: str,
     realization_index: int | None = None,
     summary_misfits: bool = False,
-    filter_on: str | None = Query(None, description="JSON string with filters"),
+    filter_on: Annotated[
+        str | None, Query(description="JSON string with filters")
+    ] = None,
 ) -> Response:
     ensemble = storage.get_ensemble(ensemble_id)
     dataframe = data_for_response(

ert/dark_storage/endpoints/experiment_server.py

@@ -9,6 +9,7 @@ import traceback
 import uuid
 from base64 import b64decode
 from queue import SimpleQueue
+from typing import Annotated
 
 from fastapi import (
     APIRouter,
@@ -153,7 +154,7 @@ def _log(request: Request) -> None:
 
 @router.get("/")
 def get_status(
-    request: Request, credentials: HTTPBasicCredentials = Depends(security)
+    request: Request, credentials: Annotated[HTTPBasicCredentials, Depends(security)]
 ) -> PlainTextResponse:
     _log(request)
     _check_user(credentials)
@@ -162,7 +163,7 @@ def get_status(
 
 @router.get("/status")
 def experiment_status(
-    request: Request, credentials: HTTPBasicCredentials = Depends(security)
+    request: Request, credentials: Annotated[HTTPBasicCredentials, Depends(security)]
 ) -> ExperimentStatus:
     _log(request)
     _check_user(credentials)
@@ -171,7 +172,7 @@ def experiment_status(
 
 @router.post("/" + EverEndpoints.stop)
 def stop(
-    request: Request, credentials: HTTPBasicCredentials = Depends(security)
+    request: Request, credentials: Annotated[HTTPBasicCredentials, Depends(security)]
 ) -> Response:
     _log(request)
     _check_user(credentials)
@@ -185,7 +186,7 @@ def stop(
 async def start_experiment(
     request: Request,
     background_tasks: BackgroundTasks,
-    credentials: HTTPBasicCredentials = Depends(security),
+    credentials: Annotated[HTTPBasicCredentials, Depends(security)],
 ) -> Response:
     _log(request)
     _check_user(credentials)
@@ -218,7 +219,7 @@ async def start_experiment(
 
 @router.get("/" + EverEndpoints.config_path)
 async def config_path(
-    request: Request, credentials: HTTPBasicCredentials = Depends(security)
+    request: Request, credentials: Annotated[HTTPBasicCredentials, Depends(security)]
 ) -> JSONResponse:
     _log(request)
     _check_user(credentials)
@@ -237,7 +238,7 @@ async def config_path(
 
 @router.get("/" + EverEndpoints.start_time)
 async def start_time(
-    request: Request, credentials: HTTPBasicCredentials = Depends(security)
+    request: Request, credentials: Annotated[HTTPBasicCredentials, Depends(security)]
 ) -> Response:
     _log(request)
     _check_user(credentials)
@@ -316,9 +317,11 @@ class ExperimentRunner:
             simulation_future = loop.run_in_executor(
                 None,
                 lambda: run_model.start_simulations_thread(
-                    EvaluatorServerConfig()
-                    if run_model.queue_config.queue_system == QueueSystem.LOCAL
-                    else EvaluatorServerConfig(use_ipc_protocol=False)
+                    EvaluatorServerConfig(
+                        use_ipc_protocol=run_model.queue_config.queue_system
+                        == QueueSystem.LOCAL,
+                        prioritize_private_ip_address=site_plugins.prioritize_private_ip_address,
+                    )
                 ),
             )
             while True:

ert/dark_storage/endpoints/experiments.py

@@ -29,7 +29,7 @@ def get_experiments(
             priors=create_priors(experiment),
             userdata={},
             parameters={
-                group: [m.model_dump() for m in config.metadata]
+                group: config.model_dump()
                 for group, config in experiment.parameter_configuration.items()
                 if not isinstance(config, SurfaceConfig)
             },
@@ -65,7 +65,7 @@ def get_experiment_by_id(
         priors=create_priors(experiment),
         userdata={},
         parameters={
-            group: [m.model_dump() for m in config.metadata]
+            group: config.model_dump()
             for group, config in experiment.parameter_configuration.items()
         },
         responses={

ert/dark_storage/endpoints/observations.py

@@ -1,7 +1,7 @@
 import json
 import logging
 import operator
-from typing import Any
+from typing import Annotated, Any
 from urllib.parse import unquote
 from uuid import UUID, uuid4
 
@@ -57,7 +57,9 @@ async def get_observations_for_response(
     storage: Storage = DEFAULT_STORAGE,
     ensemble_id: UUID,
     response_key: str,
-    filter_on: str | None = Query(None, description="JSON string with filters"),
+    filter_on: Annotated[
+        str | None, Query(description="JSON string with filters")
+    ] = None,
 ) -> list[js.ObservationOut]:
     response_key = unquote(response_key)
     try:
@@ -82,6 +84,7 @@ async def get_observations_for_response(
         ensemble.experiment,
         obs_keys,
         json.loads(filter_on) if filter_on is not None else None,
+        requested_response_type=response_type,
     )
     if not obss:
         return []
@@ -105,10 +108,17 @@ def _get_observations(
     experiment: Experiment,
     observation_keys: list[str] | None = None,
     filter_on: dict[str, Any] | None = None,
+    requested_response_type: str | None = None,
 ) -> list[dict[str, Any]]:
     observations = []
 
-    for response_type, df in experiment.observations.items():
+    for stored_response_type, df in experiment.observations.items():
+        if (
+            requested_response_type is not None
+            and stored_response_type != requested_response_type
+        ):
+            continue
+
         if observation_keys is not None:
             df = df.filter(pl.col("observation_key").is_in(observation_keys))
 
@@ -125,7 +135,7 @@ def _get_observations(
         if df.is_empty():
             continue
 
-        x_axis_fn = response_to_pandas_x_axis_fns[response_type]
+        x_axis_fn = response_to_pandas_x_axis_fns[stored_response_type]
         df = df.rename(
             {
                 "observation_key": "name",

ert/dark_storage/endpoints/parameters.py

@@ -113,27 +113,11 @@ def get_parameter_std_dev(
     return Response(content=buffer.getvalue(), media_type="application/octet-stream")
 
 
-def _extract_parameter_group_and_key(key: str) -> tuple[str, str] | tuple[None, None]:
-    key = key.removeprefix("LOG10_")
-    if ":" not in key:
-        # Assume all incoming keys are in format group:key for now
-        return None, None
-
-    param_group, param_key = key.split(":", maxsplit=1)
-    return param_group, param_key
-
-
 def data_for_parameter(ensemble: Ensemble, key: str) -> pd.DataFrame:
-    group, _ = _extract_parameter_group_and_key(key)
-    if group is None:
-        logger.warning(
-            f"Parameter with key '{key}' does not have a group, "
-            "fetching data for all parameters"
-        )
     try:
-        df = ensemble.load_scalars(group)
+        df = ensemble.load_scalar_keys([key], transformed=True)
         if df.is_empty():
-            logger.warning(f"No data found for parameter '{group}:{key}'")
+            logger.warning(f"No data found for parameter '{key}'")
             return pd.DataFrame()
     except KeyError as e:
         logger.error(e)

ert/dark_storage/endpoints/responses.py

@@ -45,7 +45,9 @@ async def get_response(
     storage: Storage = DEFAULT_STORAGE,
     ensemble_id: UUID,
     response_key: str,
-    filter_on: str | None = Query(None, description="JSON string with filters"),
+    filter_on: Annotated[
+        str | None, Query(description="JSON string with filters")
+    ] = None,
     accept: Annotated[str | None, Header()] = None,
 ) -> Response:
     try:
@@ -96,6 +98,7 @@ async def get_response(
 response_to_pandas_x_axis_fns: dict[str, Callable[[tuple[Any, ...]], Any]] = {
     "summary": lambda t: pd.Timestamp(t[2]).isoformat(),
     "gen_data": lambda t: str(t[3]),
+    "rft": lambda t: str(t[4]),
 }
 
 
@@ -147,8 +150,9 @@ def data_for_response(
         # This performs the same aggragation by mean of duplicate values
         # as in ert/analysis/_es_update.py
         df = df.groupby(["Date", "Realization"]).mean()
-        data = df.unstack(level="Date")
-        data.columns = data.columns.droplevel(0)
+        data = df.reset_index().pivot_table(
+            index="Realization", columns="Date", values=df.columns[0]
+        )
         return data.astype(float)
 
     if response_type == "rft":
@@ -159,8 +163,9 @@ def data_for_response(
             )
             .rename({"realization": "Realization"})
             .select(["Realization", "depth", "values"])
+            .unique()
             .to_pandas()
-            .pivot(index="Realization", columns="depth", values="values")
+            .pivot_table(index="Realization", columns="depth", values="values")
             .reset_index(drop=True)
         )
 
@@ -172,7 +177,7 @@ def data_for_response(
             assert "report_step" in filter_on
             report_step = int(filter_on["report_step"])
             vals = data.filter(pl.col("report_step").eq(report_step))
-            pivoted = vals.drop("response_key", "report_step").pivot(
+            pivoted = vals.drop("response_key", "report_step").pivot(  # noqa: PD010
                 on="index", values="values"
             )
             data = pivoted.to_pandas().set_index("realization")

ert/dark_storage/json_schema/experiment.py

@@ -24,6 +24,6 @@ class ExperimentOut(_Experiment):
     ensemble_ids: list[UUID]
     priors: Mapping[str, dict[str, Any]]
     userdata: Mapping[str, Any]
-    parameters: Mapping[str, list[dict[str, Any]]]
+    parameters: Mapping[str, dict[str, Any]]
     responses: Mapping[str, list[dict[str, Any]]]
     observations: Mapping[str, dict[str, list[str]]]