ert 18.0.9__py3-none-any.whl → 19.0.0rc0__py3-none-any.whl

ert/config/rft_config.py

@@ -3,15 +3,16 @@ from __future__ import annotations
 import datetime
 import fnmatch
 import logging
+import os
 import re
 from collections import defaultdict
-from typing import Literal
+from typing import IO, Any, Literal
 
 import numpy as np
 import numpy.typing as npt
 import polars as pl
 from pydantic import Field
-from resfo_utilities import InvalidRFTError, RFTReader
+from resfo_utilities import CornerpointGrid, InvalidRFTError, RFTReader
 
 from ert.substitutions import substitute_runpath_name
 
@@ -27,6 +28,7 @@ class RFTConfig(ResponseConfig):
     name: str = "rft"
     has_finalized_keys: bool = False
     data_to_read: dict[str, dict[str, list[str]]] = Field(default_factory=dict)
+    locations: list[tuple[float, float, float]] = Field(default_factory=list)
 
     @property
     def metadata(self) -> list[ResponseMetadata]:
@@ -51,6 +53,20 @@ class RFTConfig(ResponseConfig):
 
         return [f"{base}.RFT"]
 
+    def _find_indices(
+        self, egrid_file: str | os.PathLike[str] | IO[Any]
+    ) -> dict[tuple[int, int, int] | None, set[tuple[float, float, float]]]:
+        indices = defaultdict(set)
+        for a, b in zip(
+            CornerpointGrid.read_egrid(egrid_file).find_cell_containing_point(
+                self.locations
+            ),
+            self.locations,
+            strict=True,
+        ):
+            indices[a].add(b)
+        return indices
+
     def read_from_file(self, run_path: str, iens: int, iter_: int) -> pl.DataFrame:
         filename = substitute_runpath_name(self.input_files[0], iens, iter_)
         if filename.upper().endswith(".DATA"):
@@ -58,9 +74,20 @@ class RFTConfig(ResponseConfig):
             # allowed to give REFCASE and ECLBASE both
             # with and without .DATA extensions
             filename = filename[:-5]
+        grid_filename = f"{run_path}/{filename}"
+        if grid_filename.upper().endswith(".RFT"):
+            grid_filename = grid_filename[:-4]
+        grid_filename += ".EGRID"
         fetched: dict[tuple[str, datetime.date], dict[str, npt.NDArray[np.float32]]] = (
             defaultdict(dict)
         )
+        indices = {}
+        if self.locations:
+            indices = self._find_indices(grid_filename)
+        if None in indices:
+            raise InvalidResponseFile(
+                f"Did not find grid coordinate for location(s) {indices[None]}"
+            )
         # This is a somewhat complicated optimization in order to
         # support wildcards in well names, dates and properties
         # A python for loop is too slow so we use a compiled regex
@@ -72,6 +99,7 @@ class RFTConfig(ResponseConfig):
                     "time": [],
                     "depth": [],
                     "values": [],
+                    "location": [],
                 }
             )
 
@@ -104,6 +132,7 @@ class RFTConfig(ResponseConfig):
                 for time, props in inner_dict.items()
             )
         )
+        locations = {}
         try:
             with RFTReader.open(f"{run_path}/{filename}") as rft:
                 for entry in rft:
@@ -113,6 +142,15 @@ class RFTConfig(ResponseConfig):
                         key = f"{well}{sep}{date}{sep}{rft_property}"
                         if matcher.fullmatch(key) is not None:
                             values = entry[rft_property]
+                            locations[well, date] = [
+                                list(
+                                    indices.get(
+                                        (c[0] - 1, c[1] - 1, c[2] - 1),
+                                        [(None, None, None)],
+                                    )
+                                )
+                                for c in entry.connections
+                            ]
                             if np.isdtype(values.dtype, np.float32):
                                 fetched[well, date][rft_property] = values
         except (FileNotFoundError, InvalidRFTError) as err:
@@ -127,41 +165,48 @@ class RFTConfig(ResponseConfig):
                     "time": [],
                     "depth": [],
                     "values": [],
+                    "location": [],
                 }
             )
 
-        dfs = []
-
-        for (well, time), inner_dict in fetched.items():
-            wide = pl.DataFrame(
-                {k: pl.Series(v.astype("<f4")) for k, v in inner_dict.items()}
-            )
-
-            if wide.columns == ["DEPTH"]:
-                continue
-
-            if "DEPTH" not in wide.columns:
-                raise InvalidResponseFile(f"Could not find DEPTH in RFTFile {filename}")
-
-            # Unpivot all columns except DEPTH
-            long = wide.unpivot(
-                index="DEPTH",  # keep depth as column
-                # turn other prop values into response_key col
-                variable_name="response_key",
-                value_name="values",  # put values in own column
-            ).rename({"DEPTH": "depth"})
-
-            # Add wellname prefix to response_keys
-            long = long.with_columns(
-                (pl.lit(f"{well}:{time.isoformat()}:") + pl.col("response_key")).alias(
-                    "response_key"
-                ),
-                pl.lit(time).alias("time"),
+        try:
+            df = pl.concat(
+                [
+                    pl.DataFrame(
+                        {
+                            "response_key": [f"{well}:{time.isoformat()}:{prop}"],
+                            "time": [time],
+                            "depth": [fetched[well, time]["DEPTH"]],
+                            "values": [vals],
+                            "location": pl.Series(
+                                [
+                                    locations.get(
+                                        (well, time), [(None, None, None)] * len(vals)
+                                    )
+                                ],
+                                dtype=pl.Array(
+                                    pl.List(pl.Array(pl.Float32, 3)), len(vals)
+                                ),
+                            ),
+                        }
+                    )
+                    .explode("depth", "values", "location")
+                    .explode("location")
+                    for (well, time), inner_dict in fetched.items()
+                    for prop, vals in inner_dict.items()
+                    if prop != "DEPTH"
+                ]
             )
+        except KeyError as err:
+            raise InvalidResponseFile(
+                f"Could not find {err.args[0]} in RFTFile {filename}"
+            ) from err
 
-            dfs.append(long.select("response_key", "time", "depth", "values"))
-
-        return pl.concat(dfs)
+        return df.with_columns(
+            east=pl.col("location").arr.get(0),
+            north=pl.col("location").arr.get(1),
+            tvd=pl.col("location").arr.get(2),
+        ).drop("location")
 
     @property
     def response_type(self) -> str:
@@ -169,7 +214,7 @@ class RFTConfig(ResponseConfig):
 
     @property
     def primary_key(self) -> list[str]:
-        return []
+        return ["east", "north", "tvd"]
 
     @classmethod
     def from_config_dict(cls, config_dict: ConfigDict) -> RFTConfig | None:

ert/config/summary_config.py

@@ -19,14 +19,13 @@ logger = logging.getLogger(__name__)
 
 class SummaryConfig(ResponseConfig):
     type: Literal["summary"] = "summary"
-    name: str = "summary"
     has_finalized_keys: bool = False
 
     @property
     def metadata(self) -> list[ResponseMetadata]:
         return [
             ResponseMetadata(
-                response_type=self.name,
+                response_type=self.type,
                 response_key=response_key,
                 filter_on=None,
                 finalized=self.has_finalized_keys,

ert/config/surface_config.py

@@ -10,11 +10,16 @@ import xarray as xr
 from pydantic import field_serializer
 from surfio import IrapHeader, IrapSurface
 
+from ert.field_utils import (
+    calc_rho_for_2d_grid_layer,
+    transform_local_ellipse_angle_to_local_coords,
+    transform_positions_to_local_field_coordinates,
+)
 from ert.substitutions import substitute_runpath_name
 
 from ._str_to_bool import str_to_bool
 from .field import create_flattened_cube_graph
-from .parameter_config import InvalidParameterFile, ParameterConfig, ParameterMetadata
+from .parameter_config import InvalidParameterFile, ParameterConfig
 from .parsing import ConfigValidationError, ErrorInfo
 
 if TYPE_CHECKING:
@@ -46,6 +51,7 @@ class SurfaceMismatchError(InvalidParameterFile):
 
 class SurfaceConfig(ParameterConfig):
     type: Literal["surface"] = "surface"
+    dimensionality: Literal[2] = 2
     ncol: int
     nrow: int
     xori: float
@@ -70,21 +76,6 @@ class SurfaceConfig(ParameterConfig):
     def parameter_keys(self) -> list[str]:
         return []
 
-    @property
-    def metadata(self) -> list[ParameterMetadata]:
-        return [
-            ParameterMetadata(
-                key=self.name,
-                dimensionality=2,
-                transformation=None,
-                userdata={
-                    "data_origin": "SURFACE",
-                    "nx": self.ncol,
-                    "ny": self.nrow,
-                },
-            )
-        ]
-
     @classmethod
     def from_config_list(cls, config_list: list[str | dict[str, str]]) -> Self:
         name = cast(str, config_list[0])
@@ -248,3 +239,55 @@ class SurfaceConfig(ParameterConfig):
         this flattening process"""
 
         return create_flattened_cube_graph(px=self.ncol, py=self.nrow, pz=1)
+
+    def calc_rho_for_2d_grid_layer(
+        self,
+        obs_xpos: npt.NDArray[np.float64],
+        obs_ypos: npt.NDArray[np.float64],
+        obs_main_range: npt.NDArray[np.float64],
+        obs_perp_range: npt.NDArray[np.float64],
+        obs_anisotropy_angle: npt.NDArray[np.float64],
+    ) -> npt.NDArray[np.float64]:
+        """Function to calculate scaling values to be used in the RHO matrix
+        for distance-based localization.
+
+        Args:
+            obs_xpos: x-coordinates in global coordinates of observations
+            obs_ypos: y-coordinates in global coordinates of observations
+            obs_main_range: Size of influence ellipse main principal direction.
+            obs_perp_range: Size of influence ellipse second principal direction.
+            obs_anisotropy_angle: Rotation angle anticlock wise of main principal
+              direction of influence ellipse relative to global coordinate
+              system's x-axis.
+
+        Returns:
+            Scaling values (elements of the RHO matrix) as a numpy array
+              of shape=(nx,ny,nobservations)
+
+        """
+        # Transform observation positions to local surface coordinates
+        xpos, ypos = transform_positions_to_local_field_coordinates(
+            (self.xori, self.yori), self.rotation, obs_xpos, obs_ypos
+        )
+        # Transform ellipse orientation to local surface coordinates
+        rotation_angle_of_localization_ellipse = (
+            transform_local_ellipse_angle_to_local_coords(
+                self.rotation, obs_anisotropy_angle
+            )
+        )
+
+        # Assume the coordinate system is not flipped.
+        # This means the right_handed_grid_indexing is False
+        assert self.yflip == 1
+        return calc_rho_for_2d_grid_layer(
+            self.ncol,
+            self.nrow,
+            self.xinc,
+            self.yinc,
+            xpos,
+            ypos,
+            obs_main_range,
+            obs_perp_range,
+            rotation_angle_of_localization_ellipse,
+            right_handed_grid_indexing=False,
+        )

ert/dark_storage/common.py

@@ -28,7 +28,7 @@ def get_storage() -> Storage:
         except ErtStorageException as err:
             logger.exception(f"Error accessing storage: {err!s}")
             raise InternalServerError("Error accessing storage") from None
-    _storage.refresh()
+    _storage.reload()
     return _storage
 
 

ert/dark_storage/compute/misfits.py

@@ -29,7 +29,10 @@ def calculate_signed_chi_squared_misfits(
     for realization_index in reponses_dict:
         misfits_dict[realization_index] = _calculate_signed_chi_squared_misfit(
             observation["values"],
-            reponses_dict[realization_index].loc[:, observation.index].values.flatten(),
+            reponses_dict[realization_index]
+            .loc[:, observation.index]
+            .to_numpy()
+            .flatten(),
             observation["errors"],
         )
 

ert/dark_storage/endpoints/compute/misfits.py

@@ -1,6 +1,6 @@
 import json
 from datetime import datetime
-from typing import Any
+from typing import Annotated, Any
 from uuid import UUID
 
 import pandas as pd
@@ -36,7 +36,9 @@ async def get_response_misfits(
     response_name: str,
     realization_index: int | None = None,
     summary_misfits: bool = False,
-    filter_on: str | None = Query(None, description="JSON string with filters"),
+    filter_on: Annotated[
+        str | None, Query(description="JSON string with filters")
+    ] = None,
 ) -> Response:
     ensemble = storage.get_ensemble(ensemble_id)
     dataframe = data_for_response(

ert/dark_storage/endpoints/experiment_server.py

@@ -9,6 +9,7 @@ import traceback
 import uuid
 from base64 import b64decode
 from queue import SimpleQueue
+from typing import Annotated
 
 from fastapi import (
     APIRouter,
@@ -153,7 +154,7 @@ def _log(request: Request) -> None:
 
 @router.get("/")
 def get_status(
-    request: Request, credentials: HTTPBasicCredentials = Depends(security)
+    request: Request, credentials: Annotated[HTTPBasicCredentials, Depends(security)]
 ) -> PlainTextResponse:
     _log(request)
     _check_user(credentials)
@@ -162,7 +163,7 @@ def get_status(
 
 @router.get("/status")
 def experiment_status(
-    request: Request, credentials: HTTPBasicCredentials = Depends(security)
+    request: Request, credentials: Annotated[HTTPBasicCredentials, Depends(security)]
 ) -> ExperimentStatus:
     _log(request)
     _check_user(credentials)
@@ -171,7 +172,7 @@ def experiment_status(
 
 @router.post("/" + EverEndpoints.stop)
 def stop(
-    request: Request, credentials: HTTPBasicCredentials = Depends(security)
+    request: Request, credentials: Annotated[HTTPBasicCredentials, Depends(security)]
 ) -> Response:
     _log(request)
     _check_user(credentials)
@@ -185,7 +186,7 @@ def stop(
 async def start_experiment(
     request: Request,
     background_tasks: BackgroundTasks,
-    credentials: HTTPBasicCredentials = Depends(security),
+    credentials: Annotated[HTTPBasicCredentials, Depends(security)],
 ) -> Response:
     _log(request)
     _check_user(credentials)
@@ -218,7 +219,7 @@ async def start_experiment(
 
 @router.get("/" + EverEndpoints.config_path)
 async def config_path(
-    request: Request, credentials: HTTPBasicCredentials = Depends(security)
+    request: Request, credentials: Annotated[HTTPBasicCredentials, Depends(security)]
 ) -> JSONResponse:
     _log(request)
     _check_user(credentials)
@@ -237,7 +238,7 @@ async def config_path(
 
 @router.get("/" + EverEndpoints.start_time)
 async def start_time(
-    request: Request, credentials: HTTPBasicCredentials = Depends(security)
+    request: Request, credentials: Annotated[HTTPBasicCredentials, Depends(security)]
 ) -> Response:
     _log(request)
     _check_user(credentials)
@@ -316,9 +317,11 @@ class ExperimentRunner:
             simulation_future = loop.run_in_executor(
                 None,
                 lambda: run_model.start_simulations_thread(
-                    EvaluatorServerConfig()
-                    if run_model.queue_config.queue_system == QueueSystem.LOCAL
-                    else EvaluatorServerConfig(use_ipc_protocol=False)
+                    EvaluatorServerConfig(
+                        use_ipc_protocol=run_model.queue_config.queue_system
+                        == QueueSystem.LOCAL,
+                        prioritize_private_ip_address=site_plugins.prioritize_private_ip_address,
+                    )
                 ),
             )
             while True:

ert/dark_storage/endpoints/experiments.py

@@ -29,7 +29,7 @@ def get_experiments(
             priors=create_priors(experiment),
             userdata={},
             parameters={
-                group: [m.model_dump() for m in config.metadata]
+                group: config.model_dump()
                 for group, config in experiment.parameter_configuration.items()
                 if not isinstance(config, SurfaceConfig)
             },
@@ -65,7 +65,7 @@ def get_experiment_by_id(
         priors=create_priors(experiment),
         userdata={},
         parameters={
-            group: [m.model_dump() for m in config.metadata]
+            group: config.model_dump()
             for group, config in experiment.parameter_configuration.items()
         },
         responses={

ert/dark_storage/endpoints/observations.py

@@ -1,7 +1,7 @@
 import json
 import logging
 import operator
-from typing import Any
+from typing import Annotated, Any
 from urllib.parse import unquote
 from uuid import UUID, uuid4
 
@@ -57,7 +57,9 @@ async def get_observations_for_response(
     storage: Storage = DEFAULT_STORAGE,
     ensemble_id: UUID,
     response_key: str,
-    filter_on: str | None = Query(None, description="JSON string with filters"),
+    filter_on: Annotated[
+        str | None, Query(description="JSON string with filters")
+    ] = None,
 ) -> list[js.ObservationOut]:
     response_key = unquote(response_key)
     try:

ert/dark_storage/endpoints/parameters.py

@@ -113,27 +113,11 @@ def get_parameter_std_dev(
     return Response(content=buffer.getvalue(), media_type="application/octet-stream")
 
 
-def _extract_parameter_group_and_key(key: str) -> tuple[str, str] | tuple[None, None]:
-    key = key.removeprefix("LOG10_")
-    if ":" not in key:
-        # Assume all incoming keys are in format group:key for now
-        return None, None
-
-    param_group, param_key = key.split(":", maxsplit=1)
-    return param_group, param_key
-
-
 def data_for_parameter(ensemble: Ensemble, key: str) -> pd.DataFrame:
-    group, _ = _extract_parameter_group_and_key(key)
-    if group is None:
-        logger.warning(
-            f"Parameter with key '{key}' does not have a group, "
-            "fetching data for all parameters"
-        )
     try:
-        df = ensemble.load_scalars(group)
+        df = ensemble.load_scalar_keys([key], transformed=True)
         if df.is_empty():
-            logger.warning(f"No data found for parameter '{group}:{key}'")
+            logger.warning(f"No data found for parameter '{key}'")
             return pd.DataFrame()
     except KeyError as e:
         logger.error(e)

ert/dark_storage/endpoints/responses.py

@@ -45,7 +45,9 @@ async def get_response(
     storage: Storage = DEFAULT_STORAGE,
     ensemble_id: UUID,
     response_key: str,
-    filter_on: str | None = Query(None, description="JSON string with filters"),
+    filter_on: Annotated[
+        str | None, Query(description="JSON string with filters")
+    ] = None,
     accept: Annotated[str | None, Header()] = None,
 ) -> Response:
     try:
@@ -96,6 +98,7 @@ async def get_response(
 response_to_pandas_x_axis_fns: dict[str, Callable[[tuple[Any, ...]], Any]] = {
     "summary": lambda t: pd.Timestamp(t[2]).isoformat(),
     "gen_data": lambda t: str(t[3]),
+    "rft": lambda t: str(t[4]),
 }
 
 
@@ -147,8 +150,9 @@ def data_for_response(
         # This performs the same aggragation by mean of duplicate values
         # as in ert/analysis/_es_update.py
         df = df.groupby(["Date", "Realization"]).mean()
-        data = df.unstack(level="Date")
-        data.columns = data.columns.droplevel(0)
+        data = df.reset_index().pivot_table(
+            index="Realization", columns="Date", values=df.columns[0]
+        )
         return data.astype(float)
 
     if response_type == "rft":
@@ -159,8 +163,9 @@ def data_for_response(
             )
             .rename({"realization": "Realization"})
             .select(["Realization", "depth", "values"])
+            .unique()
             .to_pandas()
-            .pivot(index="Realization", columns="depth", values="values")
+            .pivot_table(index="Realization", columns="depth", values="values")
             .reset_index(drop=True)
         )
 
@@ -172,7 +177,7 @@ def data_for_response(
             assert "report_step" in filter_on
             report_step = int(filter_on["report_step"])
             vals = data.filter(pl.col("report_step").eq(report_step))
-            pivoted = vals.drop("response_key", "report_step").pivot(
+            pivoted = vals.drop("response_key", "report_step").pivot(  # noqa: PD010
                 on="index", values="values"
             )
             data = pivoted.to_pandas().set_index("realization")

ert/dark_storage/json_schema/experiment.py

@@ -24,6 +24,6 @@ class ExperimentOut(_Experiment):
     ensemble_ids: list[UUID]
     priors: Mapping[str, dict[str, Any]]
     userdata: Mapping[str, Any]
-    parameters: Mapping[str, list[dict[str, Any]]]
+    parameters: Mapping[str, dict[str, Any]]
     responses: Mapping[str, list[dict[str, Any]]]
     observations: Mapping[str, dict[str, list[str]]]

ert/data/_measured_data.py

@@ -141,13 +141,14 @@ class MeasuredData:
 
         # Pandas differentiates vs int and str keys.
         # Legacy-wise we use int keys for realizations
-        pddf.rename(
-            columns={str(k): int(k) for k in active_realizations},
-            inplace=True,
+        pddf = (
+            pddf.rename(
+                columns={str(k): int(k) for k in active_realizations},
+            )
+            .set_index(["observation_key", "key_index"])
+            .transpose()
         )
 
-        pddf = pddf.set_index(["observation_key", "key_index"]).transpose()
-
         return pddf
 
 

ert/ensemble_evaluator/config.py

@@ -27,6 +27,7 @@ class EvaluatorServerConfig:
         use_token: bool = True,
         host: str | None = None,
         use_ipc_protocol: bool = True,
+        prioritize_private_ip_address: bool = False,
     ) -> None:
         self.host: str | None = host
         self.router_port: int | None = None
@@ -50,7 +51,7 @@ class EvaluatorServerConfig:
         if use_ipc_protocol:
             self.uri = f"ipc:///tmp/socket-{uuid.uuid4().hex[:8]}"
         elif self.host is None:
-            self.host = get_ip_address()
+            self.host = get_ip_address(prioritize_private_ip_address)
 
         if use_token:
             self.server_public_key, self.server_secret_key = zmq.curve_keypair()

ert/field_utils/field_utils.py

@@ -238,7 +238,7 @@ def read_field(
         ext = path.suffix
         raise ValueError(f'Could not read {field_path}. Unrecognized suffix "{ext}"')
 
-    return np.ma.MaskedArray(data=values, fill_value=np.nan)  # type: ignore
+    return np.ma.MaskedArray(data=values, fill_value=np.nan)
 
 
 def save_field(

ert/field_utils/grdecl_io.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import io
 import operator
 import os
 from collections.abc import Iterator
@@ -258,9 +257,6 @@ def import_bgrdecl(
     raise ValueError(f"Did not find field parameter {field_name} in {file_path}")
 
 
-_BUFFER_SIZE = 2**20  # 1.04 megabytes
-
-
 def export_grdecl(
     values: np.ma.MaskedArray[Any, np.dtype[np.float32]] | npt.NDArray[np.float32],
     file_path: str | os.PathLike[str],
@@ -275,25 +271,12 @@ def export_grdecl(
     if binary:
         resfo.write(file_path, [(param_name.ljust(8), values.astype(np.float32))])
     else:
-        length = values.shape[0]
-        per_line = 6
-        iters = 5
-        per_iter = per_line * iters
-        fmt = " ".join(["%3e"] * per_line)
-        fmt = "\n".join([fmt] * iters) + "\n"
-        with (
-            open(file_path, "wb+", 0) as fh,
-            io.BufferedWriter(fh, _BUFFER_SIZE) as bw,
-            io.TextIOWrapper(bw, write_through=True, encoding="utf-8") as tw,
-        ):
-            tw.write(param_name + "\n")
-            i = 0
-            while i + per_iter <= length:
-                tw.write(fmt % tuple(values[i : i + per_iter]))
-                i += per_iter
-
-            for j, v in enumerate(values[length - (length % per_iter) :]):
-                tw.write(f" {v:3e}")
-                if j % 6 == 5:
-                    tw.write("\n")
-            tw.write(" /\n")
+        with open(file_path, "w", encoding="utf-8") as fh:
+            fh.write(param_name + "\n")
+            for i, v in enumerate(values):
+                fh.write(" ")
+                fh.write(f"{v:3e}")
+                if i % 6 == 5:
+                    fh.write("\n")
+
+            fh.write(" /\n")

ert/field_utils/roff_io.py

@@ -23,7 +23,7 @@ def export_roff(
     binary: bool,
 ) -> None:
     dimensions = data.shape
-    data = np.flip(data, -1).ravel()  # type: ignore
+    data = np.flip(data, -1).ravel()
     data = data.astype(np.float32).filled(RMS_UNDEFINED_FLOAT)  # type: ignore
     if not np.isfinite(data).all():
         raise ValueError(