ert 17.0.0__py3-none-any.whl → 19.0.0rc2__py3-none-any.whl

ert/config/surface_config.py

@@ -10,11 +10,16 @@ import xarray as xr
 from pydantic import field_serializer
 from surfio import IrapHeader, IrapSurface
 
+from ert.field_utils import (
+    calc_rho_for_2d_grid_layer,
+    transform_local_ellipse_angle_to_local_coords,
+    transform_positions_to_local_field_coordinates,
+)
 from ert.substitutions import substitute_runpath_name
 
 from ._str_to_bool import str_to_bool
 from .field import create_flattened_cube_graph
-from .parameter_config import InvalidParameterFile, ParameterConfig, ParameterMetadata
+from .parameter_config import InvalidParameterFile, ParameterConfig
 from .parsing import ConfigValidationError, ErrorInfo
 
 if TYPE_CHECKING:
@@ -46,6 +51,7 @@ class SurfaceMismatchError(InvalidParameterFile):
 
 class SurfaceConfig(ParameterConfig):
     type: Literal["surface"] = "surface"
+    dimensionality: Literal[2] = 2
     ncol: int
     nrow: int
     xori: float
@@ -70,21 +76,6 @@ class SurfaceConfig(ParameterConfig):
     def parameter_keys(self) -> list[str]:
         return []
 
-    @property
-    def metadata(self) -> list[ParameterMetadata]:
-        return [
-            ParameterMetadata(
-                key=self.name,
-                dimensionality=2,
-                transformation=None,
-                userdata={
-                    "data_origin": "SURFACE",
-                    "nx": self.ncol,
-                    "ny": self.nrow,
-                },
-            )
-        ]
-
     @classmethod
     def from_config_list(cls, config_list: list[str | dict[str, str]]) -> Self:
         name = cast(str, config_list[0])
@@ -248,3 +239,55 @@ class SurfaceConfig(ParameterConfig):
         this flattening process"""
 
         return create_flattened_cube_graph(px=self.ncol, py=self.nrow, pz=1)
+
+    def calc_rho_for_2d_grid_layer(
+        self,
+        obs_xpos: npt.NDArray[np.float64],
+        obs_ypos: npt.NDArray[np.float64],
+        obs_main_range: npt.NDArray[np.float64],
+        obs_perp_range: npt.NDArray[np.float64],
+        obs_anisotropy_angle: npt.NDArray[np.float64],
+    ) -> npt.NDArray[np.float64]:
+        """Function to calculate scaling values to be used in the RHO matrix
+        for distance-based localization.
+
+        Args:
+            obs_xpos: x-coordinates in global coordinates of observations
+            obs_ypos: y-coordinates in global coordinates of observations
+            obs_main_range: Size of influence ellipse main principal direction.
+            obs_perp_range: Size of influence ellipse second principal direction.
+            obs_anisotropy_angle: Rotation angle anticlock wise of main principal
+              direction of influence ellipse relative to global coordinate
+              system's x-axis.
+
+        Returns:
+            Scaling values (elements of the RHO matrix) as a numpy array
+              of shape=(nx,ny,nobservations)
+
+        """
+        # Transform observation positions to local surface coordinates
+        xpos, ypos = transform_positions_to_local_field_coordinates(
+            (self.xori, self.yori), self.rotation, obs_xpos, obs_ypos
+        )
+        # Transform ellipse orientation to local surface coordinates
+        rotation_angle_of_localization_ellipse = (
+            transform_local_ellipse_angle_to_local_coords(
+                self.rotation, obs_anisotropy_angle
+            )
+        )
+
+        # Assume the coordinate system is not flipped.
+        # This means the right_handed_grid_indexing is False
+        assert self.yflip == 1
+        return calc_rho_for_2d_grid_layer(
+            self.ncol,
+            self.nrow,
+            self.xinc,
+            self.yinc,
+            xpos,
+            ypos,
+            obs_main_range,
+            obs_perp_range,
+            rotation_angle_of_localization_ellipse,
+            right_handed_grid_indexing=False,
+        )

ert/config/workflow_fixtures.py

@@ -6,12 +6,13 @@ import typing
 from dataclasses import dataclass, fields
 from typing import TYPE_CHECKING, Literal
 
-from PyQt6.QtWidgets import QWidget
 from typing_extensions import TypedDict
 
 from ert.config.parsing.hook_runtime import HookRuntime
 
 if TYPE_CHECKING:
+    from PyQt6.QtWidgets import QWidget
+
     from ert.config import ESSettings, ObservationSettings
     from ert.runpaths import Runpaths
     from ert.storage import Ensemble, Storage

ert/dark_storage/client/__init__.py

@@ -1,4 +1,4 @@
-from ._session import ConnInfo
+from ._session import ErtClientConnectionInfo
 from .client import Client
 
-__all__ = ["Client", "ConnInfo"]
+__all__ = ["Client", "ErtClientConnectionInfo"]

ert/dark_storage/client/_session.py

@@ -5,7 +5,7 @@ from pathlib import Path
 from pydantic import BaseModel, ValidationError
 
 
-class ConnInfo(BaseModel):
+class ErtClientConnectionInfo(BaseModel):
     base_url: str
     auth_token: str | None = None
     cert: str | bool = False
@@ -17,10 +17,10 @@ ENV_VAR = "ERT_STORAGE_CONNECTION_STRING"
 # that a single client process will only ever want to connect to a single ERT
 # Storage server during its lifetime, so we don't provide an API for managing
 # this cache.
-_CACHED_CONN_INFO: ConnInfo | None = None
+_CACHED_CONN_INFO: ErtClientConnectionInfo | None = None
 
 
-def find_conn_info() -> ConnInfo:
+def find_conn_info() -> ErtClientConnectionInfo:
     """
     The base url and auth token are read from either:
     The file `storage_server.json`, starting from the current working directory
@@ -54,7 +54,7 @@ def find_conn_info() -> ConnInfo:
         raise RuntimeError("No Storage connection configuration found")
 
     try:
-        conn_info = ConnInfo.model_validate_json(conn_str)
+        conn_info = ErtClientConnectionInfo.model_validate_json(conn_str)
     except (json.JSONDecodeError, ValidationError) as e:
         raise RuntimeError("Invalid storage connection configuration") from e
     else:

ert/dark_storage/client/client.py

@@ -3,7 +3,7 @@ import ssl
 import httpx
 from httpx_retries import Retry, RetryTransport
 
-from ._session import ConnInfo, find_conn_info
+from ._session import ErtClientConnectionInfo, find_conn_info
 
 
 class Client(httpx.Client):
@@ -14,7 +14,7 @@ class Client(httpx.Client):
     Stores 'conn_info' to bridge the gap to the Everest client setup
     """
 
-    def __init__(self, conn_info: ConnInfo | None = None) -> None:
+    def __init__(self, conn_info: ErtClientConnectionInfo | None = None) -> None:
         if conn_info is None:
             conn_info = find_conn_info()
 

ert/dark_storage/common.py

@@ -28,7 +28,7 @@ def get_storage() -> Storage:
         except ErtStorageException as err:
             logger.exception(f"Error accessing storage: {err!s}")
             raise InternalServerError("Error accessing storage") from None
-    _storage.refresh()
+    _storage.reload()
     return _storage
 
 

ert/dark_storage/compute/misfits.py

@@ -5,17 +5,18 @@ import numpy.typing as npt
 import pandas as pd
 
 
-def _calculate_misfit(
+def _calculate_signed_chi_squared_misfit(
     obs_value: npt.NDArray[np.float64],
     response_value: npt.NDArray[np.float64],
     obs_std: npt.NDArray[np.float64],
 ) -> list[float]:
-    difference = response_value - obs_value
-    misfit = (difference / obs_std) ** 2
-    return (misfit * np.sign(difference)).tolist()
+    """The signed version is intended for visualization. For data assimiliation one
+    would normally use the normal chi-square"""
+    residual = response_value - obs_value
+    return (np.sign(residual) * residual * residual / (obs_std * obs_std)).tolist()
 
 
-def calculate_misfits_from_pandas(
+def calculate_signed_chi_squared_misfits(
     reponses_dict: Mapping[int, pd.DataFrame],
     observation: pd.DataFrame,
     summary_misfits: bool = False,
@@ -26,9 +27,12 @@ def calculate_misfits_from_pandas(
     """
     misfits_dict = {}
     for realization_index in reponses_dict:
-        misfits_dict[realization_index] = _calculate_misfit(
+        misfits_dict[realization_index] = _calculate_signed_chi_squared_misfit(
             observation["values"],
-            reponses_dict[realization_index].loc[:, observation.index].values.flatten(),
+            reponses_dict[realization_index]
+            .loc[:, observation.index]
+            .to_numpy()
+            .flatten(),
             observation["errors"],
         )
 

ert/dark_storage/endpoints/compute/misfits.py

@@ -1,6 +1,6 @@
 import json
 from datetime import datetime
-from typing import Any
+from typing import Annotated, Any
 from uuid import UUID
 
 import pandas as pd
@@ -10,7 +10,7 @@ from fastapi.responses import Response
 
 from ert.dark_storage import exceptions as exc
 from ert.dark_storage.common import get_storage
-from ert.dark_storage.compute.misfits import calculate_misfits_from_pandas
+from ert.dark_storage.compute.misfits import calculate_signed_chi_squared_misfits
 from ert.dark_storage.endpoints.observations import (
     _get_observations,
 )
@@ -36,7 +36,9 @@ async def get_response_misfits(
     response_name: str,
     realization_index: int | None = None,
     summary_misfits: bool = False,
-    filter_on: str | None = Query(None, description="JSON string with filters"),
+    filter_on: Annotated[
+        str | None, Query(description="JSON string with filters")
+    ] = None,
 ) -> Response:
     ensemble = storage.get_ensemble(ensemble_id)
     dataframe = data_for_response(
@@ -80,7 +82,7 @@ async def get_response_misfits(
         index=[parse_index(x) for x in o["x_axis"]],
     )
     try:
-        result_df = calculate_misfits_from_pandas(
+        result_df = calculate_signed_chi_squared_misfits(
             response_dict, observation_df, summary_misfits
         )
     except Exception as misfits_exc:

ert/dark_storage/endpoints/experiment_server.py

@@ -9,6 +9,7 @@ import traceback
 import uuid
 from base64 import b64decode
 from queue import SimpleQueue
+from typing import Annotated
 
 from fastapi import (
     APIRouter,
@@ -153,7 +154,7 @@ def _log(request: Request) -> None:
 
 @router.get("/")
 def get_status(
-    request: Request, credentials: HTTPBasicCredentials = Depends(security)
+    request: Request, credentials: Annotated[HTTPBasicCredentials, Depends(security)]
 ) -> PlainTextResponse:
     _log(request)
     _check_user(credentials)
@@ -162,7 +163,7 @@ def get_status(
 
 @router.get("/status")
 def experiment_status(
-    request: Request, credentials: HTTPBasicCredentials = Depends(security)
+    request: Request, credentials: Annotated[HTTPBasicCredentials, Depends(security)]
 ) -> ExperimentStatus:
     _log(request)
     _check_user(credentials)
@@ -171,7 +172,7 @@ def experiment_status(
 
 @router.post("/" + EverEndpoints.stop)
 def stop(
-    request: Request, credentials: HTTPBasicCredentials = Depends(security)
+    request: Request, credentials: Annotated[HTTPBasicCredentials, Depends(security)]
 ) -> Response:
     _log(request)
     _check_user(credentials)
@@ -185,7 +186,7 @@ def stop(
 async def start_experiment(
     request: Request,
     background_tasks: BackgroundTasks,
-    credentials: HTTPBasicCredentials = Depends(security),
+    credentials: Annotated[HTTPBasicCredentials, Depends(security)],
 ) -> Response:
     _log(request)
     _check_user(credentials)
@@ -218,7 +219,7 @@ async def start_experiment(
 
 @router.get("/" + EverEndpoints.config_path)
 async def config_path(
-    request: Request, credentials: HTTPBasicCredentials = Depends(security)
+    request: Request, credentials: Annotated[HTTPBasicCredentials, Depends(security)]
 ) -> JSONResponse:
     _log(request)
     _check_user(credentials)
@@ -237,7 +238,7 @@ async def config_path(
 
 @router.get("/" + EverEndpoints.start_time)
 async def start_time(
-    request: Request, credentials: HTTPBasicCredentials = Depends(security)
+    request: Request, credentials: Annotated[HTTPBasicCredentials, Depends(security)]
 ) -> Response:
     _log(request)
     _check_user(credentials)
@@ -316,9 +317,11 @@ class ExperimentRunner:
             simulation_future = loop.run_in_executor(
                 None,
                 lambda: run_model.start_simulations_thread(
-                    EvaluatorServerConfig()
-                    if run_model.queue_config.queue_system == QueueSystem.LOCAL
-                    else EvaluatorServerConfig(use_ipc_protocol=False)
+                    EvaluatorServerConfig(
+                        use_ipc_protocol=run_model.queue_config.queue_system
+                        == QueueSystem.LOCAL,
+                        prioritize_private_ip_address=site_plugins.prioritize_private_ip_address,
+                    )
                 ),
             )
             while True:

ert/dark_storage/endpoints/experiments.py

@@ -29,7 +29,7 @@ def get_experiments(
             priors=create_priors(experiment),
             userdata={},
             parameters={
-                group: [m.model_dump() for m in config.metadata]
+                group: config.model_dump()
                 for group, config in experiment.parameter_configuration.items()
                 if not isinstance(config, SurfaceConfig)
             },
@@ -65,7 +65,7 @@ def get_experiment_by_id(
         priors=create_priors(experiment),
         userdata={},
         parameters={
-            group: [m.model_dump() for m in config.metadata]
+            group: config.model_dump()
             for group, config in experiment.parameter_configuration.items()
         },
         responses={

ert/dark_storage/endpoints/observations.py

@@ -1,7 +1,7 @@
 import json
 import logging
 import operator
-from typing import Any
+from typing import Annotated, Any
 from urllib.parse import unquote
 from uuid import UUID, uuid4
 
@@ -57,7 +57,9 @@ async def get_observations_for_response(
     storage: Storage = DEFAULT_STORAGE,
     ensemble_id: UUID,
     response_key: str,
-    filter_on: str | None = Query(None, description="JSON string with filters"),
+    filter_on: Annotated[
+        str | None, Query(description="JSON string with filters")
+    ] = None,
 ) -> list[js.ObservationOut]:
     response_key = unquote(response_key)
     try:
@@ -136,13 +138,13 @@ def _get_observations(
         df = df.with_columns(pl.Series(name="x_axis", values=df.map_rows(x_axis_fn)))
         df = df.sort("x_axis")
 
-        for obs_key, _obs_df in df.group_by("name"):
+        for obs_key, obs_df in df.group_by("name"):
             observations.append(
                 {
                     "name": obs_key[0],
-                    "values": _obs_df["values"].to_list(),
-                    "errors": _obs_df["errors"].to_list(),
-                    "x_axis": _obs_df["x_axis"].to_list(),
+                    "values": obs_df["values"].to_list(),
+                    "errors": obs_df["errors"].to_list(),
+                    "x_axis": obs_df["x_axis"].to_list(),
                 }
             )
 

ert/dark_storage/endpoints/parameters.py

@@ -113,27 +113,11 @@ def get_parameter_std_dev(
     return Response(content=buffer.getvalue(), media_type="application/octet-stream")
 
 
-def _extract_parameter_group_and_key(key: str) -> tuple[str, str] | tuple[None, None]:
-    key = key.removeprefix("LOG10_")
-    if ":" not in key:
-        # Assume all incoming keys are in format group:key for now
-        return None, None
-
-    param_group, param_key = key.split(":", maxsplit=1)
-    return param_group, param_key
-
-
 def data_for_parameter(ensemble: Ensemble, key: str) -> pd.DataFrame:
-    group, _ = _extract_parameter_group_and_key(key)
-    if group is None:
-        logger.warning(
-            f"Parameter with key '{key}' does not have a group, "
-            "fetching data for all parameters"
-        )
     try:
-        df = ensemble.load_scalars(group)
+        df = ensemble.load_scalar_keys([key], transformed=True)
         if df.is_empty():
-            logger.warning(f"No data found for parameter '{group}:{key}'")
+            logger.warning(f"No data found for parameter '{key}'")
             return pd.DataFrame()
     except KeyError as e:
         logger.error(e)

ert/dark_storage/endpoints/responses.py

@@ -45,7 +45,9 @@ async def get_response(
     storage: Storage = DEFAULT_STORAGE,
     ensemble_id: UUID,
     response_key: str,
-    filter_on: str | None = Query(None, description="JSON string with filters"),
+    filter_on: Annotated[
+        str | None, Query(description="JSON string with filters")
+    ] = None,
     accept: Annotated[str | None, Header()] = None,
 ) -> Response:
     try:
@@ -96,6 +98,7 @@ async def get_response(
 response_to_pandas_x_axis_fns: dict[str, Callable[[tuple[Any, ...]], Any]] = {
     "summary": lambda t: pd.Timestamp(t[2]).isoformat(),
     "gen_data": lambda t: str(t[3]),
+    "rft": lambda t: str(t[4]),
 }
 
 
@@ -116,7 +119,7 @@ def _extract_response_type_and_key(
 
 def data_for_response(
     ensemble: Ensemble, key: str, filter_on: dict[str, Any] | None = None
-) -> pd.DataFrame:
+) -> pd.DataFrame | pd.Series:
     response_key, response_type = _extract_response_type_and_key(
         key, ensemble.experiment.response_key_to_response_type
     )
@@ -147,10 +150,25 @@ def data_for_response(
         # This performs the same aggragation by mean of duplicate values
         # as in ert/analysis/_es_update.py
         df = df.groupby(["Date", "Realization"]).mean()
-        data = df.unstack(level="Date")
-        data.columns = data.columns.droplevel(0)
+        data = df.reset_index().pivot_table(
+            index="Realization", columns="Date", values=df.columns[0]
+        )
         return data.astype(float)
 
+    if response_type == "rft":
+        return (
+            ensemble.load_responses(
+                response_key,
+                tuple(realizations_with_responses),
+            )
+            .rename({"realization": "Realization"})
+            .select(["Realization", "depth", "values"])
+            .unique()
+            .to_pandas()
+            .pivot_table(index="Realization", columns="depth", values="values")
+            .reset_index(drop=True)
+        )
+
     if response_type == "gen_data":
         data = ensemble.load_responses(response_key, tuple(realizations_with_responses))
 
@@ -159,7 +177,7 @@ def data_for_response(
             assert "report_step" in filter_on
             report_step = int(filter_on["report_step"])
             vals = data.filter(pl.col("report_step").eq(report_step))
-            pivoted = vals.drop("response_key", "report_step").pivot(
+            pivoted = vals.drop("response_key", "report_step").pivot(  # noqa: PD010
                 on="index", values="values"
             )
             data = pivoted.to_pandas().set_index("realization")
@@ -169,3 +187,4 @@ def data_for_response(
 
         except (ValueError, KeyError, ColumnNotFoundError):
             return pd.DataFrame()
+    return pd.DataFrame()

ert/dark_storage/json_schema/experiment.py

@@ -24,6 +24,6 @@ class ExperimentOut(_Experiment):
     ensemble_ids: list[UUID]
     priors: Mapping[str, dict[str, Any]]
     userdata: Mapping[str, Any]
-    parameters: Mapping[str, list[dict[str, Any]]]
+    parameters: Mapping[str, dict[str, Any]]
     responses: Mapping[str, list[dict[str, Any]]]
     observations: Mapping[str, dict[str, list[str]]]

ert/data/_measured_data.py

@@ -141,13 +141,14 @@ class MeasuredData:
 
         # Pandas differentiates vs int and str keys.
         # Legacy-wise we use int keys for realizations
-        pddf.rename(
-            columns={str(k): int(k) for k in active_realizations},
-            inplace=True,
+        pddf = (
+            pddf.rename(
+                columns={str(k): int(k) for k in active_realizations},
+            )
+            .set_index(["observation_key", "key_index"])
+            .transpose()
         )
 
-        pddf = pddf.set_index(["observation_key", "key_index"]).transpose()
-
         return pddf
 
 

ert/ensemble_evaluator/__init__.py

@@ -2,7 +2,13 @@ from ._ensemble import LegacyEnsemble as Ensemble
 from ._ensemble import Realization
 from .config import EvaluatorServerConfig
 from .evaluator import EnsembleEvaluator
-from .event import EndEvent, FullSnapshotEvent, SnapshotUpdateEvent, WarningEvent
+from .event import (
+    EndEvent,
+    FullSnapshotEvent,
+    SnapshotUpdateEvent,
+    StartEvent,
+    WarningEvent,
+)
 from .snapshot import EnsembleSnapshot, FMStepSnapshot, RealizationSnapshot
 
 __all__ = [
@@ -16,5 +22,6 @@ __all__ = [
     "Realization",
     "RealizationSnapshot",
     "SnapshotUpdateEvent",
+    "StartEvent",
     "WarningEvent",
 ]

ert/ensemble_evaluator/config.py

@@ -27,6 +27,7 @@ class EvaluatorServerConfig:
         use_token: bool = True,
         host: str | None = None,
         use_ipc_protocol: bool = True,
+        prioritize_private_ip_address: bool = False,
     ) -> None:
         self.host: str | None = host
         self.router_port: int | None = None
@@ -50,7 +51,7 @@ class EvaluatorServerConfig:
         if use_ipc_protocol:
             self.uri = f"ipc:///tmp/socket-{uuid.uuid4().hex[:8]}"
         elif self.host is None:
-            self.host = get_ip_address()
+            self.host = get_ip_address(prioritize_private_ip_address)
 
         if use_token:
             self.server_public_key, self.server_secret_key = zmq.curve_keypair()