ert 16.0.9__py3-none-any.whl → 19.0.0rc2__py3-none-any.whl

ert/config/surface_config.py

@@ -10,11 +10,16 @@ import xarray as xr
 from pydantic import field_serializer
 from surfio import IrapHeader, IrapSurface
 
+from ert.field_utils import (
+    calc_rho_for_2d_grid_layer,
+    transform_local_ellipse_angle_to_local_coords,
+    transform_positions_to_local_field_coordinates,
+)
 from ert.substitutions import substitute_runpath_name
 
 from ._str_to_bool import str_to_bool
 from .field import create_flattened_cube_graph
-from .parameter_config import InvalidParameterFile, ParameterConfig, ParameterMetadata
+from .parameter_config import InvalidParameterFile, ParameterConfig
 from .parsing import ConfigValidationError, ErrorInfo
 
 if TYPE_CHECKING:
@@ -46,6 +51,7 @@ class SurfaceMismatchError(InvalidParameterFile):
 
 class SurfaceConfig(ParameterConfig):
     type: Literal["surface"] = "surface"
+    dimensionality: Literal[2] = 2
     ncol: int
     nrow: int
     xori: float
@@ -70,25 +76,10 @@ class SurfaceConfig(ParameterConfig):
     def parameter_keys(self) -> list[str]:
         return []
 
-    @property
-    def metadata(self) -> list[ParameterMetadata]:
-        return [
-            ParameterMetadata(
-                key=self.name,
-                dimensionality=2,
-                transformation=None,
-                userdata={
-                    "data_origin": "SURFACE",
-                    "nx": self.ncol,
-                    "ny": self.nrow,
-                },
-            )
-        ]
-
     @classmethod
-    def from_config_list(cls, surface: list[str | dict[str, str]]) -> Self:
-        name = cast(str, surface[0])
-        options = cast(dict[str, str], surface[1])
+    def from_config_list(cls, config_list: list[str | dict[str, str]]) -> Self:
+        name = cast(str, config_list[0])
+        options = cast(dict[str, str], config_list[1])
         init_file = options.get("INIT_FILES")
         out_file = options.get("OUTPUT_FILE")
         base_surface = options.get("BASE_SURFACE")
@@ -97,23 +88,27 @@ class SurfaceConfig(ParameterConfig):
         errors = []
         if not out_file:
             errors.append(
-                ErrorInfo("Missing required OUTPUT_FILE").set_context(surface)
+                ErrorInfo("Missing required OUTPUT_FILE").set_context(config_list)
             )
         if not init_file:
-            errors.append(ErrorInfo("Missing required INIT_FILES").set_context(surface))
+            errors.append(
+                ErrorInfo("Missing required INIT_FILES").set_context(config_list)
+            )
         elif not forward_init and not ("%d" in init_file or "<IENS>" in init_file):
             errors.append(
                 ErrorInfo(
                     "INIT_FILES must contain %d or <IENS> when FORWARD_INIT:FALSE"
-                ).set_context(surface)
+                ).set_context(config_list)
             )
         if not base_surface:
             errors.append(
-                ErrorInfo("Missing required BASE_SURFACE").set_context(surface)
+                ErrorInfo("Missing required BASE_SURFACE").set_context(config_list)
             )
         elif not Path(base_surface).exists():
             errors.append(
-                ErrorInfo(f"BASE_SURFACE:{base_surface} not found").set_context(surface)
+                ErrorInfo(f"BASE_SURFACE:{base_surface} not found").set_context(
+                    config_list
+                )
             )
         if errors:
             raise ConfigValidationError.from_collected(errors)
@@ -121,11 +116,11 @@ class SurfaceConfig(ParameterConfig):
         assert out_file is not None
         assert base_surface is not None
         try:
-            surf = IrapSurface.from_ascii_file(base_surface)
+            surf = IrapSurface.from_ascii_file(Path(base_surface))
             yflip = -1 if surf.header.yinc < 0 else 1
         except Exception as err:
             raise ConfigValidationError.with_context(
-                f"Could not load surface {base_surface!r}", surface
+                f"Could not load surface {base_surface!r}", config_list
             ) from err
         return cls(
             ncol=surf.header.ncol,
@@ -244,3 +239,55 @@ class SurfaceConfig(ParameterConfig):
         this flattening process"""
 
         return create_flattened_cube_graph(px=self.ncol, py=self.nrow, pz=1)
+
+    def calc_rho_for_2d_grid_layer(
+        self,
+        obs_xpos: npt.NDArray[np.float64],
+        obs_ypos: npt.NDArray[np.float64],
+        obs_main_range: npt.NDArray[np.float64],
+        obs_perp_range: npt.NDArray[np.float64],
+        obs_anisotropy_angle: npt.NDArray[np.float64],
+    ) -> npt.NDArray[np.float64]:
+        """Function to calculate scaling values to be used in the RHO matrix
+        for distance-based localization.
+
+        Args:
+            obs_xpos: x-coordinates in global coordinates of observations
+            obs_ypos: y-coordinates in global coordinates of observations
+            obs_main_range: Size of influence ellipse main principal direction.
+            obs_perp_range: Size of influence ellipse second principal direction.
+            obs_anisotropy_angle: Rotation angle anticlock wise of main principal
+              direction of influence ellipse relative to global coordinate
+              system's x-axis.
+
+        Returns:
+            Scaling values (elements of the RHO matrix) as a numpy array
+              of shape=(nx,ny,nobservations)
+
+        """
+        # Transform observation positions to local surface coordinates
+        xpos, ypos = transform_positions_to_local_field_coordinates(
+            (self.xori, self.yori), self.rotation, obs_xpos, obs_ypos
+        )
+        # Transform ellipse orientation to local surface coordinates
+        rotation_angle_of_localization_ellipse = (
+            transform_local_ellipse_angle_to_local_coords(
+                self.rotation, obs_anisotropy_angle
+            )
+        )
+
+        # Assume the coordinate system is not flipped.
+        # This means the right_handed_grid_indexing is False
+        assert self.yflip == 1
+        return calc_rho_for_2d_grid_layer(
+            self.ncol,
+            self.nrow,
+            self.xinc,
+            self.yinc,
+            xpos,
+            ypos,
+            obs_main_range,
+            obs_perp_range,
+            rotation_angle_of_localization_ellipse,
+            right_handed_grid_indexing=False,
+        )

ert/config/workflow_fixtures.py

@@ -6,12 +6,13 @@ import typing
 from dataclasses import dataclass, fields
 from typing import TYPE_CHECKING, Literal
 
-from PyQt6.QtWidgets import QWidget
 from typing_extensions import TypedDict
 
 from ert.config.parsing.hook_runtime import HookRuntime
 
 if TYPE_CHECKING:
+    from PyQt6.QtWidgets import QWidget
+
     from ert.config import ESSettings, ObservationSettings
     from ert.runpaths import Runpaths
     from ert.storage import Ensemble, Storage

ert/config/workflow_job.py

@@ -6,10 +6,23 @@ import os
 import textwrap
 from abc import ABC, abstractmethod
 from dataclasses import field
-from typing import TYPE_CHECKING, Self, TypeAlias, cast
+from typing import (
+    TYPE_CHECKING,
+    Annotated,
+    Any,
+    Literal,
+    Self,
+    TypeAlias,
+    cast,
+)
 
-from pydantic import field_serializer, field_validator, model_validator
+from pydantic import (
+    Field,
+    model_serializer,
+    model_validator,
+)
 from pydantic_core.core_schema import ValidationInfo
+from typing_extensions import TypedDict
 
 from ert.base_model_context import BaseModelWithContextSupport
 
@@ -42,7 +55,11 @@ def workflow_job_parser(file: str) -> ConfigDict:
     return parse(file, schema=schema)
 
 
-def workflow_job_from_file(config_file: str, name: str | None = None) -> WorkflowJob:
+def workflow_job_from_file(
+    config_file: str,
+    origin: Literal["user", "site"] = "site",
+    name: str | None = None,
+) -> WorkflowJob:
     if not name:
         name = os.path.basename(config_file)
 
@@ -71,20 +88,32 @@ def workflow_job_from_file(config_file: str, name: str | None = None) -> Workflo
         )
         logger.warning(msg)
         ConfigWarning.deprecation_warn(msg, content_dict["SCRIPT"])
+
         try:
             ert_script = ErtScript.loadScriptFromFile(script)
         # Bare Exception here as we have no control
         # of exceptions in the loaded ErtScript
         except Exception as err:
             raise ErtScriptLoadFailure(f"Failed to load {name}: {err}") from err
-        return ErtScriptWorkflow(
-            ert_script=ert_script,
-            name=name,
-            min_args=min_args,
-            max_args=max_args,
-            arg_types=arg_types_list,
-            stop_on_fail=bool(content_dict.get("STOP_ON_FAIL")),
-        )
+
+        if origin == "user":
+            return UserInstalledErtScriptWorkflow(
+                name=name,
+                min_args=min_args,
+                max_args=max_args,
+                arg_types=arg_types_list,
+                source=script,
+                stop_on_fail=bool(content_dict.get("STOP_ON_FAIL")),
+            )
+        else:
+            return ErtScriptWorkflow(
+                ert_script=ert_script,
+                name=name,
+                min_args=min_args,
+                max_args=max_args,
+                arg_types=arg_types_list,
+                stop_on_fail=bool(content_dict.get("STOP_ON_FAIL")),
+            )
     else:
         return ExecutableWorkflow(
             name=name,
@@ -139,72 +168,124 @@ class _WorkflowJob(BaseModelWithContextSupport, ABC):
 
 
 class ExecutableWorkflow(_WorkflowJob):
+    type: Literal["user_installed_executable"] = "user_installed_executable"
     executable: str | None = None
 
     def location(self) -> str | None:
         return self.executable
 
 
-class ErtScriptWorkflow(_WorkflowJob):
+class BaseErtScriptWorkflow(_WorkflowJob, ABC):
+    @abstractmethod
+    def load_ert_script_class(self) -> builtins.type[ErtScript]: ...
+
+    def location(self) -> str | None:
+        return (
+            str(self.load_ert_script_class()) if self.load_ert_script_class() else None
+        )
+
+    @model_validator(mode="after")
+    def validate_types(self) -> Self:
+        ertscript_class = self.load_ert_script_class()
+        if not isinstance(ertscript_class, type):
+            raise ErtScriptLoadFailure(
+                f"Failed to load {self.name}, ert_script is instance, expected "
+                f"type, got {ertscript_class}"
+            )
+        elif not issubclass(ertscript_class, ErtScript):
+            raise ErtScriptLoadFailure(
+                f"Failed to load {self.name}, script had wrong "
+                f"type, expected ErtScript, got {ertscript_class}"
+            )
+        if ertscript_class.__doc__:
+            self.description = textwrap.dedent(ertscript_class.__doc__.strip())
+        return self
+
+    @property
+    def source_package(self) -> str:
+        return self.load_ert_script_class().__module__.partition(".")[2]
+
+    def is_plugin(self) -> bool:
+        return issubclass(self.load_ert_script_class(), ErtPlugin)
+
+
+class _SerializedSiteInstalledErtScriptWorkflow(TypedDict):
+    type: Literal["site_installed"]
+    name: str
+
+
+class SiteInstalledErtScriptWorkflow(BaseErtScriptWorkflow):
     """
-    Single workflow configuration object
+    Single workflow configuration object installed from site plugins
     """
 
+    type: Literal["site_installed"] = "site_installed"
     ert_script: builtins.type[ErtScript] = None  # type: ignore
     description: str = ""
     examples: str | None = None
     category: str = "other"
 
-    @field_serializer("ert_script")
-    def serialize_ert_script(self, _: str | builtins.type[ErtScript]) -> str:
-        return self.name
+    @model_serializer(mode="plain")
+    def serialize_model(self) -> _SerializedSiteInstalledErtScriptWorkflow:
+        return {"type": "site_installed", "name": self.name}
 
-    @field_validator("ert_script", mode="before")
-    @classmethod
-    def deserialize_ert_script(
-        cls, ert_script: str | builtins.type[ErtScript], info: ValidationInfo
-    ) -> builtins.type[ErtScript]:
-        if isinstance(ert_script, type) and issubclass(ert_script, ErtScript):
-            return ert_script
+    def load_ert_script_class(self) -> builtins.type[ErtScript]:
+        return self.ert_script
 
+    @model_validator(mode="before")
+    @classmethod
+    def deserialize_model(
+        cls, values: dict[str, Any], info: ValidationInfo
+    ) -> dict[str, Any]:
         runtime_plugins = cast("ErtRuntimePlugins", info.context)
-        ertscript_workflow_job = runtime_plugins.installed_workflow_jobs.get(ert_script)
-
-        if ertscript_workflow_job is None:
+        name = values["name"]
+
+        if runtime_plugins is None:
+            if set(values.keys()) == {"name", "type"}:
+                raise ValueError(
+                    f"Cannot resolve workflow job {values},"
+                    f"as it expects a the workflow job {name}"
+                    f"to be installed."
+                )
+            return values
+
+        if name not in runtime_plugins.installed_workflow_jobs:
             raise KeyError(
-                f"Did not find installed workflow job: {ert_script}. "
-                f"installed workflow jobs are: "
-                f"{runtime_plugins.installed_workflow_jobs.keys()}"
+                f"Expected workflow job {name} to be installed "
+                f"via plugins, but it was not found. Please check that "
+                f"your python environment has it installed."
             )
-        assert isinstance(ertscript_workflow_job, ErtScriptWorkflow)
+        site_installed_wfjob = runtime_plugins.installed_workflow_jobs[name]
 
-        return ertscript_workflow_job.ert_script
+        # Intent: copy the site installed workflow to this instance.
+        # bypassing the model_serializer
+        return {
+            k: getattr(site_installed_wfjob, k)
+            for k in SiteInstalledErtScriptWorkflow.model_fields
+        }
 
-    def location(self) -> str | None:
-        return str(self.ert_script) if self.ert_script else None
 
-    @model_validator(mode="after")
-    def validate_types(self) -> Self:
-        if not isinstance(self.ert_script, type):
-            raise ErtScriptLoadFailure(
-                f"Failed to load {self.name}, ert_script is instance, expected "
-                f"type, got {self.ert_script}"
-            )
-        elif not issubclass(self.ert_script, ErtScript):
-            raise ErtScriptLoadFailure(
-                f"Failed to load {self.name}, script had wrong "
-                f"type, expected ErtScript, got {self.ert_script}"
-            )
-        if self.ert_script.__doc__ is not None:
-            self.description = textwrap.dedent(self.ert_script.__doc__.strip())
-        return self
+# We keep the old name for compatability with .legacy_ertscript_workflow
+# all of which add ErtScriptWorkflow (always through plugins, i.e., site-installed)
+ErtScriptWorkflow = SiteInstalledErtScriptWorkflow
 
-    @property
-    def source_package(self) -> str:
-        return self.ert_script.__module__.partition(".")[2]
 
-    def is_plugin(self) -> bool:
-        return issubclass(self.ert_script, ErtPlugin)
+class UserInstalledErtScriptWorkflow(BaseErtScriptWorkflow):
+    type: Literal["user_installed_ertscript"] = "user_installed_ertscript"
+    source: str
+
+    def load_ert_script_class(self) -> builtins.type[ErtScript]:
+        try:
+            return ErtScript.loadScriptFromFile(self.source)
+        except Exception as err:
+            raise ErtScriptLoadFailure(f"Failed to load {self.name}: {err}") from err
 
 
-WorkflowJob: TypeAlias = ErtScriptWorkflow | ExecutableWorkflow
+WorkflowJob: TypeAlias = Annotated[
+    (
+        SiteInstalledErtScriptWorkflow
+        | UserInstalledErtScriptWorkflow
+        | ExecutableWorkflow
+    ),
+    Field(discriminator="type"),
+]

ert/dark_storage/client/__init__.py

@@ -1,4 +1,4 @@
-from ._session import ConnInfo
+from ._session import ErtClientConnectionInfo
 from .client import Client
 
-__all__ = ["Client", "ConnInfo"]
+__all__ = ["Client", "ErtClientConnectionInfo"]

ert/dark_storage/client/_session.py

@@ -5,7 +5,7 @@ from pathlib import Path
 from pydantic import BaseModel, ValidationError
 
 
-class ConnInfo(BaseModel):
+class ErtClientConnectionInfo(BaseModel):
     base_url: str
     auth_token: str | None = None
     cert: str | bool = False
@@ -17,10 +17,10 @@ ENV_VAR = "ERT_STORAGE_CONNECTION_STRING"
 # that a single client process will only ever want to connect to a single ERT
 # Storage server during its lifetime, so we don't provide an API for managing
 # this cache.
-_CACHED_CONN_INFO: ConnInfo | None = None
+_CACHED_CONN_INFO: ErtClientConnectionInfo | None = None
 
 
-def find_conn_info() -> ConnInfo:
+def find_conn_info() -> ErtClientConnectionInfo:
     """
     The base url and auth token are read from either:
     The file `storage_server.json`, starting from the current working directory
@@ -54,7 +54,7 @@ def find_conn_info() -> ConnInfo:
         raise RuntimeError("No Storage connection configuration found")
 
     try:
-        conn_info = ConnInfo.model_validate_json(conn_str)
+        conn_info = ErtClientConnectionInfo.model_validate_json(conn_str)
     except (json.JSONDecodeError, ValidationError) as e:
         raise RuntimeError("Invalid storage connection configuration") from e
     else:

ert/dark_storage/client/client.py

@@ -3,7 +3,7 @@ import ssl
 import httpx
 from httpx_retries import Retry, RetryTransport
 
-from ._session import ConnInfo, find_conn_info
+from ._session import ErtClientConnectionInfo, find_conn_info
 
 
 class Client(httpx.Client):
@@ -14,7 +14,7 @@ class Client(httpx.Client):
     Stores 'conn_info' to bridge the gap to the Everest client setup
     """
 
-    def __init__(self, conn_info: ConnInfo | None = None) -> None:
+    def __init__(self, conn_info: ErtClientConnectionInfo | None = None) -> None:
         if conn_info is None:
             conn_info = find_conn_info()
 

ert/dark_storage/common.py

@@ -4,7 +4,12 @@ import re
 from importlib import metadata
 
 from ert.dark_storage.exceptions import InternalServerError
-from ert.storage import ErtStorageException, Storage, open_storage
+from ert.storage import (
+    ErtStorageException,
+    ErtStoragePermissionError,
+    Storage,
+    open_storage,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -17,9 +22,13 @@ def get_storage() -> Storage:
     if _storage is None:
         try:
             return (_storage := open_storage(os.environ["ERT_STORAGE_ENS_PATH"]))
+        except ErtStoragePermissionError as err:
+            logger.error(f"Permission error accessing storage: {err!s}")
+            raise InternalServerError("Permission error accessing storage") from None
         except ErtStorageException as err:
-            raise InternalServerError(f"{err!s}") from err
-    _storage.refresh()
+            logger.exception(f"Error accessing storage: {err!s}")
+            raise InternalServerError("Error accessing storage") from None
+    _storage.reload()
     return _storage
 
 

ert/dark_storage/compute/misfits.py

@@ -5,17 +5,18 @@ import numpy.typing as npt
 import pandas as pd
 
 
-def _calculate_misfit(
+def _calculate_signed_chi_squared_misfit(
     obs_value: npt.NDArray[np.float64],
     response_value: npt.NDArray[np.float64],
     obs_std: npt.NDArray[np.float64],
 ) -> list[float]:
-    difference = response_value - obs_value
-    misfit = (difference / obs_std) ** 2
-    return (misfit * np.sign(difference)).tolist()
+    """The signed version is intended for visualization. For data assimiliation one
+    would normally use the normal chi-square"""
+    residual = response_value - obs_value
+    return (np.sign(residual) * residual * residual / (obs_std * obs_std)).tolist()
 
 
-def calculate_misfits_from_pandas(
+def calculate_signed_chi_squared_misfits(
     reponses_dict: Mapping[int, pd.DataFrame],
     observation: pd.DataFrame,
     summary_misfits: bool = False,
@@ -26,9 +27,12 @@ def calculate_misfits_from_pandas(
     """
     misfits_dict = {}
     for realization_index in reponses_dict:
-        misfits_dict[realization_index] = _calculate_misfit(
+        misfits_dict[realization_index] = _calculate_signed_chi_squared_misfit(
             observation["values"],
-            reponses_dict[realization_index].loc[:, observation.index].values.flatten(),
+            reponses_dict[realization_index]
+            .loc[:, observation.index]
+            .to_numpy()
+            .flatten(),
             observation["errors"],
         )
 

ert/dark_storage/endpoints/compute/misfits.py

@@ -1,6 +1,6 @@
 import json
 from datetime import datetime
-from typing import Any
+from typing import Annotated, Any
 from uuid import UUID
 
 import pandas as pd
@@ -10,7 +10,7 @@ from fastapi.responses import Response
 
 from ert.dark_storage import exceptions as exc
 from ert.dark_storage.common import get_storage
-from ert.dark_storage.compute.misfits import calculate_misfits_from_pandas
+from ert.dark_storage.compute.misfits import calculate_signed_chi_squared_misfits
 from ert.dark_storage.endpoints.observations import (
     _get_observations,
 )
@@ -36,7 +36,9 @@ async def get_response_misfits(
     response_name: str,
     realization_index: int | None = None,
     summary_misfits: bool = False,
-    filter_on: str | None = Query(None, description="JSON string with filters"),
+    filter_on: Annotated[
+        str | None, Query(description="JSON string with filters")
+    ] = None,
 ) -> Response:
     ensemble = storage.get_ensemble(ensemble_id)
     dataframe = data_for_response(
@@ -80,7 +82,7 @@ async def get_response_misfits(
         index=[parse_index(x) for x in o["x_axis"]],
     )
     try:
-        result_df = calculate_misfits_from_pandas(
+        result_df = calculate_signed_chi_squared_misfits(
             response_dict, observation_df, summary_misfits
         )
     except Exception as misfits_exc:

ert/dark_storage/endpoints/ensembles.py

@@ -1,4 +1,5 @@
 import logging
+from collections import Counter
 from uuid import UUID
 
 from fastapi import APIRouter, Body, Depends, HTTPException
@@ -38,4 +39,7 @@ def get_ensemble(
             "started_at": ensemble.started_at,
         },
         size=ensemble.ensemble_size,
+        realization_storage_states=Counter(
+            state for states in ensemble.get_ensemble_state() for state in states
+        ),
     )