ert 19.0.0rc1__py3-none-any.whl → 19.0.0rc3__py3-none-any.whl

ert/dark_storage/compute/misfits.py

@@ -0,0 +1,42 @@
+from collections.abc import Mapping
+
+import numpy as np
+import numpy.typing as npt
+import pandas as pd
+
+
+def _calculate_signed_chi_squared_misfit(
+    obs_value: npt.NDArray[np.float64],
+    response_value: npt.NDArray[np.float64],
+    obs_std: npt.NDArray[np.float64],
+) -> list[float]:
+    """The signed version is intended for visualization. For data assimiliation one
+    would normally use the normal chi-square"""
+    residual = response_value - obs_value
+    return (np.sign(residual) * residual * residual / (obs_std * obs_std)).tolist()
+
+
+def calculate_signed_chi_squared_misfits(
+    reponses_dict: Mapping[int, pd.DataFrame],
+    observation: pd.DataFrame,
+    summary_misfits: bool = False,
+) -> pd.DataFrame:
+    """
+    Compute misfits from reponses_dict (real_id, values in dataframe)
+    and observation
+    """
+    misfits_dict = {}
+    for realization_index in reponses_dict:
+        misfits_dict[realization_index] = _calculate_signed_chi_squared_misfit(
+            observation["values"],
+            reponses_dict[realization_index]
+            .loc[:, observation.index]
+            .to_numpy()
+            .flatten(),
+            observation["errors"],
+        )
+
+    df = pd.DataFrame(data=misfits_dict, index=observation.index)
+    if summary_misfits:
+        df = pd.DataFrame([df.abs().sum(axis=0)], columns=df.columns, index=[0])
+    return df.T

ert/dark_storage/endpoints/__init__.py

@@ -1,5 +1,6 @@
 from fastapi import APIRouter
 
+from .compute.misfits import router as misfits_router
 from .ensembles import router as ensembles_router
 from .experiment_server import router as experiment_server_router
 from .experiments import router as experiments_router
@@ -14,6 +15,7 @@ router.include_router(experiments_router)
 router.include_router(ensembles_router)
 router.include_router(observations_router)
 router.include_router(updates_router)
+router.include_router(misfits_router)
 router.include_router(parameters_router)
 router.include_router(responses_router)
 router.include_router(experiment_server_router)

ert/dark_storage/endpoints/compute/misfits.py

@@ -0,0 +1,95 @@
+import json
+from datetime import datetime
+from typing import Annotated, Any
+from uuid import UUID
+
+import pandas as pd
+from dateutil.parser import parse
+from fastapi import APIRouter, Depends, Query, status
+from fastapi.responses import Response
+
+from ert.dark_storage import exceptions as exc
+from ert.dark_storage.common import get_storage
+from ert.dark_storage.compute.misfits import calculate_signed_chi_squared_misfits
+from ert.dark_storage.endpoints.observations import (
+    _get_observations,
+)
+from ert.dark_storage.endpoints.responses import data_for_response
+from ert.storage import Storage
+
+router = APIRouter(tags=["misfits"])
+DEFAULT_STORAGEREADER = Depends(get_storage)
+
+
+@router.get(
+    "/compute/misfits",
+    responses={
+        status.HTTP_200_OK: {
+            "content": {"text/csv": {}},
+        }
+    },
+)
+async def get_response_misfits(
+    *,
+    storage: Storage = DEFAULT_STORAGEREADER,
+    ensemble_id: UUID,
+    response_name: str,
+    realization_index: int | None = None,
+    summary_misfits: bool = False,
+    filter_on: Annotated[
+        str | None, Query(description="JSON string with filters")
+    ] = None,
+) -> Response:
+    ensemble = storage.get_ensemble(ensemble_id)
+    dataframe = data_for_response(
+        ensemble,
+        response_name,
+        json.loads(filter_on) if filter_on is not None else None,
+    )
+    if realization_index is not None:
+        dataframe = pd.DataFrame(dataframe.loc[realization_index]).T
+
+    response_dict = {}
+    for index, data in dataframe.iterrows():
+        data_df = pd.DataFrame(data).T
+        response_dict[index] = data_df
+
+    experiment = ensemble.experiment
+    response_type = experiment.response_key_to_response_type[response_name]
+    obs_keys = experiment.response_key_to_observation_key[response_type].get(
+        response_name, []
+    )
+    obs = _get_observations(
+        ensemble.experiment,
+        obs_keys,
+        json.loads(filter_on) if filter_on is not None else None,
+    )
+
+    if not obs_keys:
+        raise ValueError(f"No observations for key {response_name}")
+    if not obs:
+        raise ValueError(f"Cant fetch observations for key {response_name}")
+    o = obs[0]
+
+    def parse_index(x: Any) -> int | datetime:
+        try:
+            return int(x)
+        except ValueError:
+            return parse(x)
+
+    observation_df = pd.DataFrame(
+        data={"values": o["values"], "errors": o["errors"]},
+        index=[parse_index(x) for x in o["x_axis"]],
+    )
+    try:
+        result_df = calculate_signed_chi_squared_misfits(
+            response_dict, observation_df, summary_misfits
+        )
+    except Exception as misfits_exc:
+        raise exc.UnprocessableError(
+            f"Unable to compute misfits: {misfits_exc}"
+        ) from misfits_exc
+    return Response(
+        content=result_df.to_csv().encode(),
+        media_type="text/csv",
+    )

ert/dark_storage/endpoints/experiments.py

@@ -6,6 +6,7 @@ from fastapi import APIRouter, Body, Depends, HTTPException
 from ert.config import SurfaceConfig
 from ert.dark_storage import json_schema as js
 from ert.dark_storage.common import get_storage
+from ert.shared.storage.extraction import create_priors
 from ert.storage import Storage
 
 router = APIRouter(tags=["experiment"])
@@ -25,6 +26,7 @@ def get_experiments(
             id=experiment.id,
             name=experiment.name,
             ensemble_ids=[ens.id for ens in experiment.ensembles],
+            priors=create_priors(experiment),
             userdata={},
             parameters={
                 group: config.model_dump()
@@ -60,6 +62,7 @@ def get_experiment_by_id(
         name=experiment.name,
         id=experiment.id,
         ensemble_ids=[ens.id for ens in experiment.ensembles],
+        priors=create_priors(experiment),
         userdata={},
         parameters={
             group: config.model_dump()

ert/dark_storage/json_schema/experiment.py

@@ -22,6 +22,7 @@ class ExperimentIn(_Experiment):
 class ExperimentOut(_Experiment):
     id: UUID
     ensemble_ids: list[UUID]
+    priors: Mapping[str, dict[str, Any]]
     userdata: Mapping[str, Any]
     parameters: Mapping[str, dict[str, Any]]
     responses: Mapping[str, list[dict[str, Any]]]

ert/gui/main_window.py

@@ -116,10 +116,8 @@ class ErtMainWindow(QMainWindow):
 
         if is_dark_mode():
             self.side_frame.setStyleSheet("background-color: rgb(64, 64, 64);")
-            logger.info("Running Ert with dark mode")
         else:
             self.side_frame.setStyleSheet("background-color: lightgray;")
-            logger.info("Running Ert with light mode")
 
         if is_high_contrast_mode():
             msg_box = QMessageBox()

ert/gui/tools/manage_experiments/export_dialog.py

@@ -1,5 +1,4 @@
 import contextlib
-import logging
 from pathlib import Path
 from typing import cast
 
@@ -23,8 +22,6 @@ from PyQt6.QtWidgets import (
     QWidget,
 )
 
-logger = logging.getLogger(__name__)
-
 
 class ExportDialog(QDialog):
     """Base dialog for exporting ensemble-related data to files."""
@@ -88,7 +85,6 @@ class ExportDialog(QDialog):
                 f"<span style='color: red;'>Could not export data: {e!s}</span><br>"
             )
         finally:
-            logger.info(f"Export dialog used: '{self.windowTitle()}'")
             self._export_button.setEnabled(True)
 
     @Slot()

ert/gui/tools/manage_experiments/storage_info_widget.py

@@ -11,6 +11,7 @@ from polars import DataFrame
 from PyQt6.QtCore import Qt
 from PyQt6.QtCore import pyqtSlot as Slot
 from PyQt6.QtWidgets import (
+    QAbstractItemView,
     QFrame,
     QHBoxLayout,
     QLabel,
@@ -165,6 +166,9 @@ class _EnsembleWidget(QWidget):
         observations_frame.setLayout(observations_layout)
 
         self._parameters_table = QTableWidget()
+        self._parameters_table.setEditTriggers(
+            QAbstractItemView.EditTrigger.NoEditTriggers
+        )
         self._export_params_button = QPushButton("Export...")
         self._export_params_button.clicked.connect(self.onClickExportParameters)
 
@@ -249,8 +253,8 @@ class _EnsembleWidget(QWidget):
                 df[x_axis_col]
                 .map_elements(
                     lambda x: response_config.display_column(x, x_axis_col),
+                    return_dtype=pl.String,
                 )
-                .cast(pl.String)
                 .alias("temp")
             ).filter(pl.col("temp").eq(observation_label))[
                 [x for x in df.columns if x != "temp"]

ert/gui/tools/plot/plot_api.py

@@ -19,7 +19,7 @@ from pandas.errors import ParserError
 from resfo_utilities import history_key
 
 from ert.config import ParameterConfig, ResponseMetadata
-from ert.services import create_ertserver_client
+from ert.services import ErtServer
 from ert.storage.local_experiment import _parameters_adapter as parameter_config_adapter
 from ert.storage.realization_storage_state import RealizationStorageState
 
@@ -51,13 +51,13 @@ class PlotApiKeyDefinition(NamedTuple):
 
 class PlotApi:
     def __init__(self, ens_path: Path) -> None:
-        self.ens_path: Path = ens_path
+        self.ens_path = ens_path
         self._all_ensembles: list[EnsembleObject] | None = None
         self._timeout = 120
 
     @property
     def api_version(self) -> str:
-        with create_ertserver_client(self.ens_path) as client:
+        with ErtServer.session(project=self.ens_path) as client:
             try:
                 response = client.get("/version", timeout=self._timeout)
                 self._check_response(response)
@@ -83,7 +83,7 @@ class PlotApi:
             return self._all_ensembles
 
         self._all_ensembles = []
-        with create_ertserver_client(self.ens_path) as client:
+        with ErtServer.session(project=self.ens_path) as client:
             try:
                 response = client.get("/experiments", timeout=self._timeout)
                 self._check_response(response)
@@ -139,7 +139,7 @@ class PlotApi:
         all_keys: dict[str, PlotApiKeyDefinition] = {}
         all_params = {}
 
-        with create_ertserver_client(self.ens_path) as client:
+        with ErtServer.session(project=self.ens_path) as client:
             response = client.get("/experiments", timeout=self._timeout)
             self._check_response(response)
 
@@ -166,7 +166,7 @@ class PlotApi:
     def responses_api_key_defs(self) -> list[PlotApiKeyDefinition]:
         key_defs: dict[str, PlotApiKeyDefinition] = {}
 
-        with create_ertserver_client(self.ens_path) as client:
+        with ErtServer.session(project=self.ens_path) as client:
             response = client.get("/experiments", timeout=self._timeout)
             self._check_response(response)
 
@@ -228,7 +228,7 @@ class PlotApi:
         response_key: str,
         filter_on: dict[str, Any] | None = None,
     ) -> pd.DataFrame:
-        with create_ertserver_client(self.ens_path) as client:
+        with ErtServer.session(project=self.ens_path) as client:
             response = client.get(
                 f"/ensembles/{ensemble_id}/responses/{PlotApi.escape(response_key)}",
                 headers={"accept": "application/x-parquet"},
@@ -256,7 +256,7 @@ class PlotApi:
                 return df
 
     def data_for_parameter(self, ensemble_id: str, parameter_key: str) -> pd.DataFrame:
-        with create_ertserver_client(self.ens_path) as client:
+        with ErtServer.session(project=self.ens_path) as client:
             parameter = client.get(
                 f"/ensembles/{ensemble_id}/parameters/{PlotApi.escape(parameter_key)}",
                 headers={"accept": "application/x-parquet"},
@@ -298,7 +298,7 @@ class PlotApi:
             assert key_def.response_metadata is not None
             actual_response_key = key_def.response_metadata.response_key
             filter_on = key_def.filter_on
-            with create_ertserver_client(self.ens_path) as client:
+            with ErtServer.session(project=self.ens_path) as client:
                 response = client.get(
                     f"/ensembles/{ensemble.id}/responses/{PlotApi.escape(actual_response_key)}/observations",
                     timeout=self._timeout,
@@ -386,7 +386,7 @@ class PlotApi:
         if not ensemble:
             return np.array([])
 
-        with create_ertserver_client(self.ens_path) as client:
+        with ErtServer.session(project=self.ens_path) as client:
             response = client.get(
                 f"/ensembles/{ensemble.id}/parameters/{PlotApi.escape(key)}/std_dev",
                 params={"z": z},

ert/gui/tools/plot/plot_widget.py

@@ -153,7 +153,6 @@ class PlotWidget(QWidget):
         # only for histogram plot see _sync_log_checkbox
         self._log_checkbox.setVisible(False)
         self._log_checkbox.setToolTip("Toggle data domain to log scale and back")
-        self._log_checkbox.clicked.connect(self.logLogScaleButtonUsage)
 
         log_checkbox_row = QHBoxLayout()
         log_checkbox_row.addWidget(self._log_checkbox)
@@ -189,10 +188,6 @@ class PlotWidget(QWidget):
     def name(self) -> str:
         return self._name
 
-    def logLogScaleButtonUsage(self) -> None:
-        logger.info(f"Plotwidget utility used: 'Log scale button' in tab '{self.name}'")
-        self._log_checkbox.clicked.disconnect()  # Log only once
-
     def updatePlot(
         self,
         plot_context: "PlotContext",

ert/gui/tools/plot/plot_window.py

@@ -25,7 +25,7 @@ from PyQt6.QtWidgets import (
 from ert.config.field import Field
 from ert.dark_storage.common import get_storage_api_version
 from ert.gui.ertwidgets import CopyButton, showWaitCursorWhileWaiting
-from ert.services import ServerBootFail
+from ert.services._base_service import ServerBootFail
 from ert.utils import log_duration
 
 from .customize import PlotCustomizer

ert/services/__init__.py

@@ -1,8 +1,4 @@
-from .ert_server import (
-    ErtServer,
-    ErtServerExit,
-    ServerBootFail,
-    create_ertserver_client,
-)
+from .ert_server import ErtServer
+from .webviz_ert_service import WebvizErt
 
-__all__ = ["ErtServer", "ErtServerExit", "ServerBootFail", "create_ertserver_client"]
+__all__ = ["ErtServer", "WebvizErt"]