ert 19.0.0__py3-none-any.whl → 19.0.0rc1__py3-none-any.whl

ert/config/rft_config.py

@@ -194,7 +194,7 @@ class RFTConfig(ResponseConfig):
                     .explode("location")
                     for (well, time), inner_dict in fetched.items()
                     for prop, vals in inner_dict.items()
-                    if prop != "DEPTH"
+                    if prop != "DEPTH" and len(vals) > 0
                 ]
             )
         except KeyError as err:

ert/dark_storage/compute/misfits.py

@@ -1,42 +0,0 @@
-from collections.abc import Mapping
-
-import numpy as np
-import numpy.typing as npt
-import pandas as pd
-
-
-def _calculate_signed_chi_squared_misfit(
-    obs_value: npt.NDArray[np.float64],
-    response_value: npt.NDArray[np.float64],
-    obs_std: npt.NDArray[np.float64],
-) -> list[float]:
-    """The signed version is intended for visualization. For data assimiliation one
-    would normally use the normal chi-square"""
-    residual = response_value - obs_value
-    return (np.sign(residual) * residual * residual / (obs_std * obs_std)).tolist()
-
-
-def calculate_signed_chi_squared_misfits(
-    reponses_dict: Mapping[int, pd.DataFrame],
-    observation: pd.DataFrame,
-    summary_misfits: bool = False,
-) -> pd.DataFrame:
-    """
-    Compute misfits from reponses_dict (real_id, values in dataframe)
-    and observation
-    """
-    misfits_dict = {}
-    for realization_index in reponses_dict:
-        misfits_dict[realization_index] = _calculate_signed_chi_squared_misfit(
-            observation["values"],
-            reponses_dict[realization_index]
-            .loc[:, observation.index]
-            .to_numpy()
-            .flatten(),
-            observation["errors"],
-        )
-
-    df = pd.DataFrame(data=misfits_dict, index=observation.index)
-    if summary_misfits:
-        df = pd.DataFrame([df.abs().sum(axis=0)], columns=df.columns, index=[0])
-    return df.T

ert/dark_storage/endpoints/__init__.py

@@ -1,6 +1,5 @@
 from fastapi import APIRouter
 
-from .compute.misfits import router as misfits_router
 from .ensembles import router as ensembles_router
 from .experiment_server import router as experiment_server_router
 from .experiments import router as experiments_router
@@ -15,7 +14,6 @@ router.include_router(experiments_router)
 router.include_router(ensembles_router)
 router.include_router(observations_router)
 router.include_router(updates_router)
-router.include_router(misfits_router)
 router.include_router(parameters_router)
 router.include_router(responses_router)
 router.include_router(experiment_server_router)

ert/dark_storage/endpoints/experiments.py

@@ -6,7 +6,6 @@ from fastapi import APIRouter, Body, Depends, HTTPException
 from ert.config import SurfaceConfig
 from ert.dark_storage import json_schema as js
 from ert.dark_storage.common import get_storage
-from ert.shared.storage.extraction import create_priors
 from ert.storage import Storage
 
 router = APIRouter(tags=["experiment"])
@@ -26,7 +25,6 @@ def get_experiments(
             id=experiment.id,
             name=experiment.name,
             ensemble_ids=[ens.id for ens in experiment.ensembles],
-            priors=create_priors(experiment),
             userdata={},
             parameters={
                 group: config.model_dump()
@@ -62,7 +60,6 @@ def get_experiment_by_id(
         name=experiment.name,
         id=experiment.id,
         ensemble_ids=[ens.id for ens in experiment.ensembles],
-        priors=create_priors(experiment),
         userdata={},
         parameters={
             group: config.model_dump()

ert/dark_storage/json_schema/experiment.py

@@ -22,7 +22,6 @@ class ExperimentIn(_Experiment):
 class ExperimentOut(_Experiment):
     id: UUID
     ensemble_ids: list[UUID]
-    priors: Mapping[str, dict[str, Any]]
     userdata: Mapping[str, Any]
     parameters: Mapping[str, dict[str, Any]]
     responses: Mapping[str, list[dict[str, Any]]]

ert/field_utils/grdecl_io.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import io
 import operator
 import os
 from collections.abc import Iterator
@@ -258,9 +257,6 @@ def import_bgrdecl(
     raise ValueError(f"Did not find field parameter {field_name} in {file_path}")
 
 
-_BUFFER_SIZE = 2**20  # 1.04 megabytes
-
-
 def export_grdecl(
     values: np.ma.MaskedArray[Any, np.dtype[np.float32]] | npt.NDArray[np.float32],
     file_path: str | os.PathLike[str],
@@ -275,25 +271,12 @@ def export_grdecl(
     if binary:
         resfo.write(file_path, [(param_name.ljust(8), values.astype(np.float32))])
     else:
-        length = values.shape[0]
-        per_line = 6
-        iters = 5
-        per_iter = per_line * iters
-        fmt = " ".join(["%3e"] * per_line)
-        fmt = "\n".join([fmt] * iters) + "\n"
-        with (
-            open(file_path, "wb+", 0) as fh,
-            io.BufferedWriter(fh, _BUFFER_SIZE) as bw,
-            io.TextIOWrapper(bw, write_through=True, encoding="utf-8") as tw,
-        ):
-            tw.write(param_name + "\n")
-            i = 0
-            while i + per_iter <= length:
-                tw.write(fmt % tuple(values[i : i + per_iter]))
-                i += per_iter
-
-            for j, v in enumerate(values[length - (length % per_iter) :]):
-                tw.write(f" {v:3e}")
-                if j % 6 == 5:
-                    tw.write("\n")
-            tw.write(" /\n")
+        with open(file_path, "w", encoding="utf-8") as fh:
+            fh.write(param_name + "\n")
+            for i, v in enumerate(values):
+                fh.write(" ")
+                fh.write(f"{v:3e}")
+                if i % 6 == 5:
+                    fh.write("\n")
+
+            fh.write(" /\n")

ert/gui/main_window.py

@@ -116,8 +116,10 @@ class ErtMainWindow(QMainWindow):
 
         if is_dark_mode():
             self.side_frame.setStyleSheet("background-color: rgb(64, 64, 64);")
+            logger.info("Running Ert with dark mode")
         else:
             self.side_frame.setStyleSheet("background-color: lightgray;")
+            logger.info("Running Ert with light mode")
 
         if is_high_contrast_mode():
             msg_box = QMessageBox()

ert/gui/tools/manage_experiments/export_dialog.py

@@ -1,4 +1,5 @@
 import contextlib
+import logging
 from pathlib import Path
 from typing import cast
 
@@ -22,6 +23,8 @@ from PyQt6.QtWidgets import (
     QWidget,
 )
 
+logger = logging.getLogger(__name__)
+
 
 class ExportDialog(QDialog):
     """Base dialog for exporting ensemble-related data to files."""
@@ -85,6 +88,7 @@ class ExportDialog(QDialog):
                 f"<span style='color: red;'>Could not export data: {e!s}</span><br>"
             )
         finally:
+            logger.info(f"Export dialog used: '{self.windowTitle()}'")
             self._export_button.setEnabled(True)
 
     @Slot()

ert/gui/tools/manage_experiments/storage_info_widget.py

@@ -11,7 +11,6 @@ from polars import DataFrame
 from PyQt6.QtCore import Qt
 from PyQt6.QtCore import pyqtSlot as Slot
 from PyQt6.QtWidgets import (
-    QAbstractItemView,
     QFrame,
     QHBoxLayout,
     QLabel,
@@ -166,9 +165,6 @@ class _EnsembleWidget(QWidget):
         observations_frame.setLayout(observations_layout)
 
         self._parameters_table = QTableWidget()
-        self._parameters_table.setEditTriggers(
-            QAbstractItemView.EditTrigger.NoEditTriggers
-        )
         self._export_params_button = QPushButton("Export...")
         self._export_params_button.clicked.connect(self.onClickExportParameters)
 
@@ -253,8 +249,8 @@ class _EnsembleWidget(QWidget):
                 df[x_axis_col]
                 .map_elements(
                     lambda x: response_config.display_column(x, x_axis_col),
-                    return_dtype=pl.String,
                 )
+                .cast(pl.String)
                 .alias("temp")
             ).filter(pl.col("temp").eq(observation_label))[
                 [x for x in df.columns if x != "temp"]

ert/gui/tools/plot/plot_api.py

@@ -19,7 +19,7 @@ from pandas.errors import ParserError
 from resfo_utilities import history_key
 
 from ert.config import ParameterConfig, ResponseMetadata
-from ert.services import ErtServer
+from ert.services import create_ertserver_client
 from ert.storage.local_experiment import _parameters_adapter as parameter_config_adapter
 from ert.storage.realization_storage_state import RealizationStorageState
 
@@ -51,13 +51,13 @@ class PlotApiKeyDefinition(NamedTuple):
 
 class PlotApi:
     def __init__(self, ens_path: Path) -> None:
-        self.ens_path = ens_path
+        self.ens_path: Path = ens_path
         self._all_ensembles: list[EnsembleObject] | None = None
         self._timeout = 120
 
     @property
     def api_version(self) -> str:
-        with ErtServer.session(project=self.ens_path) as client:
+        with create_ertserver_client(self.ens_path) as client:
             try:
                 response = client.get("/version", timeout=self._timeout)
                 self._check_response(response)
@@ -83,7 +83,7 @@ class PlotApi:
             return self._all_ensembles
 
         self._all_ensembles = []
-        with ErtServer.session(project=self.ens_path) as client:
+        with create_ertserver_client(self.ens_path) as client:
             try:
                 response = client.get("/experiments", timeout=self._timeout)
                 self._check_response(response)
@@ -139,7 +139,7 @@ class PlotApi:
         all_keys: dict[str, PlotApiKeyDefinition] = {}
         all_params = {}
 
-        with ErtServer.session(project=self.ens_path) as client:
+        with create_ertserver_client(self.ens_path) as client:
             response = client.get("/experiments", timeout=self._timeout)
             self._check_response(response)
 
@@ -166,7 +166,7 @@ class PlotApi:
     def responses_api_key_defs(self) -> list[PlotApiKeyDefinition]:
         key_defs: dict[str, PlotApiKeyDefinition] = {}
 
-        with ErtServer.session(project=self.ens_path) as client:
+        with create_ertserver_client(self.ens_path) as client:
             response = client.get("/experiments", timeout=self._timeout)
             self._check_response(response)
 
@@ -228,7 +228,7 @@ class PlotApi:
         response_key: str,
         filter_on: dict[str, Any] | None = None,
     ) -> pd.DataFrame:
-        with ErtServer.session(project=self.ens_path) as client:
+        with create_ertserver_client(self.ens_path) as client:
             response = client.get(
                 f"/ensembles/{ensemble_id}/responses/{PlotApi.escape(response_key)}",
                 headers={"accept": "application/x-parquet"},
@@ -256,7 +256,7 @@ class PlotApi:
                 return df
 
     def data_for_parameter(self, ensemble_id: str, parameter_key: str) -> pd.DataFrame:
-        with ErtServer.session(project=self.ens_path) as client:
+        with create_ertserver_client(self.ens_path) as client:
             parameter = client.get(
                 f"/ensembles/{ensemble_id}/parameters/{PlotApi.escape(parameter_key)}",
                 headers={"accept": "application/x-parquet"},
@@ -298,7 +298,7 @@ class PlotApi:
             assert key_def.response_metadata is not None
             actual_response_key = key_def.response_metadata.response_key
             filter_on = key_def.filter_on
-            with ErtServer.session(project=self.ens_path) as client:
+            with create_ertserver_client(self.ens_path) as client:
                 response = client.get(
                     f"/ensembles/{ensemble.id}/responses/{PlotApi.escape(actual_response_key)}/observations",
                     timeout=self._timeout,
@@ -386,7 +386,7 @@ class PlotApi:
         if not ensemble:
             return np.array([])
 
-        with ErtServer.session(project=self.ens_path) as client:
+        with create_ertserver_client(self.ens_path) as client:
             response = client.get(
                 f"/ensembles/{ensemble.id}/parameters/{PlotApi.escape(key)}/std_dev",
                 params={"z": z},

ert/gui/tools/plot/plot_widget.py

@@ -153,6 +153,7 @@ class PlotWidget(QWidget):
         # only for histogram plot see _sync_log_checkbox
         self._log_checkbox.setVisible(False)
         self._log_checkbox.setToolTip("Toggle data domain to log scale and back")
+        self._log_checkbox.clicked.connect(self.logLogScaleButtonUsage)
 
         log_checkbox_row = QHBoxLayout()
         log_checkbox_row.addWidget(self._log_checkbox)
@@ -163,7 +164,6 @@ class PlotWidget(QWidget):
         vbox.addSpacing(8)
         self.setLayout(vbox)
 
-        self._negative_values_in_data = False
         self._dirty = True
         self._active = False
         self.resetPlot()
@@ -176,15 +176,11 @@ class PlotWidget(QWidget):
         self._figure.clear()
 
     def _sync_log_checkbox(self) -> None:
-        if (
-            type(self._plotter).__name__
-            in {
-                "HistogramPlot",
-                "DistributionPlot",
-                "GaussianKDEPlot",
-            }
-            and self._negative_values_in_data is False
-        ):
+        if type(self._plotter).__name__ in {
+            "HistogramPlot",
+            "DistributionPlot",
+            "GaussianKDEPlot",
+        }:
             self._log_checkbox.setVisible(True)
         else:
             self._log_checkbox.setVisible(False)
@@ -193,6 +189,10 @@ class PlotWidget(QWidget):
     def name(self) -> str:
         return self._name
 
+    def logLogScaleButtonUsage(self) -> None:
+        logger.info(f"Plotwidget utility used: 'Log scale button' in tab '{self.name}'")
+        self._log_checkbox.clicked.disconnect()  # Log only once
+
     def updatePlot(
         self,
         plot_context: "PlotContext",
@@ -203,11 +203,8 @@ class PlotWidget(QWidget):
     ) -> None:
         self.resetPlot()
         try:
-            self._sync_log_checkbox()
             plot_context.log_scale = (
-                self._log_checkbox.isVisible()
-                and self._log_checkbox.isChecked()
-                and self._negative_values_in_data is False
+                self._log_checkbox.isVisible() and self._log_checkbox.isChecked()
             )
             self._plotter.plot(
                 self._figure,
@@ -218,6 +215,7 @@ class PlotWidget(QWidget):
                 key_def,
             )
             self._canvas.draw()
+            self._sync_log_checkbox()
         except Exception as e:
             exc_type, _, exc_tb = sys.exc_info()
             sys.stderr.write("-" * 80 + "\n")

ert/gui/tools/plot/plot_window.py

@@ -25,7 +25,7 @@ from PyQt6.QtWidgets import (
 from ert.config.field import Field
 from ert.dark_storage.common import get_storage_api_version
 from ert.gui.ertwidgets import CopyButton, showWaitCursorWhileWaiting
-from ert.services._base_service import ServerBootFail
+from ert.services import ServerBootFail
 from ert.utils import log_duration
 
 from .customize import PlotCustomizer
@@ -284,15 +284,6 @@ class PlotWindow(QMainWindow):
                 except BaseException as e:
                     handle_exception(e)
 
-            negative_values_in_data = False
-            if key_def.parameter is not None and key_def.parameter.type == "gen_kw":
-                for data in ensemble_to_data_map.values():
-                    data = data.T
-                    if data.le(0).any().any():
-                        negative_values_in_data = True
-                        break
-
-            plot_widget._negative_values_in_data = negative_values_in_data
             observations = None
             if key_def.observations and selected_ensembles:
                 try:

ert/services/__init__.py

@@ -1,4 +1,8 @@
-from .ert_server import ErtServer
-from .webviz_ert_service import WebvizErt
+from .ert_server import (
+    ErtServer,
+    ErtServerExit,
+    ServerBootFail,
+    create_ertserver_client,
+)
 
-__all__ = ["ErtServer", "WebvizErt"]
+__all__ = ["ErtServer", "ErtServerExit", "ServerBootFail", "create_ertserver_client"]

ert/services/_storage_main.py

@@ -13,6 +13,7 @@ import sys
 import threading
 import time
 import warnings
+from argparse import ArgumentParser
 from base64 import b64encode
 from pathlib import Path
 from typing import Any
@@ -29,12 +30,11 @@ from uvicorn.supervisors import ChangeReload
 
 from ert.logging import STORAGE_LOG_CONFIG
 from ert.plugins import setup_site_logging
-from ert.services._base_service import BaseServiceExit
+from ert.services import ErtServerExit
 from ert.shared import __file__ as ert_shared_path
 from ert.shared import find_available_socket, get_machine_name
-from ert.shared.storage.command import add_parser_options
 from ert.trace import tracer
-from everest.util import makedirs_if_needed
+from ert.utils import makedirs_if_needed
 
 DARK_STORAGE_APP = "ert.dark_storage.app:app"
 
@@ -82,7 +82,7 @@ def _get_host_list() -> list[str]:
 
 
 def _create_connection_info(
-    sock: socket.socket, authtoken: str, cert: str | os.PathLike[str]
+    sock: socket.socket, authtoken: str, cert: str | os.PathLike[str] | Path
 ) -> dict[str, Any]:
     connection_info = {
         "urls": [
@@ -91,7 +91,7 @@ def _create_connection_info(
         "authtoken": authtoken,
         "host": get_machine_name(),
         "port": sock.getsockname()[1],
-        "cert": cert,
+        "cert": str(cert),
         "auth": authtoken,
     }
 
@@ -102,14 +102,17 @@ def _create_connection_info(
     return connection_info
 
 
-def _generate_certificate(cert_folder: str) -> tuple[str, str, bytes]:
+def _generate_certificate(cert_folder: Path) -> tuple[Path, Path, bytes]:
     """Generate a private key and a certificate signed with it
 
     Both the certificate and the key are written to files in the folder given
     by `get_certificate_dir(config)`. The key is encrypted before being
     stored.
-    Returns the path to the certificate file, the path to the key file, and
-    the password used for encrypting the key
+
+    Returns a 3-tuple with
+        * Certificate file path
+        * Key file path
+        * Password used for encrypting the key
     """
     # Generate private key
     key = rsa.generate_private_key(
@@ -150,11 +153,11 @@ def _generate_certificate(cert_folder: str) -> tuple[str, str, bytes]:
 
     # Write certificate and key to disk
     makedirs_if_needed(cert_folder)
-    cert_path = os.path.join(cert_folder, dns_name + ".crt")
-    Path(cert_path).write_bytes(cert.public_bytes(serialization.Encoding.PEM))
-    key_path = os.path.join(cert_folder, dns_name + ".key")
+    cert_path = cert_folder / f"{dns_name}.crt"
+    cert_path.write_bytes(cert.public_bytes(serialization.Encoding.PEM))
+    key_path = cert_folder / f"{dns_name}.key"
     pw = bytes(os.urandom(28))
-    Path(key_path).write_bytes(
+    key_path.write_bytes(
         key.private_bytes(
             encoding=serialization.Encoding.PEM,
             format=serialization.PrivateFormat.TraditionalOpenSSL,
@@ -184,16 +187,16 @@ def run_server(
 
     config_args: dict[str, Any] = {}
     if args.debug or debug:
-        config_args.update(reload=True, reload_dirs=[os.path.dirname(ert_shared_path)])
+        config_args.update(reload=True, reload_dirs=[Path(ert_shared_path).parent])
         os.environ["ERT_STORAGE_DEBUG"] = "1"
 
-    sock = find_available_socket(
+    sock: socket.socket = find_available_socket(
         host=get_machine_name(), port_range=range(51850, 51870 + 1)
     )
 
     # Appropriated from uvicorn.main:run
     os.environ["ERT_STORAGE_NO_TOKEN"] = "1"
-    os.environ["ERT_STORAGE_ENS_PATH"] = os.path.abspath(args.project)
+    os.environ["ERT_STORAGE_ENS_PATH"] = str(args.project.absolute())
     config = (
         # uvicorn.Config() resets the logging config (overriding additional
         # handlers added to loggers like e.g. the ert_azurelogger handler
@@ -253,11 +256,11 @@ def _join_terminate_thread(terminate_on_parent_death_thread: threading.Thread) -
     """Join the terminate thread, handling BaseServiceExit (which is used by Everest)"""
     try:
         terminate_on_parent_death_thread.join()
-    except BaseServiceExit:
+    except ErtServerExit:
         logger = logging.getLogger("ert.shared.storage.info")
         logger.info(
             "Got BaseServiceExit while joining terminate thread, "
-            "as expected from _base_service.py"
+            "as expected from ert_server.py"
         )
 
 
@@ -265,9 +268,7 @@ def main() -> None:
     args = parse_args()
     authentication = _generate_authentication()
     os.environ["ERT_STORAGE_TOKEN"] = authentication
-    cert_path, key_path, key_pw = _generate_certificate(
-        os.path.join(args.project, "cert")
-    )
+    cert_path, key_path, key_pw = _generate_certificate(args.project / "cert")
     config_args: dict[str, Any] = {
         "ssl_keyfile": key_path,
         "ssl_certfile": cert_path,
@@ -283,7 +284,7 @@ def main() -> None:
     warnings.filterwarnings("ignore", category=DeprecationWarning)
 
     if args.debug:
-        config_args.update(reload=True, reload_dirs=[os.path.dirname(ert_shared_path)])
+        config_args.update(reload=True, reload_dirs=[Path(ert_shared_path).parent])
 
     # Need to run uvicorn.Config before entering the ErtPluginContext because
     # uvicorn.Config overrides the configuration of existing loggers, thus removing
@@ -310,12 +311,48 @@ def main() -> None:
             logger.info("Starting dark storage")
             logger.info(f"Started dark storage with parent {args.parent_pid}")
             run_server(args, debug=False, uvicorn_config=uvicorn_config)
-        except (SystemExit, BaseServiceExit):
+        except (SystemExit, ErtServerExit):
             logger.info("Stopping dark storage")
         finally:
             stopped.set()
             _join_terminate_thread(terminate_on_parent_death_thread)
 
 
+def add_parser_options(ap: ArgumentParser) -> None:
+    ap.add_argument(
+        "config",
+        type=str,
+        help=("ERT config file to start the server from "),
+        nargs="?",  # optional
+    )
+    ap.add_argument(
+        "--project",
+        "-p",
+        type=Path,
+        help="Path to directory in which to create storage_server.json",
+        default=Path.cwd(),
+    )
+    ap.add_argument(
+        "--traceparent",
+        type=str,
+        help="Trace parent id to be used by the storage root span",
+        default=None,
+    )
+    ap.add_argument(
+        "--parent_pid",
+        type=int,
+        help="The parent process id",
+        default=os.getppid(),
+    )
+    ap.add_argument(
+        "--host", type=str, default=os.environ.get("ERT_STORAGE_HOST", "127.0.0.1")
+    )
+    ap.add_argument("--logging-config", type=str, default=None)
+    ap.add_argument(
+        "--verbose", action="store_true", help="Show verbose output.", default=False
+    )
+    ap.add_argument("--debug", action="store_true", default=False)
+
+
 if __name__ == "__main__":
     main()