ert 19.0.1__py3-none-any.whl → 20.0.0b1__py3-none-any.whl

everest/bin/kill_script.py

@@ -12,7 +12,7 @@ from functools import partial
 from pathlib import Path
 from typing import Any
 
-from ert.services import ErtServer
+from ert.services import create_ertserver_client
 from everest.bin.utils import setup_logging
 from everest.config import EverestConfig, ServerConfig
 from everest.detached import stop_server, wait_for_server_to_stop
@@ -74,7 +74,7 @@ def _handle_keyboard_interrupt(signal: int, _: Any, after: bool = False) -> None
 
 def kill_everest(options: argparse.Namespace) -> None:
     try:
-        client = ErtServer.session(
+        client = create_ertserver_client(
             Path(ServerConfig.get_session_dir(options.config.output_dir)), timeout=1
         )
         server_context = ServerConfig.get_server_context_from_conn_info(

everest/bin/monitor_script.py

@@ -7,7 +7,7 @@ from functools import partial
 from pathlib import Path
 from textwrap import dedent
 
-from ert.services import ErtServer
+from ert.services import create_ertserver_client
 from ert.storage import ErtStorageException, ExperimentState
 from everest.config import EverestConfig, ServerConfig
 from everest.everest_storage import EverestStorage
@@ -82,7 +82,7 @@ def _build_args_parser() -> argparse.ArgumentParser:
 def monitor_everest(options: argparse.Namespace) -> None:
     config: EverestConfig = options.config
     try:
-        with ErtServer.session(
+        with create_ertserver_client(
             Path(ServerConfig.get_session_dir(config.output_dir)), timeout=1
         ) as client:
             server_context = ServerConfig.get_server_context_from_conn_info(

everest/bin/utils.py

@@ -26,11 +26,12 @@ from ert.ensemble_evaluator import (
 from ert.ensemble_evaluator.event import EndEvent
 from ert.logging import LOGGING_CONFIG
 from ert.plugins.plugin_manager import ErtPluginManager
-from ert.services import ErtServer
+from ert.services import create_ertserver_client
 from ert.storage import (
     ExperimentStatus,
     open_storage,
 )
+from ert.utils import makedirs_if_needed
 from everest.config import EverestConfig
 from everest.config.server_config import ServerConfig
 from everest.detached import (
@@ -40,7 +41,6 @@ from everest.detached import (
     wait_for_server_to_stop,
 )
 from everest.strings import EVEREST, OPT_PROGRESS_ID, SIM_PROGRESS_ID
-from everest.util import makedirs_if_needed
 
 JOB_SUCCESS = "Finished"
 JOB_RUNNING = "Running"
@@ -54,7 +54,7 @@ def cleanup_logging() -> None:
 @contextmanager
 def setup_logging(options: argparse.Namespace) -> Generator[None, None, None]:
     if isinstance(options.config, EverestConfig):
-        makedirs_if_needed(options.config.output_dir, roll_if_exists=False)
+        makedirs_if_needed(Path(options.config.output_dir), roll_if_exists=False)
         log_dir = Path(options.config.output_dir) / "logs"
     else:
         # `everest branch` gives a tuple object here.
@@ -106,7 +106,7 @@ def handle_keyboard_interrupt(signum: int, _: Any, options: argparse.Namespace)
             "The optimization will be stopped and the program will exit..."
         )
         try:
-            client = ErtServer.session(
+            client = create_ertserver_client(
                 Path(ServerConfig.get_session_dir(options.config.output_dir))
             )
             server_context = ServerConfig.get_server_context_from_conn_info(

everest/detached/everserver.py

@@ -12,11 +12,11 @@ import yaml
 from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
 
 from ert.plugins.plugin_manager import ErtPluginManager
-from ert.services import ErtServer
-from ert.services._base_service import BaseServiceExit
+from ert.services import ErtServer, ErtServerExit, create_ertserver_client
 from ert.storage import ExperimentStatus
 from ert.storage.local_experiment import ExperimentState
 from ert.trace import tracer
+from ert.utils import makedirs_if_needed
 from everest.config import ServerConfig
 from everest.strings import (
     DEFAULT_LOGGING_FORMAT,
@@ -25,7 +25,7 @@ from everest.strings import (
     EXPERIMENT_SERVER,
     OPTIMIZATION_LOG_DIR,
 )
-from everest.util import makedirs_if_needed, version_info
+from everest.util import version_info
 
 logger = logging.getLogger(__name__)
 
@@ -34,7 +34,7 @@ def _configure_loggers(
     log_dir: Path, logging_level: int, output_file: str | None
 ) -> None:
     def make_handler_config(path: Path, log_level: int) -> dict[str, Any]:
-        makedirs_if_needed(str(path.parent))
+        makedirs_if_needed(path.parent)
         return {
             "class": "logging.FileHandler",
             "formatter": "default",
@@ -166,7 +166,7 @@ def main() -> None:
                 timeout=240, project=Path(server_path), logging_config=log_file.name
             ) as server:
                 server.fetch_connection_info()
-                with ErtServer.session(project=Path(server_path)) as client:
+                with create_ertserver_client(Path(server_path)) as client:
                     done = False
                     while not done:
                         response = client.get(
@@ -178,7 +178,7 @@ def main() -> None:
                             ExperimentState.running,
                         }
                         time.sleep(0.5)
-        except BaseServiceExit:
+        except ErtServerExit:
             # Server exit, happens on normal shutdown and keyboard interrupt
             logging.getLogger(EVERSERVER).info("Everserver stopped by user")
         except Exception as e:

everest/gui/everest_client.py

@@ -62,12 +62,6 @@ class EverestClient:
     def config(self) -> dict[str, str]:
         return self._http_get(EverEndpoints.config_path).json()
 
-    def get_runtime(self) -> int:
-        if self._start_time is None:
-            self._start_time = int(self._http_get(EverEndpoints.start_time).text)
-
-        return int(time.time()) - self._start_time
-
     @property
     def credentials(self) -> str:
         return b64encode(f"{self._username}:{self._password}".encode()).decode()

everest/gui/main_window.py

@@ -13,7 +13,7 @@ from PyQt6.QtWidgets import (
 from ert.gui.ertnotifier import ErtNotifier
 from ert.gui.simulation.run_dialog import RunDialog
 from ert.plugins import ErtPluginManager
-from ert.services import ErtServer
+from ert.services import create_ertserver_client
 from everest.config import ServerConfig
 from everest.detached import wait_for_server
 from everest.gui.everest_client import EverestClient
@@ -43,7 +43,7 @@ class EverestMainWindow(QMainWindow):
         self.setCentralWidget(self.central_widget)
 
     def run(self) -> None:
-        storage_client = ErtServer.session(
+        storage_client = create_ertserver_client(
             Path(ServerConfig.get_session_dir(self.output_dir))
         )
         wait_for_server(storage_client, 60)

everest/util/__init__.py

@@ -1,6 +1,4 @@
 import logging
-import os
-from datetime import UTC, datetime
 
 from ropt.version import version as ropt_version
 
@@ -8,7 +6,7 @@ try:
     from ert.shared.version import version as ert_version
 except ImportError:
     ert_version = "0.0.0"
-from _ert.utils import file_safe_timestamp
+
 from everest.strings import EVEREST
 
 
@@ -16,14 +14,6 @@ def version_info() -> str:
     return f"everest:{ert_version}, ropt:{ropt_version}, ert:{ert_version}"
 
 
-def makedirs_if_needed(path: str, roll_if_exists: bool = False) -> None:
-    if os.path.isdir(path):
-        if not roll_if_exists:
-            return
-        _roll_dir(path)  # exists and should be rolled
-    os.makedirs(path)
-
-
 def warn_user_that_runpath_is_nonempty() -> None:
     print(
         "Everest is running in an existing runpath.\n\n"
@@ -34,11 +24,3 @@ def warn_user_that_runpath_is_nonempty() -> None:
         "be used if not configured correctly.\n"
     )
     logging.getLogger(EVEREST).warning("Everest is running in an existing runpath")
-
-
-def _roll_dir(old_name: str) -> None:
-    old_name = os.path.realpath(old_name)
-    timestamp = file_safe_timestamp(datetime.now(UTC).isoformat())
-    new_name = f"{old_name}__{timestamp}"
-    os.rename(old_name, new_name)
-    logging.getLogger(EVEREST).info(f"renamed {old_name} to {new_name}")

ert/dark_storage/endpoints/compute/misfits.py

@@ -1,95 +0,0 @@
-import json
-from datetime import datetime
-from typing import Annotated, Any
-from uuid import UUID
-
-import pandas as pd
-from dateutil.parser import parse
-from fastapi import APIRouter, Depends, Query, status
-from fastapi.responses import Response
-
-from ert.dark_storage import exceptions as exc
-from ert.dark_storage.common import get_storage
-from ert.dark_storage.compute.misfits import calculate_signed_chi_squared_misfits
-from ert.dark_storage.endpoints.observations import (
-    _get_observations,
-)
-from ert.dark_storage.endpoints.responses import data_for_response
-from ert.storage import Storage
-
-router = APIRouter(tags=["misfits"])
-DEFAULT_STORAGEREADER = Depends(get_storage)
-
-
-@router.get(
-    "/compute/misfits",
-    responses={
-        status.HTTP_200_OK: {
-            "content": {"text/csv": {}},
-        }
-    },
-)
-async def get_response_misfits(
-    *,
-    storage: Storage = DEFAULT_STORAGEREADER,
-    ensemble_id: UUID,
-    response_name: str,
-    realization_index: int | None = None,
-    summary_misfits: bool = False,
-    filter_on: Annotated[
-        str | None, Query(description="JSON string with filters")
-    ] = None,
-) -> Response:
-    ensemble = storage.get_ensemble(ensemble_id)
-    dataframe = data_for_response(
-        ensemble,
-        response_name,
-        json.loads(filter_on) if filter_on is not None else None,
-    )
-    if realization_index is not None:
-        dataframe = pd.DataFrame(dataframe.loc[realization_index]).T
-
-    response_dict = {}
-    for index, data in dataframe.iterrows():
-        data_df = pd.DataFrame(data).T
-        response_dict[index] = data_df
-
-    experiment = ensemble.experiment
-    response_type = experiment.response_key_to_response_type[response_name]
-    obs_keys = experiment.response_key_to_observation_key[response_type].get(
-        response_name, []
-    )
-    obs = _get_observations(
-        ensemble.experiment,
-        obs_keys,
-        json.loads(filter_on) if filter_on is not None else None,
-    )
-
-    if not obs_keys:
-        raise ValueError(f"No observations for key {response_name}")
-    if not obs:
-        raise ValueError(f"Cant fetch observations for key {response_name}")
-    o = obs[0]
-
-    def parse_index(x: Any) -> int | datetime:
-        try:
-            return int(x)
-        except ValueError:
-            return parse(x)
-
-    observation_df = pd.DataFrame(
-        data={"values": o["values"], "errors": o["errors"]},
-        index=[parse_index(x) for x in o["x_axis"]],
-    )
-    try:
-        result_df = calculate_signed_chi_squared_misfits(
-            response_dict, observation_df, summary_misfits
-        )
-    except Exception as misfits_exc:
-        raise exc.UnprocessableError(
-            f"Unable to compute misfits: {misfits_exc}"
-        ) from misfits_exc
-    return Response(
-        content=result_df.to_csv().encode(),
-        media_type="text/csv",
-    )

ert/services/_base_service.py

@@ -1,387 +0,0 @@
-"""
-This file contains a more generic version of "ert services", and
-is scheduled for removal when WebvizErt is removed.
-"""
-
-from __future__ import annotations
-
-import contextlib
-import io
-import json
-import os
-import signal
-import sys
-import threading
-import types
-from collections.abc import Callable, Mapping, Sequence
-from logging import Logger, getLogger
-from pathlib import Path
-from select import PIPE_BUF, select
-from subprocess import Popen, TimeoutExpired
-from tempfile import NamedTemporaryFile
-from time import sleep
-from types import FrameType
-from typing import TYPE_CHECKING, Any, Generic, Self, TypedDict, TypeVar
-
-if TYPE_CHECKING:
-    pass
-
-T = TypeVar("T", bound="BaseService")
-
-
-class ErtServerConnectionInfo(TypedDict):
-    urls: list[str]
-    authtoken: str
-    host: str
-    port: str
-    cert: str
-    auth: str
-
-
-SERVICE_CONF_PATHS: set[str] = set()
-
-
-class BaseServiceExit(OSError):
-    pass
-
-
-def cleanup_service_files(signum: int, frame: FrameType | None) -> None:
-    for file_path in SERVICE_CONF_PATHS:
-        file = Path(file_path)
-        if file.exists():
-            file.unlink()
-    raise BaseServiceExit(f"Signal {signum} received.")
-
-
-if threading.current_thread() is threading.main_thread():
-    signal.signal(signal.SIGTERM, cleanup_service_files)
-    signal.signal(signal.SIGINT, cleanup_service_files)
-
-
-def local_exec_args(script_args: str | list[str]) -> list[str]:
-    """
-    Convenience function that returns the exec_args for executing a Python
-    script in the directory of '_base_service.py'.
-
-    This is done instead of using 'python -m [module path]' due to the '-m' flag
-    adding the user's current working directory to sys.path. Executing a Python
-    script by itself will add the directory of the script rather than the
-    current working directory, thus we avoid accidentally importing user's
-    directories that just happen to have the same names as the ones we use.
-    """
-    if isinstance(script_args, str):
-        script = script_args
-        rest: list[str] = []
-    else:
-        script = script_args[0]
-        rest = script_args[1:]
-    script = f"_{script}_main.py"
-    return [sys.executable, str(Path(__file__).parent / script), *rest]
-
-
-class _Context(Generic[T]):
-    def __init__(self, service: T) -> None:
-        self._service = service
-
-    def __enter__(self) -> T:
-        return self._service
-
-    def __exit__(
-        self,
-        exc_type: type[BaseException] | None,
-        exc_value: BaseException | None,
-        traceback: types.TracebackType | None,
-    ) -> bool:
-        self._service.shutdown()
-        return exc_type is None
-
-
-class _Proc(threading.Thread):
-    def __init__(
-        self,
-        service_name: str,
-        exec_args: Sequence[str],
-        timeout: int,
-        on_connection_info_received: Callable[
-            [ErtServerConnectionInfo | Exception | None], None
-        ],
-        project: Path,
-    ) -> None:
-        super().__init__()
-
-        self._shutdown = threading.Event()
-
-        self._service_name = service_name
-        self._exec_args = exec_args
-        self._timeout = timeout
-        self._propagate_connection_info_from_childproc = on_connection_info_received
-        self._service_config_path = project / f"{self._service_name}_server.json"
-
-        fd_read, fd_write = os.pipe()
-        self._comm_pipe = os.fdopen(fd_read)
-
-        env = os.environ.copy()
-        env["ERT_COMM_FD"] = str(fd_write)
-
-        SERVICE_CONF_PATHS.add(str(self._service_config_path))
-
-        # The process is waited for in _do_shutdown()
-        self._childproc = Popen(
-            self._exec_args,
-            pass_fds=(fd_write,),
-            env=env,
-            close_fds=True,
-        )
-        os.close(fd_write)
-
-    def run(self) -> None:
-        comm = self._read_connection_info_from_process(self._childproc)
-
-        if comm is None:
-            self._propagate_connection_info_from_childproc(TimeoutError())
-            return  # _read_conn_info() has already cleaned up in this case
-
-        conn_info: ErtServerConnectionInfo | Exception | None = None
-        try:
-            conn_info = json.loads(comm)
-        except json.JSONDecodeError:
-            conn_info = ServerBootFail()
-        except Exception as exc:
-            conn_info = exc
-
-        try:
-            self._propagate_connection_info_from_childproc(conn_info)
-
-            while True:
-                if self._childproc.poll() is not None:
-                    break
-                if self._shutdown.wait(1):
-                    self._do_shutdown()
-                    break
-
-        except Exception as e:
-            print(str(e))
-            self.logger.exception(e)
-
-        finally:
-            self._ensure_connection_info_file_is_deleted()
-
-    def shutdown(self) -> int:
-        """Shutdown the server."""
-        self._shutdown.set()
-        self.join()
-
-        return self._childproc.returncode
-
-    def _read_connection_info_from_process(self, proc: Popen[bytes]) -> str | None:
-        comm_buf = io.StringIO()
-        first_iter = True
-        while first_iter or proc.poll() is None:
-            first_iter = False
-            ready = select([self._comm_pipe], [], [], self._timeout)
-
-            # Timeout reached, exit with a failure
-            if ready == ([], [], []):
-                self._do_shutdown()
-                self._ensure_connection_info_file_is_deleted()
-                return None
-
-            x = self._comm_pipe.read(PIPE_BUF)
-            if not x:  # EOF
-                break
-            comm_buf.write(x)
-        return comm_buf.getvalue()
-
-    def _do_shutdown(self) -> None:
-        if self._childproc is None:
-            return
-        try:
-            self._childproc.terminate()
-            self._childproc.wait(10)  # Give it 10s to shut down cleanly..
-        except TimeoutExpired:
-            try:
-                self._childproc.kill()  # ... then kick it harder...
-                self._childproc.wait(self._timeout)  # ... and wait again
-            except TimeoutExpired:
-                self.logger.error(
-                    f"waiting for child-process exceeded timeout {self._timeout}s"
-                )
-
-    def _ensure_connection_info_file_is_deleted(self) -> None:
-        """
-        Ensure that the JSON connection information file is deleted
-        """
-        with contextlib.suppress(OSError):
-            if self._service_config_path.exists():
-                self._service_config_path.unlink()
-
-    @property
-    def logger(self) -> Logger:
-        return getLogger(f"ert.shared.{self._service_name}")
-
-
-class ServerBootFail(RuntimeError):
-    pass
-
-
-class BaseService:
-    """
-    BaseService provides a block-only-when-needed mechanism for starting and
-    maintaining services as subprocesses.
-
-    This is achieved by using a POSIX communication pipe, over which the service
-    can communicate that it has started. The contents of the communication is
-    also written to a file inside of the ERT storage directory.
-
-    The service itself can implement the other side of the pipe as such::
-
-        import os
-
-        # ... perform initialisation ...
-
-        # BaseService provides this environment variable with the pipe's FD
-        comm_fd = os.environ["ERT_COMM_FD"]
-
-        # Open the pipe with Python's IO classes for ease of use
-        with os.fdopen(comm_fd, "wb") as comm:
-            # Write JSON over the pipe, which will be interpreted by a subclass
-            # of BaseService on ERT's side
-            comm.write('{"some": "json"}')
-
-        # The pipe is flushed and closed here. This tells BaseService that
-        # initialisation is finished and it will try to read the JSON data.
-    """
-
-    _instance: BaseService | None = None
-
-    def __init__(
-        self,
-        exec_args: Sequence[str] = (),
-        timeout: int = 120,
-        conn_info: ErtServerConnectionInfo | Exception | None = None,
-        project: str | None = None,
-    ) -> None:
-        self._exec_args = exec_args
-        self._timeout = timeout
-
-        self._proc: _Proc | None = None
-        self._conn_info: ErtServerConnectionInfo | Exception | None = conn_info
-        self._conn_info_event = threading.Event()
-        self._project = Path(project) if project is not None else Path.cwd()
-
-        # Flag that we have connection information
-        if self._conn_info:
-            self._conn_info_event.set()
-        else:
-            self._proc = _Proc(
-                self.service_name, exec_args, timeout, self.set_conn_info, self._project
-            )
-
-    @classmethod
-    def start_server(cls, *args: Any, **kwargs: Any) -> _Context[Self]:
-        if cls._instance is not None:
-            raise RuntimeError("Server already running")
-        cls._instance = obj = cls(*args, **kwargs)
-        if obj._proc is not None:
-            obj._proc.start()
-        return _Context(obj)
-
-    @classmethod
-    def connect(
-        cls,
-        *,
-        project: os.PathLike[str],
-        timeout: int | None = None,
-    ) -> Self:
-        if cls._instance is not None:
-            cls._instance.wait_until_ready()
-            assert isinstance(cls._instance, cls)
-            return cls._instance
-
-        path = Path(project)
-        name = f"{cls.service_name}_server.json"
-        # Note: If the caller actually pass None, we override that here...
-        if timeout is None:
-            timeout = 240
-        t = -1
-        while t < timeout:
-            if (path / name).exists():
-                with (path / name).open() as f:
-                    return cls((), conn_info=json.load(f), project=str(path))
-
-            sleep(1)
-            t += 1
-
-        raise TimeoutError("Server not started")
-
-    def wait_until_ready(self, timeout: int | None = None) -> bool:
-        if timeout is None:
-            timeout = self._timeout
-
-        if self._conn_info_event.wait(timeout):
-            return not (
-                self._conn_info is None or isinstance(self._conn_info, Exception)
-            )
-        if isinstance(self._conn_info, TimeoutError):
-            self.logger.critical(f"startup exceeded defined timeout {timeout}s")
-        return False  # Timeout reached
-
-    def wait(self) -> None:
-        if self._proc is not None:
-            self._proc.join()
-
-    def set_conn_info(self, info: ErtServerConnectionInfo | Exception | None) -> None:
-        if self._conn_info is not None:
-            raise ValueError("Connection information already set")
-        if info is None:
-            raise ValueError
-        self._conn_info = info
-
-        if self._project is not None:
-            if not Path(self._project).exists():
-                raise RuntimeError(f"No storage exists at : {self._project}")
-            path = f"{self._project}/{self.service_name}_server.json"
-        else:
-            path = f"{self.service_name}_server.json"
-
-        if isinstance(info, Mapping):
-            with NamedTemporaryFile(dir=f"{self._project}", delete=False) as f:
-                f.write(json.dumps(info, indent=4).encode("utf-8"))
-                f.flush()
-                os.rename(f.name, path)
-
-        self._conn_info_event.set()
-
-    def fetch_conn_info(self) -> Mapping[str, Any]:
-        is_ready = self.wait_until_ready(self._timeout)
-        if isinstance(self._conn_info, Exception):
-            raise self._conn_info
-        if not is_ready:
-            raise TimeoutError
-        if self._conn_info is None:
-            raise ValueError("conn_info is None")
-        return self._conn_info
-
-    def shutdown(self) -> int:
-        """Shutdown the server."""
-        if self._proc is None:
-            return -1
-        self.__class__._instance = None
-        proc, self._proc = self._proc, None
-        return proc.shutdown()
-
-    @property
-    def service_name(self) -> str:
-        """
-        Subclass should return the name of the service, eg 'storage' for ERT Storage.
-        Used for identifying the server information JSON file.
-        """
-        raise NotImplementedError
-
-    @property
-    def logger(self) -> Logger:
-        return getLogger(f"ert.shared.{self.service_name}")
-
-    @property
-    def _service_file(self) -> str:
-        return f"{self.service_name}_server.json"