ert 19.0.1__py3-none-any.whl → 20.0.0b1__py3-none-any.whl

ert/scheduler/job.py

@@ -482,6 +482,7 @@ async def log_warnings_from_forward_model(
     real: Realization,
     job_submission_time: float,
     timeout_seconds: int = Job.DEFAULT_FILE_VERIFICATION_TIMEOUT,
+    max_logged_warnings: int = 200,
 ) -> int:
     """Parse all stdout and stderr files from running the forward model
     for anything that looks like a Warning, and log it.
@@ -513,11 +514,19 @@ async def log_warnings_from_forward_model(
         )
 
     async def log_warnings_from_file(
-        file: Path, iens: int, step: ForwardModelStep, step_idx: int, filetype: str
-    ) -> None:
+        file: Path,
+        iens: int,
+        step: ForwardModelStep,
+        step_idx: int,
+        filetype: str,
+        max_warnings_to_log: int,
+    ) -> int:
+        """Returns how many times a warning was logged"""
         captured: list[str] = []
         file_text = await anyio.Path(file).read_text(encoding="utf-8")
         for line in file_text.splitlines():
+            if len(captured) >= max_warnings_to_log:
+                break
             if line_contains_warning(line):
                 captured.append(line[:max_length])
 
@@ -528,6 +537,7 @@ async def log_warnings_from_forward_model(
             )
             warnings.warn(warning_msg, PostSimulationWarning, stacklevel=2)
             logger.warning(warning_msg)
+        return len(captured)
 
     async def wait_for_file(file_path: Path, _timeout: int) -> int:
         if _timeout <= 0:
@@ -546,6 +556,7 @@ async def log_warnings_from_forward_model(
                 break
         return remaining_timeout
 
+    log_count = 0
     with suppress(KeyError):
         runpath = Path(real.run_arg.runpath)
         for step_idx, step in enumerate(real.fm_steps):
@@ -560,9 +571,18 @@ async def log_warnings_from_forward_model(
                     if timeout_seconds <= 0:
                         break
 
-                    await log_warnings_from_file(
-                        std_path, real.iens, step, step_idx, file_type
+                    log_count += await log_warnings_from_file(
+                        std_path,
+                        real.iens,
+                        step,
+                        step_idx,
+                        file_type,
+                        max_logged_warnings - log_count,
                     )
             if timeout_seconds <= 0:
                 break
+    if log_count >= max_logged_warnings:
+        logger.warning(
+            "Reached maximum number of forward model step warnings to extract"
+        )
     return timeout_seconds

ert/services/__init__.py

@@ -1,4 +1,8 @@
-from .ert_server import ErtServer
-from .webviz_ert_service import WebvizErt
+from .ert_server import (
+    ErtServer,
+    ErtServerExit,
+    ServerBootFail,
+    create_ertserver_client,
+)
 
-__all__ = ["ErtServer", "WebvizErt"]
+__all__ = ["ErtServer", "ErtServerExit", "ServerBootFail", "create_ertserver_client"]

ert/services/_storage_main.py

@@ -13,6 +13,7 @@ import sys
 import threading
 import time
 import warnings
+from argparse import ArgumentParser
 from base64 import b64encode
 from pathlib import Path
 from typing import Any
@@ -29,12 +30,11 @@ from uvicorn.supervisors import ChangeReload
 
 from ert.logging import STORAGE_LOG_CONFIG
 from ert.plugins import setup_site_logging
-from ert.services._base_service import BaseServiceExit
+from ert.services import ErtServerExit
 from ert.shared import __file__ as ert_shared_path
 from ert.shared import find_available_socket, get_machine_name
-from ert.shared.storage.command import add_parser_options
 from ert.trace import tracer
-from everest.util import makedirs_if_needed
+from ert.utils import makedirs_if_needed
 
 DARK_STORAGE_APP = "ert.dark_storage.app:app"
 
@@ -82,7 +82,7 @@ def _get_host_list() -> list[str]:
 
 
 def _create_connection_info(
-    sock: socket.socket, authtoken: str, cert: str | os.PathLike[str]
+    sock: socket.socket, authtoken: str, cert: str | os.PathLike[str] | Path
 ) -> dict[str, Any]:
     connection_info = {
         "urls": [
@@ -91,7 +91,7 @@ def _create_connection_info(
         "authtoken": authtoken,
         "host": get_machine_name(),
         "port": sock.getsockname()[1],
-        "cert": cert,
+        "cert": str(cert),
         "auth": authtoken,
     }
 
@@ -102,14 +102,17 @@ def _create_connection_info(
     return connection_info
 
 
-def _generate_certificate(cert_folder: str) -> tuple[str, str, bytes]:
+def _generate_certificate(cert_folder: Path) -> tuple[Path, Path, bytes]:
     """Generate a private key and a certificate signed with it
 
     Both the certificate and the key are written to files in the folder given
     by `get_certificate_dir(config)`. The key is encrypted before being
     stored.
-    Returns the path to the certificate file, the path to the key file, and
-    the password used for encrypting the key
+
+    Returns a 3-tuple with
+        * Certificate file path
+        * Key file path
+        * Password used for encrypting the key
     """
     # Generate private key
     key = rsa.generate_private_key(
@@ -150,11 +153,11 @@ def _generate_certificate(cert_folder: str) -> tuple[str, str, bytes]:
 
     # Write certificate and key to disk
     makedirs_if_needed(cert_folder)
-    cert_path = os.path.join(cert_folder, dns_name + ".crt")
-    Path(cert_path).write_bytes(cert.public_bytes(serialization.Encoding.PEM))
-    key_path = os.path.join(cert_folder, dns_name + ".key")
+    cert_path = cert_folder / f"{dns_name}.crt"
+    cert_path.write_bytes(cert.public_bytes(serialization.Encoding.PEM))
+    key_path = cert_folder / f"{dns_name}.key"
     pw = bytes(os.urandom(28))
-    Path(key_path).write_bytes(
+    key_path.write_bytes(
         key.private_bytes(
             encoding=serialization.Encoding.PEM,
             format=serialization.PrivateFormat.TraditionalOpenSSL,
@@ -184,16 +187,16 @@ def run_server(
 
     config_args: dict[str, Any] = {}
     if args.debug or debug:
-        config_args.update(reload=True, reload_dirs=[os.path.dirname(ert_shared_path)])
+        config_args.update(reload=True, reload_dirs=[Path(ert_shared_path).parent])
         os.environ["ERT_STORAGE_DEBUG"] = "1"
 
-    sock = find_available_socket(
+    sock: socket.socket = find_available_socket(
         host=get_machine_name(), port_range=range(51850, 51870 + 1)
     )
 
     # Appropriated from uvicorn.main:run
     os.environ["ERT_STORAGE_NO_TOKEN"] = "1"
-    os.environ["ERT_STORAGE_ENS_PATH"] = os.path.abspath(args.project)
+    os.environ["ERT_STORAGE_ENS_PATH"] = str(args.project.absolute())
     config = (
         # uvicorn.Config() resets the logging config (overriding additional
         # handlers added to loggers like e.g. the ert_azurelogger handler
@@ -253,11 +256,11 @@ def _join_terminate_thread(terminate_on_parent_death_thread: threading.Thread) -
     """Join the terminate thread, handling BaseServiceExit (which is used by Everest)"""
     try:
         terminate_on_parent_death_thread.join()
-    except BaseServiceExit:
+    except ErtServerExit:
         logger = logging.getLogger("ert.shared.storage.info")
         logger.info(
             "Got BaseServiceExit while joining terminate thread, "
-            "as expected from _base_service.py"
+            "as expected from ert_server.py"
         )
 
 
@@ -265,9 +268,7 @@ def main() -> None:
     args = parse_args()
     authentication = _generate_authentication()
     os.environ["ERT_STORAGE_TOKEN"] = authentication
-    cert_path, key_path, key_pw = _generate_certificate(
-        os.path.join(args.project, "cert")
-    )
+    cert_path, key_path, key_pw = _generate_certificate(args.project / "cert")
     config_args: dict[str, Any] = {
         "ssl_keyfile": key_path,
         "ssl_certfile": cert_path,
@@ -283,7 +284,7 @@ def main() -> None:
     warnings.filterwarnings("ignore", category=DeprecationWarning)
 
     if args.debug:
-        config_args.update(reload=True, reload_dirs=[os.path.dirname(ert_shared_path)])
+        config_args.update(reload=True, reload_dirs=[Path(ert_shared_path).parent])
 
     # Need to run uvicorn.Config before entering the ErtPluginContext because
     # uvicorn.Config overrides the configuration of existing loggers, thus removing
@@ -310,12 +311,48 @@ def main() -> None:
             logger.info("Starting dark storage")
             logger.info(f"Started dark storage with parent {args.parent_pid}")
             run_server(args, debug=False, uvicorn_config=uvicorn_config)
-        except (SystemExit, BaseServiceExit):
+        except (SystemExit, ErtServerExit):
             logger.info("Stopping dark storage")
         finally:
             stopped.set()
             _join_terminate_thread(terminate_on_parent_death_thread)
 
 
+def add_parser_options(ap: ArgumentParser) -> None:
+    ap.add_argument(
+        "config",
+        type=str,
+        help=("ERT config file to start the server from "),
+        nargs="?",  # optional
+    )
+    ap.add_argument(
+        "--project",
+        "-p",
+        type=Path,
+        help="Path to directory in which to create storage_server.json",
+        default=Path.cwd(),
+    )
+    ap.add_argument(
+        "--traceparent",
+        type=str,
+        help="Trace parent id to be used by the storage root span",
+        default=None,
+    )
+    ap.add_argument(
+        "--parent_pid",
+        type=int,
+        help="The parent process id",
+        default=os.getppid(),
+    )
+    ap.add_argument(
+        "--host", type=str, default=os.environ.get("ERT_STORAGE_HOST", "127.0.0.1")
+    )
+    ap.add_argument("--logging-config", type=str, default=None)
+    ap.add_argument(
+        "--verbose", action="store_true", help="Show verbose output.", default=False
+    )
+    ap.add_argument("--debug", action="store_true", default=False)
+
+
 if __name__ == "__main__":
     main()

ert/services/ert_server.py

@@ -1,23 +1,187 @@
 from __future__ import annotations
 
+import contextlib
+import io
 import json
 import logging
 import os
+import signal
 import sys
 import threading
 import types
-from collections.abc import Mapping
+from collections.abc import Callable, Mapping, Sequence
 from pathlib import Path
+from select import PIPE_BUF, select
+from subprocess import Popen, TimeoutExpired
 from tempfile import NamedTemporaryFile
 from time import sleep
-from typing import Any, cast
+from typing import Any, TypedDict, cast
 
 import requests
 
 from ert.dark_storage.client import Client, ErtClientConnectionInfo
-from ert.services._base_service import ErtServerConnectionInfo, _Proc
 from ert.trace import get_traceparent
 
+SERVICE_CONF_PATHS: set[str] = set()
+
+
+class ErtServerConnectionInfo(TypedDict):
+    urls: list[str]
+    authtoken: str
+    host: str
+    port: str
+    cert: str
+    auth: str
+
+
+class ErtServerExit(OSError):
+    pass
+
+
+def cleanup_service_files(signum: int, frame: types.FrameType | None) -> None:
+    for file_path in SERVICE_CONF_PATHS:
+        file = Path(file_path)
+        if file.exists():
+            file.unlink()
+    raise ErtServerExit(f"Signal {signum} received.")
+
+
+if threading.current_thread() is threading.main_thread():
+    signal.signal(signal.SIGTERM, cleanup_service_files)
+    signal.signal(signal.SIGINT, cleanup_service_files)
+
+
+class ServerBootFail(RuntimeError):
+    pass
+
+
+class _Proc(threading.Thread):
+    def __init__(
+        self,
+        service_name: str,
+        exec_args: Sequence[str],
+        timeout: int,
+        on_connection_info_received: Callable[
+            [ErtServerConnectionInfo | Exception | None], None
+        ],
+        project: Path,
+    ) -> None:
+        super().__init__()
+
+        self._shutdown = threading.Event()
+
+        self._service_name = service_name
+        self._exec_args = exec_args
+        self._timeout = timeout
+        self._propagate_connection_info_from_childproc = on_connection_info_received
+        self._service_config_path = project / f"{self._service_name}_server.json"
+
+        fd_read, fd_write = os.pipe()
+        self._comm_pipe = os.fdopen(fd_read)
+
+        env = os.environ.copy()
+        env["ERT_COMM_FD"] = str(fd_write)
+
+        SERVICE_CONF_PATHS.add(str(self._service_config_path))
+
+        # The process is waited for in _do_shutdown()
+        self._childproc = Popen(
+            self._exec_args,
+            pass_fds=(fd_write,),
+            env=env,
+            close_fds=True,
+        )
+        os.close(fd_write)
+
+    def run(self) -> None:
+        comm = self._read_connection_info_from_process(self._childproc)
+
+        if comm is None:
+            self._propagate_connection_info_from_childproc(TimeoutError())
+            return  # _read_conn_info() has already cleaned up in this case
+
+        conn_info: ErtServerConnectionInfo | Exception | None = None
+        try:
+            conn_info = json.loads(comm)
+        except json.JSONDecodeError:
+            conn_info = ServerBootFail()
+        except Exception as exc:
+            conn_info = exc
+
+        try:
+            self._propagate_connection_info_from_childproc(conn_info)
+
+            while True:
+                if self._childproc.poll() is not None:
+                    break
+                if self._shutdown.wait(1):
+                    self._do_shutdown()
+                    break
+
+        except Exception as e:
+            print(str(e))
+            self.logger.exception(e)
+
+        finally:
+            self._ensure_connection_info_file_is_deleted()
+
+    def shutdown(self) -> int:
+        """Shutdown the server."""
+        self._shutdown.set()
+        self.join()
+
+        return self._childproc.returncode
+
+    def _read_connection_info_from_process(self, proc: Popen[bytes]) -> str | None:
+        comm_buf = io.StringIO()
+        first_iter = True
+        while first_iter or proc.poll() is None:
+            first_iter = False
+            ready = select([self._comm_pipe], [], [], self._timeout)
+
+            # Timeout reached, exit with a failure
+            if ready == ([], [], []):
+                self._do_shutdown()
+                self._ensure_connection_info_file_is_deleted()
+                return None
+
+            x = self._comm_pipe.read(PIPE_BUF)
+            if not x:  # EOF
+                break
+            comm_buf.write(x)
+        return comm_buf.getvalue()
+
+    def _do_shutdown(self) -> None:
+        if self._childproc is None:
+            return
+        try:
+            self._childproc.terminate()
+            self._childproc.wait(10)  # Give it 10s to shut down cleanly..
+        except TimeoutExpired:
+            try:
+                self._childproc.kill()  # ... then kick it harder...
+                self._childproc.wait(self._timeout)  # ... and wait again
+            except TimeoutExpired:
+                self.logger.error(
+                    f"waiting for child-process exceeded timeout {self._timeout}s"
+                )
+
+    def _ensure_connection_info_file_is_deleted(self) -> None:
+        """
+        Ensure that the JSON connection information file is deleted
+        """
+        with contextlib.suppress(OSError):
+            if self._service_config_path.exists():
+                self._service_config_path.unlink()
+
+    @property
+    def logger(self) -> logging.Logger:
+        return logging.getLogger("ert.shared.storage")
+
+
+_ERT_SERVER_CONNECTION_INFO_FILE = "storage_server.json"
+_ERT_SERVER_EXECUTABLE_FILE = str(Path(__file__).parent / "_storage_main.py")
+
 
 class ErtServerContext:
     def __init__(self, service: ErtServer) -> None:
@@ -70,7 +234,7 @@ class ErtServer:
 
         run_storage_main_cmd = [
             sys.executable,
-            str(Path(__file__).parent / "_storage_main.py"),
+            _ERT_SERVER_EXECUTABLE_FILE,
             "--project",
             storage_path,
         ]
@@ -91,7 +255,7 @@ class ErtServer:
         self._thread_that_starts_server_process = _Proc(
             service_name="storage",
             exec_args=run_storage_main_cmd,
-            timeout=120,
+            timeout=timeout,
             on_connection_info_received=self.on_connection_info_received_from_server_process,
             project=Path(self._storage_path),
         )
@@ -167,21 +331,6 @@ class ErtServer:
             "None of the URLs provided for the ert storage server worked."
         )
 
-    @classmethod
-    def session(cls, project: os.PathLike[str], timeout: int | None = None) -> Client:
-        """
-        Start a HTTP transaction with the server
-        """
-        inst = cls.connect(timeout=timeout, project=project)
-        info = inst.fetch_connection_info()
-        return Client(
-            conn_info=ErtClientConnectionInfo(
-                base_url=inst.fetch_url(),
-                auth_token=inst.fetch_auth()[1],
-                cert=info["cert"],
-            )
-        )
-
     @property
     def logger(self) -> logging.Logger:
         return logging.getLogger("ert.shared.storage")
@@ -218,12 +367,12 @@ class ErtServer:
                 timeout = 240
             t = -1
             while t < timeout:
-                storage_server_path = path / "storage_server.json"
+                storage_server_path = path / _ERT_SERVER_CONNECTION_INFO_FILE
                 if (
                     storage_server_path.exists()
                     and storage_server_path.stat().st_size > 0
                 ):
-                    with (path / "storage_server.json").open() as f:
+                    with (path / _ERT_SERVER_CONNECTION_INFO_FILE).open() as f:
                         storage_server_content = json.load(f)
 
                     return ErtServer(
@@ -277,9 +426,9 @@ class ErtServer:
         if self._storage_path is not None:
             if not Path(self._storage_path).exists():
                 raise RuntimeError(f"No storage exists at : {self._storage_path}")
-            path = f"{self._storage_path}/storage_server.json"
+            path = f"{self._storage_path}/{_ERT_SERVER_CONNECTION_INFO_FILE}"
         else:
-            path = "storage_server.json"
+            path = _ERT_SERVER_CONNECTION_INFO_FILE
 
         if isinstance(info, Mapping):
             with NamedTemporaryFile(dir=f"{self._storage_path}", delete=False) as f:
@@ -315,3 +464,16 @@ class ErtServer:
     def wait(self) -> None:
         if self._thread_that_starts_server_process is not None:
             self._thread_that_starts_server_process.join()
+
+
+def create_ertserver_client(project: Path, timeout: int | None = None) -> Client:
+    """Read connection info from file in path and create HTTP client."""
+    connection = ErtServer.connect(timeout=timeout, project=project)
+    info = connection.fetch_connection_info()
+    return Client(
+        conn_info=ErtClientConnectionInfo(
+            base_url=connection.fetch_url(),
+            auth_token=connection.fetch_auth()[1],
+            cert=info["cert"],
+        )
+    )

ert/shared/version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '19.0.1'
-__version_tuple__ = version_tuple = (19, 0, 1)
+__version__ = version = '20.0.0b1'
+__version_tuple__ = version_tuple = (20, 0, 0, 'b1')
 
-__commit_id__ = commit_id = 'gd8015f3be'
+__commit_id__ = commit_id = 'g4156859a8'