ert 19.0.0rc1__py3-none-any.whl → 19.0.0rc3__py3-none-any.whl

ert/__main__.py

@@ -8,12 +8,10 @@ import multiprocessing
 import os
 import re
 import resource
-import shutil
 import sys
 import warnings
 from argparse import ArgumentParser, ArgumentTypeError
 from collections.abc import Sequence
-from datetime import datetime
 from pathlib import Path
 from typing import Any
 from uuid import UUID
@@ -26,9 +24,6 @@ from _ert.threading import set_signal_handler
 from ert.base_model_context import use_runtime_plugins
 from ert.cli.main import ErtCliError, run_cli
 from ert.config import ConfigValidationError, ErtConfig, lint_file
-from ert.config.observation_config_migrations import (
-    remove_refcase_and_time_map_dependence_from_obs_config,
-)
 from ert.logging import LOGGING_CONFIG
 from ert.mode_definitions import (
     ENIF_MODE,
@@ -41,9 +36,9 @@ from ert.mode_definitions import (
 from ert.namespace import Namespace
 from ert.plugins import ErtRuntimePlugins, get_site_plugins, setup_site_logging
 from ert.run_models.multiple_data_assimilation import MultipleDataAssimilationConfig
-from ert.services import ErtServer
-from ert.services._storage_main import add_parser_options as ert_api_add_parser_options
+from ert.services import ErtServer, WebvizErt
 from ert.shared.status.utils import get_ert_memory_usage
+from ert.shared.storage.command import add_parser_options as ert_api_add_parser_options
 from ert.storage import ErtStorageException, ErtStoragePermissionError
 from ert.trace import trace, tracer
 from ert.validation import (
@@ -58,66 +53,6 @@ from ert.validation import (
 logger = logging.getLogger(__name__)
 
 
-def run_convert_observations(
-    args: Namespace, _: ErtRuntimePlugins | None = None
-) -> None:
-    changes = remove_refcase_and_time_map_dependence_from_obs_config(args.config)
-
-    if changes is None or changes.is_empty():
-        logger.info("convert_observations did not make any changes")
-        print(
-            "No observations dependent on TIME_MAP / REFCASE found, you can "
-            "safely remove TIME_MAP / REFCASE and the "
-            "corresponding files from ERT config."
-        )
-        return
-
-    obs_config_to_edit_path = changes.obs_config_path + ".updated"
-    print(
-        f"Making copy of obs config "
-        f"@ {changes.obs_config_path} -> {obs_config_to_edit_path}"
-    )
-
-    shutil.copy(changes.obs_config_path, obs_config_to_edit_path)
-    print(f"Applying change to obs config @ {obs_config_to_edit_path}...")
-    changes.apply_to_file(Path(obs_config_to_edit_path))
-    convert_observations_trace = ""
-    for history_change in changes.history_changes:
-        convert_observations_trace += (
-            f"History obs {history_change.source_observation.name} "
-            f"-> {len(history_change.summary_obs_declarations)} summary observations\n"
-        )
-
-    for gen_obs_change in changes.general_obs_changes:
-        convert_observations_trace += (
-            f"General obs {gen_obs_change.source_observation.name}, changing "
-            f"DATE {gen_obs_change.source_observation.date} "
-            f"to RESTART={gen_obs_change.restart}\n"
-        )
-    for summary_change in changes.summary_obs_changes:
-        convert_observations_trace += (
-            f"Summary obs {summary_change.source_observation.name}, changing "
-            f"RESTART {summary_change.source_observation.restart} "
-            f"to DATE={summary_change.date}\n"
-        )
-
-    logger.info(f"convert_observations trace: \n {convert_observations_trace}")
-    print(convert_observations_trace)
-
-    os.rename(
-        changes.obs_config_path,
-        f"{changes.obs_config_path}-{datetime.now().strftime('%Y-%m-%d-%H-%M-%S')}.old",
-    )
-    os.rename(obs_config_to_edit_path, changes.obs_config_path)
-    msg = (
-        f"Observation changes applied to {changes.obs_config_path}. The old "
-        f"observations file is now at {changes.obs_config_path}.old and can be "
-        f"safely deleted if the new one works."
-    )
-    print(msg)
-    logger.info(msg)
-
-
 def run_ert_storage(args: Namespace, _: ErtRuntimePlugins | None = None) -> None:
     with ErtServer.start_server(
         verbose=True,
@@ -128,24 +63,67 @@ def run_ert_storage(args: Namespace, _: ErtRuntimePlugins | None = None) -> None
 
 
 def run_webviz_ert(args: Namespace, _: ErtRuntimePlugins | None = None) -> None:
+    try:
+        import webviz_ert  # type: ignore  # noqa
+    except ImportError as err:
+        raise ValueError(
+            "Running `ert vis` requires that webviz_ert is installed"
+        ) from err
+
+    kwargs: dict[str, Any] = {"verbose": args.verbose}
+    ert_config = ErtConfig.with_plugins(get_site_plugins()).from_file(args.config)
+
+    os.chdir(ert_config.config_path)
+    ens_path = ert_config.ens_path
+
+    # Changing current working directory means we need to
+    # only use the base name of the config file path
+    kwargs["ert_config"] = os.path.basename(args.config)
+    kwargs["project"] = os.path.abspath(ens_path)
+
     yellow = "\x1b[33m"
     green = "\x1b[32m"
     bold = "\x1b[1m"
     reset = "\x1b[0m"
 
-    print(
-        f"""
+    try:
+        with ErtServer.init_service(project=Path(ens_path).absolute()) as storage:
+            storage.wait_until_ready()
+            print(
+                f"""
     ---------------------------------------------------------------
 
-    {yellow}{bold}Webviz-ERT is removed.
+    {yellow}{bold}Webviz-ERT is deprecated and will be removed in the near future{reset}
 
     {green}{bold}Plotting capabilities provided by Webviz-ERT are now available
     using the ERT plotter{reset}
 
+    ---------------------------------------------------------------
+
+    Starting up Webviz-ERT. This might take more than a minute.
+
     ---------------------------------------------------------------
     """
-    )
-    logger.info("Show Webviz-ert removal warning")
+            )
+            logger.info("Show Webviz-ert deprecation warning")
+            webviz_kwargs = {
+                "experimental_mode": args.experimental_mode,
+                "verbose": args.verbose,
+                "title": kwargs.get("ert_config", "ERT - Visualization tool"),
+                "project": kwargs.get("project", os.getcwd()),
+            }
+            with WebvizErt.start_server(**webviz_kwargs) as webviz_ert_server:
+                webviz_ert_server.wait()
+    except PermissionError as pe:
+        print(f"Error: {pe}", file=sys.stderr)
+        print(
+            "Cannot start or connect to storage service due to permission issues.",
+            file=sys.stderr,
+        )
+        print(
+            "This is most likely due to another user starting ERT using this storage",
+            file=sys.stderr,
+        )
 
 
 def strip_error_message_and_raise_exception(validated: ValidationStatus) -> None:
@@ -339,6 +317,19 @@ def get_ert_parser(parser: ArgumentParser | None = None) -> ArgumentParser:
     ert_api_parser.set_defaults(func=run_ert_storage)
     ert_api_add_parser_options(ert_api_parser)
 
+    ert_vis_parser = subparsers.add_parser(
+        "vis",
+        description="Launch webviz-driven visualization tool.",
+    )
+    ert_vis_parser.set_defaults(func=run_webviz_ert)
+    ert_vis_parser.add_argument("--name", "-n", type=str, default="Webviz-ERT")
+    ert_vis_parser.add_argument(
+        "--experimental-mode",
+        action="store_true",
+        help="Feature flag for enabling experimental plugins",
+    )
+    ert_api_add_parser_options(ert_vis_parser)  # ert vis shares args with ert api
+
     # test_run_parser
     test_run_description = f"Run '{TEST_RUN_MODE}' in cli"
     test_run_parser = subparsers.add_parser(
@@ -573,28 +564,6 @@ def get_ert_parser(parser: ArgumentParser | None = None) -> ArgumentParser:
         "--ensemble", help="Which ensemble to use", default=None
     )
 
-    # convert_observations_parser
-    convert_obs_parser = subparsers.add_parser(
-        "convert_observations",
-        help=(
-            "Convert HISTORY_OBSERVATION to SUMMARY_OBSERVATION and "
-            "remove REFCASE and TIME_MAP from ERT config."
-        ),
-        description=(
-            "Convert HISTORY_OBSERVATION to SUMMARY_OBSERVATION, "
-            "and embed REFCASE and TIME_MAP into observations"
-        ),
-    )
-    convert_obs_parser.set_defaults(func=run_convert_observations)
-    convert_obs_parser.add_argument(
-        "config",
-        type=valid_file,
-        help="Path to ERT config file",
-    )
-    convert_obs_parser.add_argument(
-        "--verbose", action="store_true", help="Show verbose output.", default=False
-    )
-
     # Common arguments/defaults for all non-gui modes
     for cli_parser in [
         test_run_parser,

ert/analysis/_es_update.py

@@ -323,13 +323,16 @@ def analysis_ES(
         logger.info(log_msg)
         progress_callback(AnalysisStatusEvent(msg=log_msg))
 
-        if (param_count := (~non_zero_variance_mask).sum()) > 0:
-            log_msg = (
-                f"There are {param_count} parameters with 0 variance "
-                f"that will not be updated."
-            )
-            logger.info(log_msg)
-            progress_callback(AnalysisStatusEvent(msg=log_msg))
+        log_msg = f"There are {num_obs} responses and {ensemble_size} realizations."
+        logger.info(log_msg)
+        progress_callback(AnalysisStatusEvent(msg=log_msg))
+
+        log_msg = (
+            f"There are {(~non_zero_variance_mask).sum()} parameters with 0 variance "
+            f"that will not be updated."
+        )
+        logger.info(log_msg)
+        progress_callback(AnalysisStatusEvent(msg=log_msg))
 
         if module.localization:
             config_node = source_ensemble.experiment.parameter_configuration[
@@ -379,16 +382,16 @@ def analysis_ES(
             )
 
         else:
-            log_msg = f"There are {num_obs} responses and {ensemble_size} realizations."
-            logger.info(log_msg)
-            progress_callback(AnalysisStatusEvent(msg=log_msg))
-
             # In-place multiplication is not yet supported, therefore avoiding @=
             param_ensemble_array[non_zero_variance_mask] = param_ensemble_array[  # noqa: PLR6104
                 non_zero_variance_mask
             ] @ T.astype(param_ensemble_array.dtype)
 
+        log_msg = f"Storing data for {param_group}.."
+        logger.info(log_msg)
+        progress_callback(AnalysisStatusEvent(msg=log_msg))
         start = time.time()
+
         target_ensemble.save_parameters_numpy(
             param_ensemble_array, param_group, iens_active_index
         )

ert/config/_create_observation_dataframes.py

@@ -2,19 +2,21 @@ from __future__ import annotations
 
 from collections import defaultdict
 from collections.abc import Sequence
-from datetime import datetime
-from typing import assert_never
+from datetime import datetime, timedelta
+from typing import TYPE_CHECKING, Any, assert_never
 
 import numpy as np
 import polars as pl
-from numpy import typing as npt
+from resfo_utilities import history_key
 
 from ert.validation import rangestring_to_list
 
 from ._observations import (
     ErrorModes,
     GeneralObservation,
+    HistoryObservation,
     Observation,
+    ObservationDate,
     ObservationError,
     RFTObservation,
     SummaryObservation,
@@ -23,31 +25,59 @@ from .gen_data_config import GenDataConfig
 from .parsing import (
     ConfigWarning,
     ErrorInfo,
+    HistorySource,
     ObservationConfigError,
 )
+from .refcase import Refcase
 from .rft_config import RFTConfig
 
+if TYPE_CHECKING:
+    import numpy.typing as npt
+
+
+DEFAULT_TIME_DELTA = timedelta(seconds=30)
 DEFAULT_LOCATION_RANGE_M = 3000
 
 
 def create_observation_dataframes(
     observations: Sequence[Observation],
+    refcase: Refcase | None,
     gen_data_config: GenDataConfig | None,
     rft_config: RFTConfig | None,
+    time_map: list[datetime] | None,
+    history: HistorySource,
 ) -> dict[str, pl.DataFrame]:
     if not observations:
         return {}
+    obs_time_list: list[datetime] = []
+    if refcase is not None:
+        obs_time_list = refcase.all_dates
+    elif time_map is not None:
+        obs_time_list = time_map
 
+    time_len = len(obs_time_list)
     config_errors: list[ErrorInfo] = []
     grouped: dict[str, list[pl.DataFrame]] = defaultdict(list)
     for obs in observations:
         try:
             match obs:
+                case HistoryObservation():
+                    grouped["summary"].append(
+                        _handle_history_observation(
+                            refcase,
+                            obs,
+                            obs.name,
+                            history,
+                            time_len,
+                        )
+                    )
                 case SummaryObservation():
                     grouped["summary"].append(
                         _handle_summary_observation(
                             obs,
                             obs.name,
+                            obs_time_list,
+                            bool(refcase),
                         )
                     )
                 case GeneralObservation():
@@ -56,6 +86,8 @@ def create_observation_dataframes(
                             gen_data_config,
                             obs,
                             obs.name,
+                            obs_time_list,
+                            bool(refcase),
                         )
                     )
                 case RFTObservation():
@@ -105,6 +137,92 @@ def _handle_error_mode(
             assert_never(default)
 
 
+def _handle_history_observation(
+    refcase: Refcase | None,
+    history_observation: HistoryObservation,
+    summary_key: str,
+    history_type: HistorySource,
+    time_len: int,
+) -> pl.DataFrame:
+    if refcase is None:
+        raise ObservationConfigError.with_context(
+            "REFCASE is required for HISTORY_OBSERVATION", summary_key
+        )
+
+    if history_type == HistorySource.REFCASE_HISTORY:
+        local_key = history_key(summary_key)
+    else:
+        local_key = summary_key
+    if local_key not in refcase.keys:
+        raise ObservationConfigError.with_context(
+            f"Key {local_key!r} is not present in refcase", summary_key
+        )
+    values = refcase.values[refcase.keys.index(local_key)]
+    std_dev = _handle_error_mode(values, history_observation)
+    for segment in history_observation.segments:
+        start = segment.start
+        stop = segment.stop
+        if start < 0:
+            ConfigWarning.warn(
+                f"Segment {segment.name} out of bounds."
+                " Truncating start of segment to 0.",
+                segment.name,
+            )
+            start = 0
+        if stop >= time_len:
+            ConfigWarning.warn(
+                f"Segment {segment.name} out of bounds. Truncating"
+                f" end of segment to {time_len - 1}.",
+                segment.name,
+            )
+            stop = time_len - 1
+        if start > stop:
+            ConfigWarning.warn(
+                f"Segment {segment.name} start after stop. Truncating"
+                f" end of segment to {start}.",
+                segment.name,
+            )
+            stop = start
+        if np.size(std_dev[start:stop]) == 0:
+            ConfigWarning.warn(
+                f"Segment {segment.name} does not"
+                " contain any time steps. The interval "
+                f"[{start}, {stop}) does not intersect with steps in the"
+                "time map.",
+                segment.name,
+            )
+        std_dev[start:stop] = _handle_error_mode(values[start:stop], segment)
+    dates_series = pl.Series(refcase.dates).dt.cast_time_unit("ms")
+    if (std_dev <= 0).any():
+        raise ObservationConfigError.with_context(
+            "Observation uncertainty must be strictly > 0", summary_key
+        ) from None
+
+    return pl.DataFrame(
+        {
+            "response_key": summary_key,
+            "observation_key": summary_key,
+            "time": dates_series,
+            "observations": pl.Series(values, dtype=pl.Float32),
+            "std": pl.Series(std_dev, dtype=pl.Float32),
+        }
+    )
+
+
+def _get_time(
+    date_dict: ObservationDate, start_time: datetime, context: Any = None
+) -> tuple[datetime, str]:
+    if date_dict.date is not None:
+        return _parse_date(date_dict.date), f"DATE={date_dict.date}"
+    if date_dict.days is not None:
+        days = date_dict.days
+        return start_time + timedelta(days=days), f"DAYS={days}"
+    if date_dict.hours is not None:
+        hours = date_dict.hours
+        return start_time + timedelta(hours=hours), f"HOURS={hours}"
+    raise ObservationConfigError.with_context("Missing time specifier", context=context)
+
+
 def _parse_date(date_str: str) -> datetime:
     try:
         return datetime.fromisoformat(date_str)
@@ -125,50 +243,171 @@ def _parse_date(date_str: str) -> datetime:
             return date
 
 
+def _find_nearest(
+    time_map: list[datetime],
+    time: datetime,
+    threshold: timedelta = DEFAULT_TIME_DELTA,
+) -> int:
+    nearest_index = -1
+    nearest_diff = None
+    for i, t in enumerate(time_map):
+        diff = abs(time - t)
+        if diff < threshold and (nearest_diff is None or nearest_diff > diff):
+            nearest_diff = diff
+            nearest_index = i
+    if nearest_diff is None:
+        raise IndexError(f"{time} is not in the time map")
+    return nearest_index
+
+
+def _get_restart(
+    date_dict: ObservationDate,
+    obs_name: str,
+    time_map: list[datetime],
+    has_refcase: bool,
+) -> int:
+    if date_dict.restart is not None:
+        return date_dict.restart
+    if not time_map:
+        raise ObservationConfigError.with_context(
+            f"Missing REFCASE or TIME_MAP for observations: {obs_name}",
+            obs_name,
+        )
+
+    time, date_str = _get_time(date_dict, time_map[0], context=obs_name)
+
+    try:
+        return _find_nearest(time_map, time)
+    except IndexError as err:
+        raise ObservationConfigError.with_context(
+            f"Could not find {time} ({date_str}) in "
+            f"the time map for observations {obs_name}. "
+            + (
+                "The time map is set from the REFCASE keyword. Either "
+                "the REFCASE has an incorrect/missing date, or the observation "
+                "is given an incorrect date.)"
+                if has_refcase
+                else "(The time map is set from the TIME_MAP "
+                "keyword. Either the time map file has an "
+                "incorrect/missing date, or the observation is given an "
+                "incorrect date."
+            ),
+            obs_name,
+        ) from err
+
+
 def _has_localization(summary_dict: SummaryObservation) -> bool:
-    return summary_dict.location_x is not None and summary_dict.location_y is not None
+    return any(
+        [
+            summary_dict.location_x is not None,
+            summary_dict.location_y is not None,
+            summary_dict.location_range is not None,
+        ]
+    )
+
+
+def _validate_localization_values(summary_dict: SummaryObservation) -> None:
+    """The user must provide LOCATION_X and LOCATION_Y to use localization, while
+    unprovided LOCATION_RANGE should default to some value.
+
+    This method assumes the summary dict contains at least one LOCATION key.
+    """
+    if summary_dict.location_x is None or summary_dict.location_y is None:
+        loc_values = {
+            "LOCATION_X": summary_dict.location_x,
+            "LOCATION_Y": summary_dict.location_y,
+            "LOCATION_RANGE": summary_dict.location_range,
+        }
+        provided_loc_values = {k: v for k, v in loc_values.items() if v is not None}
+
+        provided_loc_values_string = ", ".join(
+            key.upper() for key in provided_loc_values
+        )
+        raise ObservationConfigError.with_context(
+            f"Localization for observation {summary_dict.name} is misconfigured.\n"
+            f"Only {provided_loc_values_string} were provided. To enable "
+            f"localization for an observation, ensure that both LOCATION_X and "
+            f"LOCATION_Y are defined - or remove LOCATION keywords to disable "
+            f"localization.",
+            summary_dict,
+        )
 
 
 def _handle_summary_observation(
     summary_dict: SummaryObservation,
     obs_key: str,
+    time_map: list[datetime],
+    has_refcase: bool,
 ) -> pl.DataFrame:
     summary_key = summary_dict.key
     value = summary_dict.value
     std_dev = float(_handle_error_mode(np.array(value), summary_dict))
-    date = _parse_date(summary_dict.date)
 
+    if summary_dict.restart and not (time_map or has_refcase):
+        raise ObservationConfigError.with_context(
+            "Keyword 'RESTART' requires either TIME_MAP or REFCASE", context=obs_key
+        )
+
+    if summary_dict.date is not None and not time_map:
+        # We special case when the user has provided date in SUMMARY_OBS
+        # and not REFCASE or time_map so that we don't change current behavior.
+        date = _parse_date(summary_dict.date)
+        restart = None
+    else:
+        restart = _get_restart(summary_dict, obs_key, time_map, has_refcase)
+        date = time_map[restart]
+
+    if restart == 0:
+        raise ObservationConfigError.with_context(
+            "It is unfortunately not possible to use summary "
+            "observations from the start of the simulation. "
+            f"Problem with observation {obs_key}"
+            f"{' at ' + str(_get_time(summary_dict, time_map[0], obs_key)) if summary_dict.restart is None else ''}",  # noqa: E501
+            obs_key,
+        )
     if std_dev <= 0:
         raise ObservationConfigError.with_context(
             "Observation uncertainty must be strictly > 0", summary_key
         ) from None
 
-    location_range = (
-        summary_dict.location_range or DEFAULT_LOCATION_RANGE_M
-        if _has_localization(summary_dict)
-        else None
-    )
+    data_dict = {
+        "response_key": [summary_key],
+        "observation_key": [obs_key],
+        "time": pl.Series([date]).dt.cast_time_unit("ms"),
+        "observations": pl.Series([value], dtype=pl.Float32),
+        "std": pl.Series([std_dev], dtype=pl.Float32),
+    }
+
+    if _has_localization(summary_dict):
+        _validate_localization_values(summary_dict)
+        data_dict["location_x"] = summary_dict.location_x
+        data_dict["location_y"] = summary_dict.location_y
+        data_dict["location_range"] = (
+            summary_dict.location_range or DEFAULT_LOCATION_RANGE_M
+        )
 
-    return pl.DataFrame(
-        {
-            "response_key": [summary_key],
-            "observation_key": [obs_key],
-            "time": pl.Series([date]).dt.cast_time_unit("ms"),
-            "observations": pl.Series([value], dtype=pl.Float32),
-            "std": pl.Series([std_dev], dtype=pl.Float32),
-            "location_x": pl.Series([summary_dict.location_x], dtype=pl.Float32),
-            "location_y": pl.Series([summary_dict.location_y], dtype=pl.Float32),
-            "location_range": pl.Series([location_range], dtype=pl.Float32),
-        }
-    )
+    return pl.DataFrame(data_dict)
 
 
 def _handle_general_observation(
     gen_data_config: GenDataConfig | None,
     general_observation: GeneralObservation,
     obs_key: str,
+    time_map: list[datetime],
+    has_refcase: bool,
 ) -> pl.DataFrame:
     response_key = general_observation.data
+
+    if all(
+        getattr(general_observation, key) is None
+        for key in ["restart", "date", "days", "hours"]
+    ):
+        # The user has not provided RESTART or DATE, this is legal
+        # for GEN_DATA, so we default it to None
+        restart = None
+    else:
+        restart = _get_restart(general_observation, obs_key, time_map, has_refcase)
+
     if gen_data_config is None or response_key not in gen_data_config.keys:
         raise ObservationConfigError.with_context(
             f"Problem with GENERAL_OBSERVATION {obs_key}:"
@@ -176,7 +415,7 @@ def _handle_general_observation(
             response_key,
         )
     assert isinstance(gen_data_config, GenDataConfig)
-    restart = general_observation.restart
+
     _, report_steps = gen_data_config.get_args_for_key(response_key)
 
     response_report_steps = [] if report_steps is None else report_steps