ert 19.0.0__py3-none-any.whl → 19.0.0rc1__py3-none-any.whl

ert/__main__.py

@@ -8,10 +8,12 @@ import multiprocessing
 import os
 import re
 import resource
+import shutil
 import sys
 import warnings
 from argparse import ArgumentParser, ArgumentTypeError
 from collections.abc import Sequence
+from datetime import datetime
 from pathlib import Path
 from typing import Any
 from uuid import UUID
@@ -24,6 +26,9 @@ from _ert.threading import set_signal_handler
 from ert.base_model_context import use_runtime_plugins
 from ert.cli.main import ErtCliError, run_cli
 from ert.config import ConfigValidationError, ErtConfig, lint_file
+from ert.config.observation_config_migrations import (
+    remove_refcase_and_time_map_dependence_from_obs_config,
+)
 from ert.logging import LOGGING_CONFIG
 from ert.mode_definitions import (
     ENIF_MODE,
@@ -36,9 +41,9 @@ from ert.mode_definitions import (
 from ert.namespace import Namespace
 from ert.plugins import ErtRuntimePlugins, get_site_plugins, setup_site_logging
 from ert.run_models.multiple_data_assimilation import MultipleDataAssimilationConfig
-from ert.services import ErtServer, WebvizErt
+from ert.services import ErtServer
+from ert.services._storage_main import add_parser_options as ert_api_add_parser_options
 from ert.shared.status.utils import get_ert_memory_usage
-from ert.shared.storage.command import add_parser_options as ert_api_add_parser_options
 from ert.storage import ErtStorageException, ErtStoragePermissionError
 from ert.trace import trace, tracer
 from ert.validation import (
@@ -53,6 +58,66 @@ from ert.validation import (
 logger = logging.getLogger(__name__)
 
 
+def run_convert_observations(
+    args: Namespace, _: ErtRuntimePlugins | None = None
+) -> None:
+    changes = remove_refcase_and_time_map_dependence_from_obs_config(args.config)
+
+    if changes is None or changes.is_empty():
+        logger.info("convert_observations did not make any changes")
+        print(
+            "No observations dependent on TIME_MAP / REFCASE found, you can "
+            "safely remove TIME_MAP / REFCASE and the "
+            "corresponding files from ERT config."
+        )
+        return
+
+    obs_config_to_edit_path = changes.obs_config_path + ".updated"
+    print(
+        f"Making copy of obs config "
+        f"@ {changes.obs_config_path} -> {obs_config_to_edit_path}"
+    )
+
+    shutil.copy(changes.obs_config_path, obs_config_to_edit_path)
+    print(f"Applying change to obs config @ {obs_config_to_edit_path}...")
+    changes.apply_to_file(Path(obs_config_to_edit_path))
+    convert_observations_trace = ""
+    for history_change in changes.history_changes:
+        convert_observations_trace += (
+            f"History obs {history_change.source_observation.name} "
+            f"-> {len(history_change.summary_obs_declarations)} summary observations\n"
+        )
+
+    for gen_obs_change in changes.general_obs_changes:
+        convert_observations_trace += (
+            f"General obs {gen_obs_change.source_observation.name}, changing "
+            f"DATE {gen_obs_change.source_observation.date} "
+            f"to RESTART={gen_obs_change.restart}\n"
+        )
+    for summary_change in changes.summary_obs_changes:
+        convert_observations_trace += (
+            f"Summary obs {summary_change.source_observation.name}, changing "
+            f"RESTART {summary_change.source_observation.restart} "
+            f"to DATE={summary_change.date}\n"
+        )
+
+    logger.info(f"convert_observations trace: \n {convert_observations_trace}")
+    print(convert_observations_trace)
+
+    os.rename(
+        changes.obs_config_path,
+        f"{changes.obs_config_path}-{datetime.now().strftime('%Y-%m-%d-%H-%M-%S')}.old",
+    )
+    os.rename(obs_config_to_edit_path, changes.obs_config_path)
+    msg = (
+        f"Observation changes applied to {changes.obs_config_path}. The old "
+        f"observations file is now at {changes.obs_config_path}.old and can be "
+        f"safely deleted if the new one works."
+    )
+    print(msg)
+    logger.info(msg)
+
+
 def run_ert_storage(args: Namespace, _: ErtRuntimePlugins | None = None) -> None:
     with ErtServer.start_server(
         verbose=True,
@@ -63,67 +128,24 @@ def run_ert_storage(args: Namespace, _: ErtRuntimePlugins | None = None) -> None
 
 
 def run_webviz_ert(args: Namespace, _: ErtRuntimePlugins | None = None) -> None:
-    try:
-        import webviz_ert  # type: ignore  # noqa
-    except ImportError as err:
-        raise ValueError(
-            "Running `ert vis` requires that webviz_ert is installed"
-        ) from err
-
-    kwargs: dict[str, Any] = {"verbose": args.verbose}
-    ert_config = ErtConfig.with_plugins(get_site_plugins()).from_file(args.config)
-
-    os.chdir(ert_config.config_path)
-    ens_path = ert_config.ens_path
-
-    # Changing current working directory means we need to
-    # only use the base name of the config file path
-    kwargs["ert_config"] = os.path.basename(args.config)
-    kwargs["project"] = os.path.abspath(ens_path)
-
     yellow = "\x1b[33m"
     green = "\x1b[32m"
     bold = "\x1b[1m"
     reset = "\x1b[0m"
 
-    try:
-        with ErtServer.init_service(project=Path(ens_path).absolute()) as storage:
-            storage.wait_until_ready()
-            print(
-                f"""
+    print(
+        f"""
     ---------------------------------------------------------------
 
-    {yellow}{bold}Webviz-ERT is deprecated and will be removed in the near future{reset}
+    {yellow}{bold}Webviz-ERT is removed.
 
     {green}{bold}Plotting capabilities provided by Webviz-ERT are now available
     using the ERT plotter{reset}
 
-    ---------------------------------------------------------------
-
-    Starting up Webviz-ERT. This might take more than a minute.
-
     ---------------------------------------------------------------
     """
-            )
-            logger.info("Show Webviz-ert deprecation warning")
-            webviz_kwargs = {
-                "experimental_mode": args.experimental_mode,
-                "verbose": args.verbose,
-                "title": kwargs.get("ert_config", "ERT - Visualization tool"),
-                "project": kwargs.get("project", os.getcwd()),
-            }
-            with WebvizErt.start_server(**webviz_kwargs) as webviz_ert_server:
-                webviz_ert_server.wait()
-    except PermissionError as pe:
-        print(f"Error: {pe}", file=sys.stderr)
-        print(
-            "Cannot start or connect to storage service due to permission issues.",
-            file=sys.stderr,
-        )
-        print(
-            "This is most likely due to another user starting ERT using this storage",
-            file=sys.stderr,
-        )
+    )
+    logger.info("Show Webviz-ert removal warning")
 
 
 def strip_error_message_and_raise_exception(validated: ValidationStatus) -> None:
@@ -317,19 +339,6 @@ def get_ert_parser(parser: ArgumentParser | None = None) -> ArgumentParser:
     ert_api_parser.set_defaults(func=run_ert_storage)
     ert_api_add_parser_options(ert_api_parser)
 
-    ert_vis_parser = subparsers.add_parser(
-        "vis",
-        description="Launch webviz-driven visualization tool.",
-    )
-    ert_vis_parser.set_defaults(func=run_webviz_ert)
-    ert_vis_parser.add_argument("--name", "-n", type=str, default="Webviz-ERT")
-    ert_vis_parser.add_argument(
-        "--experimental-mode",
-        action="store_true",
-        help="Feature flag for enabling experimental plugins",
-    )
-    ert_api_add_parser_options(ert_vis_parser)  # ert vis shares args with ert api
-
     # test_run_parser
     test_run_description = f"Run '{TEST_RUN_MODE}' in cli"
     test_run_parser = subparsers.add_parser(
@@ -564,6 +573,28 @@ def get_ert_parser(parser: ArgumentParser | None = None) -> ArgumentParser:
         "--ensemble", help="Which ensemble to use", default=None
     )
 
+    # convert_observations_parser
+    convert_obs_parser = subparsers.add_parser(
+        "convert_observations",
+        help=(
+            "Convert HISTORY_OBSERVATION to SUMMARY_OBSERVATION and "
+            "remove REFCASE and TIME_MAP from ERT config."
+        ),
+        description=(
+            "Convert HISTORY_OBSERVATION to SUMMARY_OBSERVATION, "
+            "and embed REFCASE and TIME_MAP into observations"
+        ),
+    )
+    convert_obs_parser.set_defaults(func=run_convert_observations)
+    convert_obs_parser.add_argument(
+        "config",
+        type=valid_file,
+        help="Path to ERT config file",
+    )
+    convert_obs_parser.add_argument(
+        "--verbose", action="store_true", help="Show verbose output.", default=False
+    )
+
     # Common arguments/defaults for all non-gui modes
     for cli_parser in [
         test_run_parser,

ert/analysis/_es_update.py

@@ -323,16 +323,13 @@ def analysis_ES(
         logger.info(log_msg)
         progress_callback(AnalysisStatusEvent(msg=log_msg))
 
-        log_msg = f"There are {num_obs} responses and {ensemble_size} realizations."
-        logger.info(log_msg)
-        progress_callback(AnalysisStatusEvent(msg=log_msg))
-
-        log_msg = (
-            f"There are {(~non_zero_variance_mask).sum()} parameters with 0 variance "
-            f"that will not be updated."
-        )
-        logger.info(log_msg)
-        progress_callback(AnalysisStatusEvent(msg=log_msg))
+        if (param_count := (~non_zero_variance_mask).sum()) > 0:
+            log_msg = (
+                f"There are {param_count} parameters with 0 variance "
+                f"that will not be updated."
+            )
+            logger.info(log_msg)
+            progress_callback(AnalysisStatusEvent(msg=log_msg))
 
         if module.localization:
             config_node = source_ensemble.experiment.parameter_configuration[
@@ -382,16 +379,16 @@ def analysis_ES(
             )
 
         else:
+            log_msg = f"There are {num_obs} responses and {ensemble_size} realizations."
+            logger.info(log_msg)
+            progress_callback(AnalysisStatusEvent(msg=log_msg))
+
             # In-place multiplication is not yet supported, therefore avoiding @=
             param_ensemble_array[non_zero_variance_mask] = param_ensemble_array[  # noqa: PLR6104
                 non_zero_variance_mask
             ] @ T.astype(param_ensemble_array.dtype)
 
-        log_msg = f"Storing data for {param_group}.."
-        logger.info(log_msg)
-        progress_callback(AnalysisStatusEvent(msg=log_msg))
         start = time.time()
-
         target_ensemble.save_parameters_numpy(
             param_ensemble_array, param_group, iens_active_index
         )

ert/config/_create_observation_dataframes.py

@@ -2,21 +2,19 @@ from __future__ import annotations
 
 from collections import defaultdict
 from collections.abc import Sequence
-from datetime import datetime, timedelta
-from typing import TYPE_CHECKING, Any, assert_never
+from datetime import datetime
+from typing import assert_never
 
 import numpy as np
 import polars as pl
-from resfo_utilities import history_key
+from numpy import typing as npt
 
 from ert.validation import rangestring_to_list
 
 from ._observations import (
     ErrorModes,
     GeneralObservation,
-    HistoryObservation,
     Observation,
-    ObservationDate,
     ObservationError,
     RFTObservation,
     SummaryObservation,
@@ -25,59 +23,31 @@ from .gen_data_config import GenDataConfig
 from .parsing import (
     ConfigWarning,
     ErrorInfo,
-    HistorySource,
     ObservationConfigError,
 )
-from .refcase import Refcase
 from .rft_config import RFTConfig
 
-if TYPE_CHECKING:
-    import numpy.typing as npt
-
-
-DEFAULT_TIME_DELTA = timedelta(seconds=30)
-DEFAULT_LOCALIZATION_RADIUS = 3000
+DEFAULT_LOCATION_RANGE_M = 3000
 
 
 def create_observation_dataframes(
     observations: Sequence[Observation],
-    refcase: Refcase | None,
     gen_data_config: GenDataConfig | None,
     rft_config: RFTConfig | None,
-    time_map: list[datetime] | None,
-    history: HistorySource,
 ) -> dict[str, pl.DataFrame]:
     if not observations:
         return {}
-    obs_time_list: list[datetime] = []
-    if refcase is not None:
-        obs_time_list = refcase.all_dates
-    elif time_map is not None:
-        obs_time_list = time_map
 
-    time_len = len(obs_time_list)
     config_errors: list[ErrorInfo] = []
     grouped: dict[str, list[pl.DataFrame]] = defaultdict(list)
     for obs in observations:
         try:
             match obs:
-                case HistoryObservation():
-                    grouped["summary"].append(
-                        _handle_history_observation(
-                            refcase,
-                            obs,
-                            obs.name,
-                            history,
-                            time_len,
-                        )
-                    )
                 case SummaryObservation():
                     grouped["summary"].append(
                         _handle_summary_observation(
                             obs,
                             obs.name,
-                            obs_time_list,
-                            bool(refcase),
                         )
                     )
                 case GeneralObservation():
@@ -86,8 +56,6 @@ def create_observation_dataframes(
                             gen_data_config,
                             obs,
                             obs.name,
-                            obs_time_list,
-                            bool(refcase),
                         )
                     )
                 case RFTObservation():
@@ -137,95 +105,6 @@ def _handle_error_mode(
             assert_never(default)
 
 
-def _handle_history_observation(
-    refcase: Refcase | None,
-    history_observation: HistoryObservation,
-    summary_key: str,
-    history_type: HistorySource,
-    time_len: int,
-) -> pl.DataFrame:
-    if refcase is None:
-        raise ObservationConfigError.with_context(
-            "REFCASE is required for HISTORY_OBSERVATION", summary_key
-        )
-
-    if history_type == HistorySource.REFCASE_HISTORY:
-        local_key = history_key(summary_key)
-    else:
-        local_key = summary_key
-    if local_key not in refcase.keys:
-        raise ObservationConfigError.with_context(
-            f"Key {local_key!r} is not present in refcase", summary_key
-        )
-    values = refcase.values[refcase.keys.index(local_key)]
-    std_dev = _handle_error_mode(values, history_observation)
-    for segment in history_observation.segments:
-        start = segment.start
-        stop = segment.stop
-        if start < 0:
-            ConfigWarning.warn(
-                f"Segment {segment.name} out of bounds."
-                " Truncating start of segment to 0.",
-                segment.name,
-            )
-            start = 0
-        if stop >= time_len:
-            ConfigWarning.warn(
-                f"Segment {segment.name} out of bounds. Truncating"
-                f" end of segment to {time_len - 1}.",
-                segment.name,
-            )
-            stop = time_len - 1
-        if start > stop:
-            ConfigWarning.warn(
-                f"Segment {segment.name} start after stop. Truncating"
-                f" end of segment to {start}.",
-                segment.name,
-            )
-            stop = start
-        if np.size(std_dev[start:stop]) == 0:
-            ConfigWarning.warn(
-                f"Segment {segment.name} does not"
-                " contain any time steps. The interval "
-                f"[{start}, {stop}) does not intersect with steps in the"
-                "time map.",
-                segment.name,
-            )
-        std_dev[start:stop] = _handle_error_mode(values[start:stop], segment)
-    dates_series = pl.Series(refcase.dates).dt.cast_time_unit("ms")
-    if (std_dev <= 0).any():
-        raise ObservationConfigError.with_context(
-            "Observation uncertainty must be strictly > 0", summary_key
-        ) from None
-
-    return pl.DataFrame(
-        {
-            "response_key": summary_key,
-            "observation_key": summary_key,
-            "time": dates_series,
-            "observations": pl.Series(values, dtype=pl.Float32),
-            "std": pl.Series(std_dev, dtype=pl.Float32),
-            "east": pl.Series([None] * len(values), dtype=pl.Float32),
-            "north": pl.Series([None] * len(values), dtype=pl.Float32),
-            "radius": pl.Series([None] * len(values), dtype=pl.Float32),
-        }
-    )
-
-
-def _get_time(
-    date_dict: ObservationDate, start_time: datetime, context: Any = None
-) -> tuple[datetime, str]:
-    if date_dict.date is not None:
-        return _parse_date(date_dict.date), f"DATE={date_dict.date}"
-    if date_dict.days is not None:
-        days = date_dict.days
-        return start_time + timedelta(days=days), f"DAYS={days}"
-    if date_dict.hours is not None:
-        hours = date_dict.hours
-        return start_time + timedelta(hours=hours), f"HOURS={hours}"
-    raise ObservationConfigError.with_context("Missing time specifier", context=context)
-
-
 def _parse_date(date_str: str) -> datetime:
     try:
         return datetime.fromisoformat(date_str)
@@ -246,102 +125,26 @@ def _parse_date(date_str: str) -> datetime:
             return date
 
 
-def _find_nearest(
-    time_map: list[datetime],
-    time: datetime,
-    threshold: timedelta = DEFAULT_TIME_DELTA,
-) -> int:
-    nearest_index = -1
-    nearest_diff = None
-    for i, t in enumerate(time_map):
-        diff = abs(time - t)
-        if diff < threshold and (nearest_diff is None or nearest_diff > diff):
-            nearest_diff = diff
-            nearest_index = i
-    if nearest_diff is None:
-        raise IndexError(f"{time} is not in the time map")
-    return nearest_index
-
-
-def _get_restart(
-    date_dict: ObservationDate,
-    obs_name: str,
-    time_map: list[datetime],
-    has_refcase: bool,
-) -> int:
-    if date_dict.restart is not None:
-        return date_dict.restart
-    if not time_map:
-        raise ObservationConfigError.with_context(
-            f"Missing REFCASE or TIME_MAP for observations: {obs_name}",
-            obs_name,
-        )
-
-    time, date_str = _get_time(date_dict, time_map[0], context=obs_name)
-
-    try:
-        return _find_nearest(time_map, time)
-    except IndexError as err:
-        raise ObservationConfigError.with_context(
-            f"Could not find {time} ({date_str}) in "
-            f"the time map for observations {obs_name}. "
-            + (
-                "The time map is set from the REFCASE keyword. Either "
-                "the REFCASE has an incorrect/missing date, or the observation "
-                "is given an incorrect date.)"
-                if has_refcase
-                else "(The time map is set from the TIME_MAP "
-                "keyword. Either the time map file has an "
-                "incorrect/missing date, or the observation is given an "
-                "incorrect date."
-            ),
-            obs_name,
-        ) from err
-
-
 def _has_localization(summary_dict: SummaryObservation) -> bool:
-    return summary_dict.east is not None and summary_dict.north is not None
+    return summary_dict.location_x is not None and summary_dict.location_y is not None
 
 
 def _handle_summary_observation(
     summary_dict: SummaryObservation,
     obs_key: str,
-    time_map: list[datetime],
-    has_refcase: bool,
 ) -> pl.DataFrame:
     summary_key = summary_dict.key
     value = summary_dict.value
     std_dev = float(_handle_error_mode(np.array(value), summary_dict))
+    date = _parse_date(summary_dict.date)
 
-    if summary_dict.restart and not (time_map or has_refcase):
-        raise ObservationConfigError.with_context(
-            "Keyword 'RESTART' requires either TIME_MAP or REFCASE", context=obs_key
-        )
-
-    if summary_dict.date is not None and not time_map:
-        # We special case when the user has provided date in SUMMARY_OBS
-        # and not REFCASE or time_map so that we don't change current behavior.
-        date = _parse_date(summary_dict.date)
-        restart = None
-    else:
-        restart = _get_restart(summary_dict, obs_key, time_map, has_refcase)
-        date = time_map[restart]
-
-    if restart == 0:
-        raise ObservationConfigError.with_context(
-            "It is unfortunately not possible to use summary "
-            "observations from the start of the simulation. "
-            f"Problem with observation {obs_key}"
-            f"{' at ' + str(_get_time(summary_dict, time_map[0], obs_key)) if summary_dict.restart is None else ''}",  # noqa: E501
-            obs_key,
-        )
     if std_dev <= 0:
         raise ObservationConfigError.with_context(
             "Observation uncertainty must be strictly > 0", summary_key
         ) from None
 
-    localization_radius = (
-        summary_dict.radius or DEFAULT_LOCALIZATION_RADIUS
+    location_range = (
+        summary_dict.location_range or DEFAULT_LOCATION_RANGE_M
         if _has_localization(summary_dict)
         else None
     )
@@ -353,9 +156,9 @@ def _handle_summary_observation(
             "time": pl.Series([date]).dt.cast_time_unit("ms"),
             "observations": pl.Series([value], dtype=pl.Float32),
             "std": pl.Series([std_dev], dtype=pl.Float32),
-            "east": pl.Series([summary_dict.east], dtype=pl.Float32),
-            "north": pl.Series([summary_dict.north], dtype=pl.Float32),
-            "radius": pl.Series([localization_radius], dtype=pl.Float32),
+            "location_x": pl.Series([summary_dict.location_x], dtype=pl.Float32),
+            "location_y": pl.Series([summary_dict.location_y], dtype=pl.Float32),
+            "location_range": pl.Series([location_range], dtype=pl.Float32),
         }
     )
 
@@ -364,21 +167,8 @@ def _handle_general_observation(
     gen_data_config: GenDataConfig | None,
     general_observation: GeneralObservation,
     obs_key: str,
-    time_map: list[datetime],
-    has_refcase: bool,
 ) -> pl.DataFrame:
     response_key = general_observation.data
-
-    if all(
-        getattr(general_observation, key) is None
-        for key in ["restart", "date", "days", "hours"]
-    ):
-        # The user has not provided RESTART or DATE, this is legal
-        # for GEN_DATA, so we default it to None
-        restart = None
-    else:
-        restart = _get_restart(general_observation, obs_key, time_map, has_refcase)
-
     if gen_data_config is None or response_key not in gen_data_config.keys:
         raise ObservationConfigError.with_context(
             f"Problem with GENERAL_OBSERVATION {obs_key}:"
@@ -386,7 +176,7 @@ def _handle_general_observation(
             response_key,
         )
     assert isinstance(gen_data_config, GenDataConfig)
-
+    restart = general_observation.restart
     _, report_steps = gen_data_config.get_args_for_key(response_key)
 
     response_report_steps = [] if report_steps is None else report_steps
@@ -485,11 +275,6 @@ def _handle_general_observation(
             "index": pl.Series(indices, dtype=pl.UInt16),
             "observations": pl.Series(values, dtype=pl.Float32),
             "std": pl.Series(stds, dtype=pl.Float32),
-            # Location attributes will always be None for general observations, but are
-            # necessary to concatenate with other observation dataframes.
-            "east": pl.Series([None] * len(values), dtype=pl.Float32),
-            "north": pl.Series([None] * len(values), dtype=pl.Float32),
-            "radius": pl.Series([None] * len(values), dtype=pl.Float32),
         }
     )
 
@@ -532,6 +317,5 @@ def _handle_rft_observation(
             "tvd": pl.Series([location[2]], dtype=pl.Float32),
             "observations": pl.Series([rft_observation.value], dtype=pl.Float32),
             "std": pl.Series([rft_observation.error], dtype=pl.Float32),
-            "radius": pl.Series([None], dtype=pl.Float32),
         }
     )