ert 17.0.0__py3-none-any.whl → 19.0.0rc2__py3-none-any.whl

ert/config/parsing/_option_dict.py

@@ -48,13 +48,21 @@ def option_dict(option_list: Sequence[str], offset: int) -> dict[str, str]:
         if len(option_pair.split(":")) == 2:
             key, val = option_pair.split(":")
             if val and key:
+                if key in result:
+                    raise ConfigValidationError.with_context(
+                        f"Option {key} occured multiple times.", option_pair
+                    )
                 result[key] = val
             else:
                 raise ConfigValidationError.with_context(
-                    f"Invalid argument {option_pair!r}", option_pair
+                    "Option argument should be of the form 'key':'value', "
+                    f"got {option_pair!r}",
+                    option_pair,
                 )
         else:
             raise ConfigValidationError.with_context(
-                f"Invalid argument {option_pair!r}", option_pair
+                "Option argument should be of the form 'key':'value', "
+                f"got {option_pair!r}",
+                option_pair,
             )
     return result

ert/config/parsing/config_keywords.py

@@ -42,6 +42,7 @@ class ConfigKeys(StrEnum):
     SETENV = "SETENV"
     STD_CUTOFF = "STD_CUTOFF"
     SUMMARY = "SUMMARY"
+    RFT = "RFT"
     SURFACE = "SURFACE"
     UPDATE_LOG_PATH = "UPDATE_LOG_PATH"
     RANDOM_SEED = "RANDOM_SEED"
@@ -55,6 +56,7 @@ class ConfigKeys(StrEnum):
     REALIZATION_MEMORY = "REALIZATION_MEMORY"
     SUBMIT_SLEEP = "SUBMIT_SLEEP"
     MAX_RUNNING = "MAX_RUNNING"
+    PRIORITIZE_PRIVATE_IP_ADDRESS = "PRIORITIZE_PRIVATE_IP_ADDRESS"
 
     def __repr__(self) -> str:
         return f"{self.value!r}"

ert/config/parsing/config_schema.py

@@ -4,8 +4,8 @@ from .config_schema_deprecations import deprecated_keywords_list
 from .config_schema_item import (
     SchemaItem,
     Varies,
+    existing_file_keyword,
     existing_path_inline_keyword,
-    existing_path_keyword,
     float_keyword,
     int_keyword,
     path_keyword,
@@ -63,6 +63,17 @@ def data_kw_keyword() -> SchemaItem:
     )
 
 
+def rft_keyword() -> SchemaItem:
+    return SchemaItem(
+        kw=ConfigKeys.RFT,
+        required_set=False,
+        multi_occurrence=True,
+        options_after=0,
+        argc_min=1,
+        argc_max=1,
+    )
+
+
 def define_keyword() -> SchemaItem:
     return SchemaItem(
         kw=ConfigKeys.DEFINE,
@@ -115,6 +126,13 @@ def hook_workflow_keyword() -> SchemaItem:
     )
 
 
+def prioritize_private_ip_address_keyword() -> SchemaItem:
+    return SchemaItem(
+        kw=ConfigKeys.PRIORITIZE_PRIVATE_IP_ADDRESS,
+        type_map=[SchemaItemType.BOOL],
+    )
+
+
 def set_env_keyword() -> SchemaItem:
     # You can set environment variables which will be applied to the run-time
     # environment.
@@ -303,11 +321,12 @@ def init_user_config_schema() -> ConfigSchemaDict:
         gen_kw_keyword(),
         gen_data_keyword(),
         summary_keyword(),
+        rft_keyword(),
         surface_keyword(),
         field_keyword(),
         single_arg_keyword(ConfigKeys.ECLBASE),
-        existing_path_keyword(ConfigKeys.DATA_FILE),
-        existing_path_keyword(ConfigKeys.GRID),
+        existing_file_keyword(ConfigKeys.DATA_FILE),
+        existing_file_keyword(ConfigKeys.GRID),
         path_keyword(ConfigKeys.REFCASE),
         int_keyword(ConfigKeys.RANDOM_SEED),
         num_realizations_keyword(),
@@ -337,6 +356,7 @@ def init_user_config_schema() -> ConfigSchemaDict:
         install_job_keyword(),
         install_job_directory_keyword(),
         hook_workflow_keyword(),
+        prioritize_private_ip_address_keyword(),
     ]:
         schema[item.kw] = item
         if item.kw in ConfigAliases:

ert/config/parsing/config_schema_deprecations.py

@@ -39,9 +39,9 @@ deprecated_keywords_list = [
             keyword=kw,
             message=partial(
                 lambda line, kw: f"Using {kw} with substitution strings "
-                + "that are not of the form '<KEY>' is deprecated. "
-                + f"Please change {line[0]} to "
-                + f"<{line[0].replace('<', '').replace('>', '')}>",
+                "that are not of the form '<KEY>' is deprecated. "
+                f"Please change {line[0]} to "
+                f"<{line[0].replace('<', '').replace('>', '')}>",
                 kw=kw,
             ),
             check=lambda line: not DeprecationInfo.is_angle_bracketed(str(line[0])),
@@ -217,15 +217,4 @@ deprecated_keywords_list = [
         ),
         check=lambda line: line[0] == "DESIGN2PARAMS",
     ),
-    DeprecationInfo(
-        keyword="FORWARD_MODEL",
-        message=(
-            "FORWARD_MODEL DESIGN_KW will be replaced with RUN_TEMPLATE. "
-            "DESIGN2PARAMS has been replaced by DESIGN_MATRIX, so the "
-            "parameters are already available for magic string replacement "
-            "with the RUN_TEMPLATE keyword. Please use this format: "
-            "'RUN_TEMPLATE my_text_file_template.txt my_text_output_file.txt'"
-        ),
-        check=lambda line: line[0] == "DESIGN_KW",
-    ),
 ]

ert/config/parsing/config_schema_item.py

@@ -37,25 +37,34 @@ class SchemaItem:
 
     # The minimum number of arguments
     argc_min: NonNegativeInt = 1
+
     # The maximum number of arguments: None means no upper limit
     argc_max: NonNegativeInt | None = 1
+
     # A list of types for the items. Set along with argc_minmax()
     type_map: list[SchemaItemType | EnumType | None] = Field(default_factory=list)
+
     # A list of item's which must also be set (if this item is set). (can be NULL)
     required_children: list[str] = Field(default_factory=list)
+
     # Information about the deprecation if deprecated
     deprecation_info: list[DeprecationInfo] = Field(default_factory=list)
-    # if positive, arguments after this count will be concatenated with a " " between
+
+    # If positive, arguments after this count will be concatenated with a " " between
     join_after: PositiveInt | None = None
-    # if positive, arguments after this count will be interpreted as options
-    options_after: PositiveInt | Varies | None = None
-    # if true, will accumulate many values set for key, otherwise each entry will
+
+    # If positive, arguments after this count will be interpreted as options
+    options_after: NonNegativeInt | Varies | None = None
+
+    # If true, will accumulate many values set for key, otherwise each entry will
     # overwrite any previous value set
     multi_occurrence: bool = False
+
     # Only applies to SchemaItemType.EXISTING_PATH_INLINE where
     # the contents is then parsed
     parser: Callable[[str, str], Any] = lambda x, y: y
     expand_envvar: bool = True
+
     # Index of tokens to do substitution from until end
     substitute_from: NonNegativeInt = 1
     required_set: bool = False
@@ -155,9 +164,9 @@ class SchemaItem:
                         f"value as argument {index + 1!r}",
                         token,
                     )
-
             case (
                 SchemaItemType.PATH
+                | SchemaItemType.EXISTING_FILE
                 | SchemaItemType.EXISTING_PATH
                 | SchemaItemType.EXISTING_PATH_INLINE
             ):
@@ -166,10 +175,14 @@ class SchemaItem:
                     path = os.path.normpath(
                         os.path.join(os.path.dirname(token.filename), token)
                     )
-                if val_type in {
-                    SchemaItemType.EXISTING_PATH,
-                    SchemaItemType.EXISTING_PATH_INLINE,
-                }:
+                if val_type != SchemaItemType.PATH:
+                    if val_type == SchemaItemType.EXISTING_FILE and not os.path.isfile(
+                        str(path)
+                    ):
+                        raise ConfigValidationError.with_context(
+                            f"{self.kw} {token} is not a file.",
+                            token,
+                        )
                     if not os.path.exists(str(path)):
                         err = f'Cannot find file or directory "{token.value}". '
                         if path != token:
@@ -311,6 +324,8 @@ class SchemaItem:
         self, line: Sequence[FileContextToken]
     ) -> Sequence[FileContextToken | dict[FileContextToken, FileContextToken]]:
         n = self.options_after
+        if not line:
+            return []
         if isinstance(n, Varies):
             args, kwargs = parse_variable_options(list(line), n.max_positionals)
             return [*args, kwargs]  # type: ignore
@@ -343,8 +358,8 @@ def path_keyword(keyword: str) -> SchemaItem:
     return SchemaItem(kw=keyword, type_map=[SchemaItemType.PATH])
 
 
-def existing_path_keyword(keyword: str) -> SchemaItem:
-    return SchemaItem(kw=keyword, type_map=[SchemaItemType.EXISTING_PATH])
+def existing_file_keyword(keyword: str) -> SchemaItem:
+    return SchemaItem(kw=keyword, type_map=[SchemaItemType.EXISTING_FILE])
 
 
 def existing_path_inline_keyword(

ert/config/parsing/context_values.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 from json import JSONEncoder
-from typing import Any, TypeVar, no_type_check
+from typing import Any, Self, TypeVar, no_type_check
 
 from .file_context_token import FileContextToken
 
@@ -37,7 +37,7 @@ class ContextBool:
 
 
 class ContextInt(int):
-    def __new__(cls, val: int, token: FileContextToken) -> ContextInt:
+    def __new__(cls, val: int, token: FileContextToken) -> Self:
         obj = super().__new__(cls, val)
         obj.token = token
         return obj
@@ -50,7 +50,7 @@ class ContextInt(int):
 
 
 class ContextFloat(float):
-    def __new__(cls, val: float, token: FileContextToken) -> ContextFloat:
+    def __new__(cls, val: float, token: FileContextToken) -> Self:
         obj = super().__new__(cls, val)
         obj.token = token
         return obj

ert/config/parsing/file_context_token.py

@@ -12,7 +12,7 @@ class FileContextToken(Token):
 
     filename: str
 
-    def __new__(cls, token: Token, filename: str) -> FileContextToken:
+    def __new__(cls, token: Token, filename: str) -> FileContextToken:  # noqa: PYI034
         inst = super().__new__(
             cls,
             token.type,

ert/config/parsing/observations_parser.py

@@ -18,6 +18,7 @@ class ObservationType(StrEnum):
     HISTORY = "HISTORY_OBSERVATION"
     SUMMARY = "SUMMARY_OBSERVATION"
     GENERAL = "GENERAL_OBSERVATION"
+    RFT = "RFT_OBSERVATION"
 
 
 ObservationDict = dict[str, Any]
@@ -92,7 +93,7 @@ def parse_observations(content: str, filename: str) -> list[ObservationDict]:
             ), ["TYPE"]:
                 message = (
                     f"Unknown observation type '{unexpected_token}', "
-                    f"expected either 'GENERAL_OBSERVATION', "
+                    f"expected either 'RFT_OBSERVATION', 'GENERAL_OBSERVATION', "
                     f"'SUMMARY_OBSERVATION' or 'HISTORY_OBSERVATION'."
                 )
             case UnexpectedToken(token=unexpected_char, expected=allowed_chars), _:
@@ -122,7 +123,10 @@ observations_parser = Lark(
     r"""
     start: observation*
     ?observation: type OBSERVATION_NAME object? ";"
-    TYPE: "HISTORY_OBSERVATION" | "SUMMARY_OBSERVATION" | "GENERAL_OBSERVATION"
+    TYPE: "HISTORY_OBSERVATION"
+      | "SUMMARY_OBSERVATION"
+      | "GENERAL_OBSERVATION"
+      | "RFT_OBSERVATION"
     type: TYPE
     ?value: object
           | STRING

ert/config/parsing/queue_system.py

@@ -23,3 +23,12 @@ class QueueSystem(StrEnum):
     @staticmethod
     def ert_config_case() -> str:
         return "upper"
+
+    @property
+    def formatted_name(self) -> str:
+        return {
+            self.LSF: "LSF",
+            self.LOCAL: "Local",
+            self.TORQUE: "Torque/OpenPBS",
+            self.SLURM: "Slurm",
+        }[self]

ert/config/parsing/schema_item_type.py

@@ -9,6 +9,7 @@ class SchemaItemType(StrEnum):
     POSITIVE_FLOAT = "POSITIVE_FLOAT"
     PATH = "PATH"
     EXISTING_PATH = "EXISTING_PATH"
+    EXISTING_FILE = "EXISTING_FILE"
     # EXISTING_PATH_INLINE is a directive to the
     # schema validation to inline the contents of
     # the file.

ert/config/queue_config.py

@@ -51,7 +51,6 @@ class QueueOptions(
     BaseModelWithContextSupport,
     validate_assignment=True,
     extra="forbid",
-    use_enum_values=True,
     validate_default=True,
 ):
     name: QueueSystem
@@ -478,11 +477,11 @@ class QueueConfig(BaseModelWithContextSupport):
 
         # validate all queue options for the unselected queues
         # and show a warning
-        for _queue_system in QueueSystem:
-            if _queue_system != selected_queue_system:
+        for queue_system in QueueSystem:
+            if queue_system != selected_queue_system:
                 _ = QueueOptions.create_queue_options(
-                    _queue_system,
-                    grouped_queue_options[_queue_system],
+                    queue_system,
+                    grouped_queue_options[queue_system],
                     False,
                 )
 

ert/config/response_config.py

@@ -31,7 +31,6 @@ class ResponseMetadata(BaseModel):
 
 class ResponseConfig(BaseModel):
     type: str
-    name: str
     input_files: list[str] = Field(default_factory=list)
     keys: list[str] = Field(default_factory=list)
     has_finalized_keys: bool = False
@@ -60,13 +59,6 @@ class ResponseConfig(BaseModel):
     def expected_input_files(self) -> list[str]:
         """Returns a list of filenames expected to be produced by the forward model"""
 
-    @property
-    @abstractmethod
-    def response_type(self) -> str:
-        """Label to identify what kind of response it is.
-        Must not overlap with that of other response configs."""
-        ...
-
     @property
     @abstractmethod
     def primary_key(self) -> list[str]:

ert/config/rft_config.py

@@ -0,0 +1,275 @@
+from __future__ import annotations
+
+import datetime
+import fnmatch
+import logging
+import os
+import re
+from collections import defaultdict
+from typing import IO, Any, Literal
+
+import numpy as np
+import numpy.typing as npt
+import polars as pl
+from pydantic import Field
+from resfo_utilities import CornerpointGrid, InvalidRFTError, RFTReader
+
+from ert.substitutions import substitute_runpath_name
+
+from .parsing import ConfigDict, ConfigKeys, ConfigValidationError, ConfigWarning
+from .response_config import InvalidResponseFile, ResponseConfig, ResponseMetadata
+from .responses_index import responses_index
+
+logger = logging.getLogger(__name__)
+
+
+class RFTConfig(ResponseConfig):
+    type: Literal["rft"] = "rft"
+    name: str = "rft"
+    has_finalized_keys: bool = False
+    data_to_read: dict[str, dict[str, list[str]]] = Field(default_factory=dict)
+    locations: list[tuple[float, float, float]] = Field(default_factory=list)
+
+    @property
+    def metadata(self) -> list[ResponseMetadata]:
+        return [
+            ResponseMetadata(
+                response_type=self.name,
+                response_key=response_key,
+                filter_on=None,
+                finalized=self.has_finalized_keys,
+            )
+            for response_key in self.keys
+        ]
+
+    @property
+    def expected_input_files(self) -> list[str]:
+        base = self.input_files[0]
+        if base.upper().endswith(".DATA"):
+            # For backwards compatibility, it is
+            # allowed to give REFCASE and ECLBASE both
+            # with and without .DATA extensions
+            base = base[:-5]
+
+        return [f"{base}.RFT"]
+
+    def _find_indices(
+        self, egrid_file: str | os.PathLike[str] | IO[Any]
+    ) -> dict[tuple[int, int, int] | None, set[tuple[float, float, float]]]:
+        indices = defaultdict(set)
+        for a, b in zip(
+            CornerpointGrid.read_egrid(egrid_file).find_cell_containing_point(
+                self.locations
+            ),
+            self.locations,
+            strict=True,
+        ):
+            indices[a].add(b)
+        return indices
+
+    def read_from_file(self, run_path: str, iens: int, iter_: int) -> pl.DataFrame:
+        filename = substitute_runpath_name(self.input_files[0], iens, iter_)
+        if filename.upper().endswith(".DATA"):
+            # For backwards compatibility, it is
+            # allowed to give REFCASE and ECLBASE both
+            # with and without .DATA extensions
+            filename = filename[:-5]
+        grid_filename = f"{run_path}/{filename}"
+        if grid_filename.upper().endswith(".RFT"):
+            grid_filename = grid_filename[:-4]
+        grid_filename += ".EGRID"
+        fetched: dict[tuple[str, datetime.date], dict[str, npt.NDArray[np.float32]]] = (
+            defaultdict(dict)
+        )
+        indices = {}
+        if self.locations:
+            indices = self._find_indices(grid_filename)
+        if None in indices:
+            raise InvalidResponseFile(
+                f"Did not find grid coordinate for location(s) {indices[None]}"
+            )
+        # This is a somewhat complicated optimization in order to
+        # support wildcards in well names, dates and properties
+        # A python for loop is too slow so we use a compiled regex
+        # instead
+        if not self.data_to_read:
+            return pl.DataFrame(
+                {
+                    "response_key": [],
+                    "time": [],
+                    "depth": [],
+                    "values": [],
+                    "location": [],
+                }
+            )
+
+        sep = "\x31"
+
+        def _translate(pat: str) -> str:
+            """Translates fnmatch pattern to match anywhere"""
+            return fnmatch.translate(pat).replace("\\z", "").replace("\\Z", "")
+
+        def _props_matcher(props: list[str]) -> str:
+            """Regex for matching given props _and_ DEPTH"""
+            pattern = f"({'|'.join(_translate(p) for p in props)})"
+            if re.fullmatch(pattern, "DEPTH") is None:
+                return f"({'|'.join(_translate(p) for p in [*props, 'DEPTH'])})"
+            else:
+                return pattern
+
+        matcher = re.compile(
+            "|".join(
+                "("
+                + re.escape(sep).join(
+                    (
+                        _translate(well),
+                        _translate(time),
+                        _props_matcher(props),
+                    )
+                )
+                + ")"
+                for well, inner_dict in self.data_to_read.items()
+                for time, props in inner_dict.items()
+            )
+        )
+        locations = {}
+        try:
+            with RFTReader.open(f"{run_path}/{filename}") as rft:
+                for entry in rft:
+                    date = entry.date
+                    well = entry.well
+                    for rft_property in entry:
+                        key = f"{well}{sep}{date}{sep}{rft_property}"
+                        if matcher.fullmatch(key) is not None:
+                            values = entry[rft_property]
+                            locations[well, date] = [
+                                list(
+                                    indices.get(
+                                        (c[0] - 1, c[1] - 1, c[2] - 1),
+                                        [(None, None, None)],
+                                    )
+                                )
+                                for c in entry.connections
+                            ]
+                            if np.isdtype(values.dtype, np.float32):
+                                fetched[well, date][rft_property] = values
+        except (FileNotFoundError, InvalidRFTError) as err:
+            raise InvalidResponseFile(
+                f"Could not read RFT from {run_path}/{filename}: {err}"
+            ) from err
+
+        if not fetched:
+            return pl.DataFrame(
+                {
+                    "response_key": [],
+                    "time": [],
+                    "depth": [],
+                    "values": [],
+                    "location": [],
+                }
+            )
+
+        try:
+            df = pl.concat(
+                [
+                    pl.DataFrame(
+                        {
+                            "response_key": [f"{well}:{time.isoformat()}:{prop}"],
+                            "time": [time],
+                            "depth": [fetched[well, time]["DEPTH"]],
+                            "values": [vals],
+                            "location": pl.Series(
+                                [
+                                    locations.get(
+                                        (well, time), [(None, None, None)] * len(vals)
+                                    )
+                                ],
+                                dtype=pl.Array(
+                                    pl.List(pl.Array(pl.Float32, 3)), len(vals)
+                                ),
+                            ),
+                        }
+                    )
+                    .explode("depth", "values", "location")
+                    .explode("location")
+                    for (well, time), inner_dict in fetched.items()
+                    for prop, vals in inner_dict.items()
+                    if prop != "DEPTH"
+                ]
+            )
+        except KeyError as err:
+            raise InvalidResponseFile(
+                f"Could not find {err.args[0]} in RFTFile {filename}"
+            ) from err
+
+        return df.with_columns(
+            east=pl.col("location").arr.get(0),
+            north=pl.col("location").arr.get(1),
+            tvd=pl.col("location").arr.get(2),
+        ).drop("location")
+
+    @property
+    def response_type(self) -> str:
+        return "rft"
+
+    @property
+    def primary_key(self) -> list[str]:
+        return ["east", "north", "tvd"]
+
+    @classmethod
+    def from_config_dict(cls, config_dict: ConfigDict) -> RFTConfig | None:
+        if rfts := config_dict.get(ConfigKeys.RFT, []):
+            eclbase: str | None = config_dict.get("ECLBASE")
+            if eclbase is None:
+                raise ConfigValidationError(
+                    "In order to use rft responses, ECLBASE has to be set."
+                )
+            fm_steps = config_dict.get(ConfigKeys.FORWARD_MODEL, [])
+            names = [fm_step[0] for fm_step in fm_steps]
+            simulation_step_exists = any(
+                any(sim in name.lower() for sim in ["eclipse", "flow"])
+                for name in names
+            )
+            if not simulation_step_exists:
+                ConfigWarning.warn(
+                    "Config contains a RFT key but no forward model "
+                    "step known to generate rft files"
+                )
+
+            declared_data: dict[str, dict[datetime.date, list[str]]] = defaultdict(
+                lambda: defaultdict(list)
+            )
+            for rft in rfts:
+                for expected in ["WELL", "DATE", "PROPERTIES"]:
+                    if expected not in rft:
+                        raise ConfigValidationError.with_context(
+                            f"For RFT keyword {expected} must be specified.", rft
+                        )
+                well = rft["WELL"]
+                props = [p.strip() for p in rft["PROPERTIES"].split(",")]
+                time = rft["DATE"]
+                declared_data[well][time] += props
+            data_to_read = {
+                well: {time: sorted(set(p)) for time, p in inner_dict.items()}
+                for well, inner_dict in declared_data.items()
+            }
+            keys = sorted(
+                {
+                    f"{well}:{time}:{p}"
+                    for well, inner_dict in declared_data.items()
+                    for time, props in inner_dict.items()
+                    for p in props
+                }
+            )
+
+            return cls(
+                name="rft",
+                input_files=[eclbase.replace("%d", "<IENS>")],
+                keys=keys,
+                data_to_read=data_to_read,
+            )
+
+        return None
+
+
+responses_index.add_response_type(RFTConfig)

ert/config/summary_config.py

@@ -19,14 +19,13 @@ logger = logging.getLogger(__name__)
 
 class SummaryConfig(ResponseConfig):
     type: Literal["summary"] = "summary"
-    name: str = "summary"
     has_finalized_keys: bool = False
 
     @property
     def metadata(self) -> list[ResponseMetadata]:
         return [
             ResponseMetadata(
-                response_type=self.name,
+                response_type=self.type,
                 response_key=response_key,
                 filter_on=None,
                 finalized=self.has_finalized_keys,
@@ -72,10 +71,6 @@ class SummaryConfig(ResponseConfig):
         df = df.sort(by=["time"])
         return df
 
-    @property
-    def response_type(self) -> str:
-        return "summary"
-
     @property
     def primary_key(self) -> list[str]:
         return ["time"]
@@ -91,8 +86,8 @@ class SummaryConfig(ResponseConfig):
             fm_steps = config_dict.get(ConfigKeys.FORWARD_MODEL, [])
             names = [fm_step[0] for fm_step in fm_steps]
             simulation_step_exists = any(
-                any(sim in _name.lower() for sim in ["eclipse", "flow"])
-                for _name in names
+                any(sim in name.lower() for sim in ["eclipse", "flow"])
+                for name in names
             )
             if not simulation_step_exists:
                 ConfigWarning.warn(