ert 16.0.9__py3-none-any.whl → 19.0.0rc2__py3-none-any.whl

ert/config/parsing/context_values.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 from json import JSONEncoder
-from typing import Any, TypeVar, no_type_check
+from typing import Any, Self, TypeVar, no_type_check
 
 from .file_context_token import FileContextToken
 
@@ -37,7 +37,7 @@ class ContextBool:
 
 
 class ContextInt(int):
-    def __new__(cls, val: int, token: FileContextToken) -> ContextInt:
+    def __new__(cls, val: int, token: FileContextToken) -> Self:
         obj = super().__new__(cls, val)
         obj.token = token
         return obj
@@ -50,7 +50,7 @@ class ContextInt(int):
 
 
 class ContextFloat(float):
-    def __new__(cls, val: float, token: FileContextToken) -> ContextFloat:
+    def __new__(cls, val: float, token: FileContextToken) -> Self:
         obj = super().__new__(cls, val)
         obj.token = token
         return obj

ert/config/parsing/file_context_token.py

@@ -12,7 +12,7 @@ class FileContextToken(Token):
 
     filename: str
 
-    def __new__(cls, token: Token, filename: str) -> FileContextToken:
+    def __new__(cls, token: Token, filename: str) -> FileContextToken:  # noqa: PYI034
         inst = super().__new__(
             cls,
             token.type,

ert/config/parsing/observations_parser.py

@@ -18,6 +18,7 @@ class ObservationType(StrEnum):
     HISTORY = "HISTORY_OBSERVATION"
     SUMMARY = "SUMMARY_OBSERVATION"
     GENERAL = "GENERAL_OBSERVATION"
+    RFT = "RFT_OBSERVATION"
 
 
 ObservationDict = dict[str, Any]
@@ -92,7 +93,7 @@ def parse_observations(content: str, filename: str) -> list[ObservationDict]:
             ), ["TYPE"]:
                 message = (
                     f"Unknown observation type '{unexpected_token}', "
-                    f"expected either 'GENERAL_OBSERVATION', "
+                    f"expected either 'RFT_OBSERVATION', 'GENERAL_OBSERVATION', "
                     f"'SUMMARY_OBSERVATION' or 'HISTORY_OBSERVATION'."
                 )
             case UnexpectedToken(token=unexpected_char, expected=allowed_chars), _:
@@ -122,7 +123,10 @@ observations_parser = Lark(
     r"""
     start: observation*
     ?observation: type OBSERVATION_NAME object? ";"
-    TYPE: "HISTORY_OBSERVATION" | "SUMMARY_OBSERVATION" | "GENERAL_OBSERVATION"
+    TYPE: "HISTORY_OBSERVATION"
+      | "SUMMARY_OBSERVATION"
+      | "GENERAL_OBSERVATION"
+      | "RFT_OBSERVATION"
     type: TYPE
     ?value: object
           | STRING

ert/config/parsing/queue_system.py

@@ -23,3 +23,12 @@ class QueueSystem(StrEnum):
     @staticmethod
     def ert_config_case() -> str:
         return "upper"
+
+    @property
+    def formatted_name(self) -> str:
+        return {
+            self.LSF: "LSF",
+            self.LOCAL: "Local",
+            self.TORQUE: "Torque/OpenPBS",
+            self.SLURM: "Slurm",
+        }[self]

ert/config/parsing/schema_item_type.py

@@ -9,6 +9,7 @@ class SchemaItemType(StrEnum):
     POSITIVE_FLOAT = "POSITIVE_FLOAT"
     PATH = "PATH"
     EXISTING_PATH = "EXISTING_PATH"
+    EXISTING_FILE = "EXISTING_FILE"
     # EXISTING_PATH_INLINE is a directive to the
     # schema validation to inline the contents of
     # the file.

ert/config/queue_config.py

@@ -3,8 +3,8 @@ from __future__ import annotations
 import logging
 import os
 import re
-import shutil
 from abc import abstractmethod
+from textwrap import dedent
 from typing import (
     TYPE_CHECKING,
     Annotated,
@@ -51,7 +51,6 @@ class QueueOptions(
     BaseModelWithContextSupport,
     validate_assignment=True,
     extra="forbid",
-    use_enum_values=True,
     validate_default=True,
 ):
     name: QueueSystem
@@ -60,11 +59,26 @@ class QueueOptions(
     num_cpu: pydantic.NonNegativeInt = 1
     realization_memory: pydantic.NonNegativeInt = Field(
         default=0,
-        description="""When set, a job is only allowed to use realization_memory
-        of memory.
-
-        realization_memory may be an integer value, indicating the number of bytes, or a
-        string consisting of a number followed by a unit. The unit indicates the
+        description=dedent("""
+        Amount of memory to set aside for a forward model.
+
+        This information is propagated to the queue system as the amount of memory
+        to reserve/book for a realization to complete. It is up to the configuration
+        of the queuing system how to treat this information, but usually it will
+        stop more realizations being assigned to a compute node if the compute nodes
+        memory is already fully booked.
+
+        Setting this number lower than the peak memory consumption of each
+        realization puts the realization at risk of being killed in an out-of-memory
+        situation. Setting this number higher than needed will give longer wait
+        times in the queue.
+
+        For the local queue system, this keyword has no effect. In that scenario,
+        you can use `max_running`  to choke the memory consumption.
+        scheduling of compute jobs.
+
+        `realization_memory` may be an integer value, indicating the number of bytes, or
+        a string consisting of a number followed by a unit. The unit indicates the
         multiplier that is applied, and must start with one of these characters:
 
         * b, B: bytes
@@ -77,11 +91,8 @@ class QueueOptions(
         Spaces between the number and the unit are ignored, and so are any
         characters after the first. For example: 2g, 2G, and 2 GB all resolve
         to the same value: 2 gigabytes, equaling 2 * 1024**3 bytes.
-
-        If not set, or set to zero, the allowed amount of memory is unlimited.
-        """,
+        """),
     )
-    job_script: str = shutil.which("fm_dispatch.py") or "fm_dispatch.py"
     project_code: str | None = None
     activate_script: str | None = Field(default=None, validate_default=True)
 
@@ -184,7 +195,6 @@ class LsfQueueOptions(QueueOptions):
                 "max_running",
                 "num_cpu",
                 "realization_memory",
-                "job_script",
             }
         )
         driver_dict["exclude_hosts"] = driver_dict.pop("exclude_host")
@@ -212,7 +222,6 @@ class TorqueQueueOptions(QueueOptions):
                 "submit_sleep",
                 "num_cpu",
                 "realization_memory",
-                "job_script",
             }
         )
         driver_dict["queue_name"] = driver_dict.pop("queue")
@@ -241,7 +250,6 @@ class SlurmQueueOptions(QueueOptions):
                 "submit_sleep",
                 "num_cpu",
                 "realization_memory",
-                "job_script",
             }
         )
         driver_dict["sbatch_cmd"] = driver_dict.pop("sbatch")
@@ -390,6 +398,11 @@ class QueueConfig(BaseModelWithContextSupport):
         config_dict: ConfigDict,
         site_queue_options: QueueOptions | None = None,
     ) -> QueueConfig:
+        """Merge ConfigDict with QueueOptions.
+
+        A ConfigDict is parsed from an Ert configuration.
+
+        site_queue_options is overridden by config_dict"""
         site_queue_options_dict = (
             site_queue_options.model_dump(exclude_unset=True)
             if site_queue_options
@@ -406,48 +419,27 @@ class QueueConfig(BaseModelWithContextSupport):
         usr_queue_options_dict = cls._user_queue_options_from_dict(
             selected_queue_system, config_dict
         )
-        default_queue_options_dict = {
-            name: generic_option.default
-            for name, generic_option in QueueOptions.model_fields.items()
-        }
 
-        usr_job_script = usr_queue_options_dict.get("job_script")
-        site_job_script = site_queue_options_dict.get("job_script")
-
-        config_dict["JOB_SCRIPT"] = (
-            usr_job_script
-            or site_job_script
-            or shutil.which("fm_dispatch.py")
-            or "fm_dispatch.py"
-        )
+        merged_queue_options_dict = (
+            site_queue_options_dict
+            if str(selected_queue_system) == site_queue_system
+            else {}
+        ) | usr_queue_options_dict
 
         max_submit: int = config_dict.get(ConfigKeys.MAX_SUBMIT, 1)
         stop_long_running = config_dict.get(ConfigKeys.STOP_LONG_RUNNING, False)
 
-        usr_num_cpu = usr_queue_options_dict.get("num_cpu")
-        site_num_cpu = site_queue_options_dict.get("num_cpu")
-
-        if usr_num_cpu is not None:
-            config_dict["NUM_CPU"] = usr_num_cpu
-        elif (
-            site_num_cpu is None
-            and (data_file := config_dict.get(ConfigKeys.DATA_FILE))
-            and (num_cpu := get_num_cpu_from_data_file(data_file))
+        if "num_cpu" in merged_queue_options_dict:
+            config_dict["NUM_CPU"] = merged_queue_options_dict.get("num_cpu")
+        elif (data_file := config_dict.get(ConfigKeys.DATA_FILE)) and (
+            num_cpu := get_num_cpu_from_data_file(data_file)
         ):
             logger.info(f"Parsed NUM_CPU={num_cpu} from {data_file}")
-            usr_queue_options_dict[ConfigKeys.NUM_CPU] = num_cpu
+            merged_queue_options_dict[ConfigKeys.NUM_CPU] = num_cpu
 
         selected_queue_options = QueueOptions.create_queue_options(
             selected_queue_system,
-            (
-                default_queue_options_dict
-                | (
-                    site_queue_options_dict
-                    if str(selected_queue_system) == site_queue_system
-                    else {}
-                )
-                | usr_queue_options_dict
-            ),
+            merged_queue_options_dict,
             True,
         )
 
@@ -485,11 +477,11 @@ class QueueConfig(BaseModelWithContextSupport):
 
         # validate all queue options for the unselected queues
         # and show a warning
-        for _queue_system in QueueSystem:
-            if _queue_system != selected_queue_system:
+        for queue_system in QueueSystem:
+            if queue_system != selected_queue_system:
                 _ = QueueOptions.create_queue_options(
-                    _queue_system,
-                    grouped_queue_options[_queue_system],
+                    queue_system,
+                    grouped_queue_options[queue_system],
                     False,
                 )
 

ert/config/response_config.py

@@ -31,7 +31,6 @@ class ResponseMetadata(BaseModel):
 
 class ResponseConfig(BaseModel):
     type: str
-    name: str
     input_files: list[str] = Field(default_factory=list)
     keys: list[str] = Field(default_factory=list)
     has_finalized_keys: bool = False
@@ -60,13 +59,6 @@ class ResponseConfig(BaseModel):
     def expected_input_files(self) -> list[str]:
         """Returns a list of filenames expected to be produced by the forward model"""
 
-    @property
-    @abstractmethod
-    def response_type(self) -> str:
-        """Label to identify what kind of response it is.
-        Must not overlap with that of other response configs."""
-        ...
-
     @property
     @abstractmethod
     def primary_key(self) -> list[str]:

ert/config/rft_config.py

@@ -0,0 +1,275 @@
+from __future__ import annotations
+
+import datetime
+import fnmatch
+import logging
+import os
+import re
+from collections import defaultdict
+from typing import IO, Any, Literal
+
+import numpy as np
+import numpy.typing as npt
+import polars as pl
+from pydantic import Field
+from resfo_utilities import CornerpointGrid, InvalidRFTError, RFTReader
+
+from ert.substitutions import substitute_runpath_name
+
+from .parsing import ConfigDict, ConfigKeys, ConfigValidationError, ConfigWarning
+from .response_config import InvalidResponseFile, ResponseConfig, ResponseMetadata
+from .responses_index import responses_index
+
+logger = logging.getLogger(__name__)
+
+
+class RFTConfig(ResponseConfig):
+    type: Literal["rft"] = "rft"
+    name: str = "rft"
+    has_finalized_keys: bool = False
+    data_to_read: dict[str, dict[str, list[str]]] = Field(default_factory=dict)
+    locations: list[tuple[float, float, float]] = Field(default_factory=list)
+
+    @property
+    def metadata(self) -> list[ResponseMetadata]:
+        return [
+            ResponseMetadata(
+                response_type=self.name,
+                response_key=response_key,
+                filter_on=None,
+                finalized=self.has_finalized_keys,
+            )
+            for response_key in self.keys
+        ]
+
+    @property
+    def expected_input_files(self) -> list[str]:
+        base = self.input_files[0]
+        if base.upper().endswith(".DATA"):
+            # For backwards compatibility, it is
+            # allowed to give REFCASE and ECLBASE both
+            # with and without .DATA extensions
+            base = base[:-5]
+
+        return [f"{base}.RFT"]
+
+    def _find_indices(
+        self, egrid_file: str | os.PathLike[str] | IO[Any]
+    ) -> dict[tuple[int, int, int] | None, set[tuple[float, float, float]]]:
+        indices = defaultdict(set)
+        for a, b in zip(
+            CornerpointGrid.read_egrid(egrid_file).find_cell_containing_point(
+                self.locations
+            ),
+            self.locations,
+            strict=True,
+        ):
+            indices[a].add(b)
+        return indices
+
+    def read_from_file(self, run_path: str, iens: int, iter_: int) -> pl.DataFrame:
+        filename = substitute_runpath_name(self.input_files[0], iens, iter_)
+        if filename.upper().endswith(".DATA"):
+            # For backwards compatibility, it is
+            # allowed to give REFCASE and ECLBASE both
+            # with and without .DATA extensions
+            filename = filename[:-5]
+        grid_filename = f"{run_path}/{filename}"
+        if grid_filename.upper().endswith(".RFT"):
+            grid_filename = grid_filename[:-4]
+        grid_filename += ".EGRID"
+        fetched: dict[tuple[str, datetime.date], dict[str, npt.NDArray[np.float32]]] = (
+            defaultdict(dict)
+        )
+        indices = {}
+        if self.locations:
+            indices = self._find_indices(grid_filename)
+        if None in indices:
+            raise InvalidResponseFile(
+                f"Did not find grid coordinate for location(s) {indices[None]}"
+            )
+        # This is a somewhat complicated optimization in order to
+        # support wildcards in well names, dates and properties
+        # A python for loop is too slow so we use a compiled regex
+        # instead
+        if not self.data_to_read:
+            return pl.DataFrame(
+                {
+                    "response_key": [],
+                    "time": [],
+                    "depth": [],
+                    "values": [],
+                    "location": [],
+                }
+            )
+
+        sep = "\x31"
+
+        def _translate(pat: str) -> str:
+            """Translates fnmatch pattern to match anywhere"""
+            return fnmatch.translate(pat).replace("\\z", "").replace("\\Z", "")
+
+        def _props_matcher(props: list[str]) -> str:
+            """Regex for matching given props _and_ DEPTH"""
+            pattern = f"({'|'.join(_translate(p) for p in props)})"
+            if re.fullmatch(pattern, "DEPTH") is None:
+                return f"({'|'.join(_translate(p) for p in [*props, 'DEPTH'])})"
+            else:
+                return pattern
+
+        matcher = re.compile(
+            "|".join(
+                "("
+                + re.escape(sep).join(
+                    (
+                        _translate(well),
+                        _translate(time),
+                        _props_matcher(props),
+                    )
+                )
+                + ")"
+                for well, inner_dict in self.data_to_read.items()
+                for time, props in inner_dict.items()
+            )
+        )
+        locations = {}
+        try:
+            with RFTReader.open(f"{run_path}/{filename}") as rft:
+                for entry in rft:
+                    date = entry.date
+                    well = entry.well
+                    for rft_property in entry:
+                        key = f"{well}{sep}{date}{sep}{rft_property}"
+                        if matcher.fullmatch(key) is not None:
+                            values = entry[rft_property]
+                            locations[well, date] = [
+                                list(
+                                    indices.get(
+                                        (c[0] - 1, c[1] - 1, c[2] - 1),
+                                        [(None, None, None)],
+                                    )
+                                )
+                                for c in entry.connections
+                            ]
+                            if np.isdtype(values.dtype, np.float32):
+                                fetched[well, date][rft_property] = values
+        except (FileNotFoundError, InvalidRFTError) as err:
+            raise InvalidResponseFile(
+                f"Could not read RFT from {run_path}/{filename}: {err}"
+            ) from err
+
+        if not fetched:
+            return pl.DataFrame(
+                {
+                    "response_key": [],
+                    "time": [],
+                    "depth": [],
+                    "values": [],
+                    "location": [],
+                }
+            )
+
+        try:
+            df = pl.concat(
+                [
+                    pl.DataFrame(
+                        {
+                            "response_key": [f"{well}:{time.isoformat()}:{prop}"],
+                            "time": [time],
+                            "depth": [fetched[well, time]["DEPTH"]],
+                            "values": [vals],
+                            "location": pl.Series(
+                                [
+                                    locations.get(
+                                        (well, time), [(None, None, None)] * len(vals)
+                                    )
+                                ],
+                                dtype=pl.Array(
+                                    pl.List(pl.Array(pl.Float32, 3)), len(vals)
+                                ),
+                            ),
+                        }
+                    )
+                    .explode("depth", "values", "location")
+                    .explode("location")
+                    for (well, time), inner_dict in fetched.items()
+                    for prop, vals in inner_dict.items()
+                    if prop != "DEPTH"
+                ]
+            )
+        except KeyError as err:
+            raise InvalidResponseFile(
+                f"Could not find {err.args[0]} in RFTFile {filename}"
+            ) from err
+
+        return df.with_columns(
+            east=pl.col("location").arr.get(0),
+            north=pl.col("location").arr.get(1),
+            tvd=pl.col("location").arr.get(2),
+        ).drop("location")
+
+    @property
+    def response_type(self) -> str:
+        return "rft"
+
+    @property
+    def primary_key(self) -> list[str]:
+        return ["east", "north", "tvd"]
+
+    @classmethod
+    def from_config_dict(cls, config_dict: ConfigDict) -> RFTConfig | None:
+        if rfts := config_dict.get(ConfigKeys.RFT, []):
+            eclbase: str | None = config_dict.get("ECLBASE")
+            if eclbase is None:
+                raise ConfigValidationError(
+                    "In order to use rft responses, ECLBASE has to be set."
+                )
+            fm_steps = config_dict.get(ConfigKeys.FORWARD_MODEL, [])
+            names = [fm_step[0] for fm_step in fm_steps]
+            simulation_step_exists = any(
+                any(sim in name.lower() for sim in ["eclipse", "flow"])
+                for name in names
+            )
+            if not simulation_step_exists:
+                ConfigWarning.warn(
+                    "Config contains a RFT key but no forward model "
+                    "step known to generate rft files"
+                )
+
+            declared_data: dict[str, dict[datetime.date, list[str]]] = defaultdict(
+                lambda: defaultdict(list)
+            )
+            for rft in rfts:
+                for expected in ["WELL", "DATE", "PROPERTIES"]:
+                    if expected not in rft:
+                        raise ConfigValidationError.with_context(
+                            f"For RFT keyword {expected} must be specified.", rft
+                        )
+                well = rft["WELL"]
+                props = [p.strip() for p in rft["PROPERTIES"].split(",")]
+                time = rft["DATE"]
+                declared_data[well][time] += props
+            data_to_read = {
+                well: {time: sorted(set(p)) for time, p in inner_dict.items()}
+                for well, inner_dict in declared_data.items()
+            }
+            keys = sorted(
+                {
+                    f"{well}:{time}:{p}"
+                    for well, inner_dict in declared_data.items()
+                    for time, props in inner_dict.items()
+                    for p in props
+                }
+            )
+
+            return cls(
+                name="rft",
+                input_files=[eclbase.replace("%d", "<IENS>")],
+                keys=keys,
+                data_to_read=data_to_read,
+            )
+
+        return None
+
+
+responses_index.add_response_type(RFTConfig)

ert/config/summary_config.py

@@ -19,14 +19,13 @@ logger = logging.getLogger(__name__)
 
 class SummaryConfig(ResponseConfig):
     type: Literal["summary"] = "summary"
-    name: str = "summary"
     has_finalized_keys: bool = False
 
     @property
     def metadata(self) -> list[ResponseMetadata]:
         return [
             ResponseMetadata(
-                response_type=self.name,
+                response_type=self.type,
                 response_key=response_key,
                 filter_on=None,
                 finalized=self.has_finalized_keys,
@@ -72,10 +71,6 @@ class SummaryConfig(ResponseConfig):
         df = df.sort(by=["time"])
         return df
 
-    @property
-    def response_type(self) -> str:
-        return "summary"
-
     @property
     def primary_key(self) -> list[str]:
         return ["time"]
@@ -91,8 +86,8 @@ class SummaryConfig(ResponseConfig):
             fm_steps = config_dict.get(ConfigKeys.FORWARD_MODEL, [])
             names = [fm_step[0] for fm_step in fm_steps]
             simulation_step_exists = any(
-                any(sim in _name.lower() for sim in ["eclipse", "flow"])
-                for _name in names
+                any(sim in name.lower() for sim in ["eclipse", "flow"])
+                for name in names
             )
             if not simulation_step_exists:
                 ConfigWarning.warn(