ert 16.0.9__py3-none-any.whl → 19.0.0rc2__py3-none-any.whl

ert/storage/migration/to13.py

@@ -16,37 +16,39 @@ info = (
 def point_experiments_to_ensembles(path: Path) -> None:
     experiment_to_ensemble: dict[str, list[str]] = {}
     for ensemble in path.glob("ensembles/*"):
-        with open(ensemble / "index.json", encoding="utf-8") as fin:
-            index_ = json.load(fin)
-            ensemble_id = index_["id"]
-            experiment_id = index_["experiment_id"]
+        index_ = json.loads((ensemble / "index.json").read_text(encoding="utf-8"))
+        ensemble_id = index_["id"]
+        experiment_id = index_["experiment_id"]
 
-            if experiment_id not in experiment_to_ensemble:
-                experiment_to_ensemble[experiment_id] = []
+        if experiment_id not in experiment_to_ensemble:
+            experiment_to_ensemble[experiment_id] = []
 
-            experiment_to_ensemble[experiment_id].append(ensemble_id)
+        experiment_to_ensemble[experiment_id].append(ensemble_id)
 
     for experiment_path in path.glob("experiments/*"):
-        with open(experiment_path / "index.json", encoding="utf-8") as fin:
-            old_json = json.load(fin)
-            exp_id = old_json["id"]
+        old_json = json.loads(
+            (experiment_path / "index.json").read_text(encoding="utf-8")
+        )
+        exp_id = old_json["id"]
 
         if exp_id not in experiment_to_ensemble:
-            with open(experiment_path / "index.json", "w", encoding="utf-8") as fout:
-                json.dump(
+            (experiment_path / "index.json").write_text(
+                json.dumps(
                     old_json | {"ensembles": []},
-                    fout,
                     indent=2,
-                )
+                ),
+                encoding="utf-8",
+            )
             continue
 
-        with open(experiment_path / "index.json", "w", encoding="utf-8") as fout:
-            json.dump(
+        (experiment_path / "index.json").write_text(
+            json.dumps(
                 old_json
                 | {"ensembles": sorted(experiment_to_ensemble.get(exp_id, []))},
-                fout,
                 indent=2,
-            )
+            ),
+            encoding="utf-8",
+        )
 
 
 tfd_to_distributions = {
@@ -101,12 +103,12 @@ def migrate_genkw(path: Path) -> None:
         ensembles = path.glob("ensembles/*")
 
         experiment_id = None
-        with open(experiment / "index.json", encoding="utf-8") as f:
-            exp_index = json.load(f)
-            experiment_id = exp_index["id"]
+        exp_index = json.loads((experiment / "index.json").read_text(encoding="utf-8"))
+        experiment_id = exp_index["id"]
 
-        with open(experiment / "parameter.json", encoding="utf-8") as fin:
-            parameters_json = json.load(fin)
+        parameters_json = json.loads(
+            (experiment / "parameter.json").read_text(encoding="utf-8")
+        )
 
         new_parameter_configs = migrate_gen_kw_param(parameters_json)
         Path(experiment / "parameter.json").write_text(
@@ -115,10 +117,9 @@ def migrate_genkw(path: Path) -> None:
 
         # migrate parquet files
         for ens in ensembles:
-            with open(ens / "index.json", encoding="utf-8") as f:
-                ens_file = json.load(f)
-                if ens_file["experiment_id"] != experiment_id:
-                    continue
+            ens_file = json.loads((ens / "index.json").read_text(encoding="utf-8"))
+            if ens_file["experiment_id"] != experiment_id:
+                continue
 
             group_dfs = []
             for param_config in parameters_json.values():

ert/storage/migration/to14.py

@@ -30,9 +30,9 @@ def migrate_field_param(parameters_json: dict[str, Any]) -> dict[str, Any]:
 
 def migrate_fields(path: Path) -> None:
     for experiment in path.glob("experiments/*"):
-        with open(experiment / "parameter.json", encoding="utf-8") as fin:
-            parameters_json = json.load(fin)
-
+        parameters_json = json.loads(
+            (experiment / "parameter.json").read_text(encoding="utf-8")
+        )
         new_parameter_configs = migrate_field_param(parameters_json)
         Path(experiment / "parameter.json").write_text(
             json.dumps(new_parameter_configs, indent=2), encoding="utf-8"

ert/storage/migration/to15.py

@@ -0,0 +1,25 @@
+import json
+import pathlib
+from pathlib import Path
+from typing import Any
+
+info = "Add status field to experiment index.json files"
+
+
+def add_experiment_status_to_index_json(index_json: dict[str, Any]) -> dict[str, Any]:
+    index_json["status"] = {"message": "", "status": "completed"}
+    return index_json
+
+
+def migrate_index_json(path: Path) -> None:
+    for experiment in path.glob("experiments/*"):
+        index_path = experiment / "index.json"
+        index_data = json.loads(pathlib.Path(index_path).read_text(encoding="utf-8"))
+        pathlib.Path(index_path).write_text(
+            json.dumps(add_experiment_status_to_index_json(index_data), indent=2),
+            encoding="utf-8",
+        )
+
+
+def migrate(path: Path) -> None:
+    migrate_index_json(path)

ert/storage/migration/to16.py

@@ -0,0 +1,38 @@
+import json
+from pathlib import Path
+from typing import Any
+
+info = "Remove mask file from field config"
+
+
+def migrate_field_param(parameters_json: dict[str, Any]) -> dict[str, Any]:
+    new_configs = {}
+    for param_config in parameters_json.values():
+        if param_config["type"] == "field":
+            del param_config["mask_file"]
+
+        new_configs[param_config["name"]] = param_config
+    return new_configs
+
+
+def delete_mask_file(experiment: Path) -> None:
+    # delete grid mask file if it exists
+    grid_mask_file = experiment / "grid_mask.npy"
+    if grid_mask_file.exists():
+        grid_mask_file.unlink()
+
+
+def migrate_fields(experiment: Path) -> None:
+    with open(experiment / "parameter.json", encoding="utf-8") as fin:
+        parameters_json = json.load(fin)
+
+    new_parameter_configs = migrate_field_param(parameters_json)
+    Path(experiment / "parameter.json").write_text(
+        json.dumps(new_parameter_configs, indent=2), encoding="utf-8"
+    )
+
+
+def migrate(path: Path) -> None:
+    for experiment in path.glob("experiments/*"):
+        migrate_fields(experiment)
+        delete_mask_file(experiment)

ert/storage/migration/to17.py

@@ -0,0 +1,42 @@
+import json
+from pathlib import Path
+from typing import Any
+
+info = "Migrate realization error type from IntEnum to StrEnum"
+
+
+def migrate_realization_errors_json_content(
+    error_json: dict[str, Any],
+) -> dict[str, Any]:
+    int_to_str = {
+        1: "undefined",
+        2: "parameters_loaded",
+        4: "responses_loaded",
+        8: "failure_in_current",
+        16: "failure_in_parent",
+        # To cover cases  who ran with storage version 16
+        # and StrEnum _RealizationStorageState
+        "undefined": "undefined",
+        "parameters_loaded": "parameters_loaded",
+        "responses_loaded": "responses_loaded",
+        "failure_in_current": "failure_in_current",
+        "failure_in_parent": "failure_in_parent",
+    }
+    return error_json | {"type": int_to_str[error_json["type"]]}
+
+
+def migrate_realization_errors(path: Path) -> None:
+    for realization_error in path.glob("ensembles/*/realization-*/error.json"):
+        old_error_json_content = json.loads(
+            realization_error.read_text(encoding="utf-8")
+        )
+        realization_error.write_text(
+            json.dumps(
+                migrate_realization_errors_json_content(old_error_json_content),
+                indent=2,
+            )
+        )
+
+
+def migrate(path: Path) -> None:
+    migrate_realization_errors(path)

ert/storage/migration/to18.py

@@ -0,0 +1,11 @@
+from pathlib import Path
+
+info = (
+    "Added localization attributes to summary observations."
+    "Added RFT observations."
+    "No change to current storage, only additions. "
+)
+
+
+def migrate(path: Path) -> None:
+    pass

ert/storage/migration/to19.py

@@ -0,0 +1,34 @@
+import json
+from pathlib import Path
+from typing import Any
+
+info = "Add dimensionality attribute to parameters"
+
+
+def migrate_param(parameters_json: dict[str, Any]) -> dict[str, Any]:
+    new_configs = {}
+    for param_config in parameters_json.values():
+        if param_config["type"] == "surface":
+            param_config["dimensionality"] = 2
+        elif param_config["type"] == "field":
+            param_config["dimensionality"] = 3
+        else:
+            param_config["dimensionality"] = 1
+
+        new_configs[param_config["name"]] = param_config
+    return new_configs
+
+
+def migrate_parameters_for_experiment(experiment: Path) -> None:
+    with open(experiment / "parameter.json", encoding="utf-8") as fin:
+        parameters_json = json.load(fin)
+
+    new_parameter_configs = migrate_param(parameters_json)
+    Path(experiment / "parameter.json").write_text(
+        json.dumps(new_parameter_configs, indent=2), encoding="utf-8"
+    )
+
+
+def migrate(path: Path) -> None:
+    for experiment in path.glob("experiments/*"):
+        migrate_parameters_for_experiment(experiment)

ert/storage/migration/to20.py

@@ -0,0 +1,23 @@
+import json
+from pathlib import Path
+from typing import Any
+
+info = "Remove redundant .name attribute from responses."
+
+
+def config_without_name_attr(config: dict[str, Any]) -> dict[str, Any]:
+    new_json = {**config}
+    new_json.pop("name", None)
+
+    return new_json
+
+
+def migrate(path: Path) -> None:
+    for response_json_path in path.glob("experiments/*/responses.json"):
+        old_json = json.loads((response_json_path).read_text(encoding="utf-8"))
+        new_json = {
+            response_type: config_without_name_attr(config)
+            for response_type, config in old_json.items()
+        }
+
+        response_json_path.write_text(json.dumps(new_json, indent=2), encoding="utf-8")

ert/storage/migration/to21.py

@@ -0,0 +1,25 @@
+import json
+from pathlib import Path
+from typing import Any
+
+info = "Remove refcase from summary response configs"
+
+
+def config_without_refcase(summary_config: dict[str, Any]) -> dict[str, Any]:
+    new_json = {**summary_config}
+    new_json.pop("refcase", None)
+
+    return new_json
+
+
+def migrate(path: Path) -> None:
+    for response_json_path in path.glob("experiments/*/responses.json"):
+        old_json = json.loads((response_json_path).read_text(encoding="utf-8"))
+        new_json = {
+            response_type: config_without_refcase(response_config)
+            if response_config["type"] == "summary"
+            else response_config
+            for response_type, response_config in old_json.items()
+        }
+
+        response_json_path.write_text(json.dumps(new_json, indent=2), encoding="utf-8")

ert/storage/migration/to6.py

@@ -6,8 +6,9 @@ info = "Rename and change transfer_function_definitions"
 
 def migrate(path: Path) -> None:
     for experiment in path.glob("experiments/*"):
-        with open(experiment / "parameter.json", encoding="utf-8") as fin:
-            parameters_json = json.load(fin)
+        parameters_json = json.loads(
+            (experiment / "parameter.json").read_text(encoding="utf-8")
+        )
 
         for param in parameters_json.values():
             if "transfer_function_definitions" in param:

ert/storage/migration/to7.py

@@ -9,8 +9,9 @@ info = "Standardize response configs"
 
 def _migrate_response_configs(path: Path) -> None:
     for experiment in path.glob("experiments/*"):
-        with open(experiment / "responses.json", encoding="utf-8") as fin:
-            responses = json.load(fin)
+        responses = json.loads(
+            (experiment / "responses.json").read_text(encoding="utf-8")
+        )
 
         # If we for example do a .to2() migration
         # this will implicitly upgrade it to v7 since
@@ -58,8 +59,9 @@ def _migrate_response_configs(path: Path) -> None:
                 }
             )
 
-        with open(experiment / "responses.json", "w", encoding="utf-8") as fout:
-            json.dump(migrated, fout)
+        (experiment / "responses.json").write_text(
+            json.dumps(migrated), encoding="utf-8"
+        )
 
 
 def _ensure_coord_order(
@@ -83,13 +85,11 @@ def _migrate_response_datasets(path: Path) -> None:
         ensembles = path.glob("ensembles/*")
 
         experiment_id = None
-        with open(experiment / "index.json", encoding="utf-8") as f:
-            exp_index = json.load(f)
-            experiment_id = exp_index["id"]
+        exp_index = json.loads((experiment / "index.json").read_text(encoding="utf-8"))
+        experiment_id = exp_index["id"]
 
         responses_file = experiment / "responses.json"
-        with open(responses_file, encoding="utf-8") as f:
-            responses_obj = json.load(f)
+        responses_obj = json.loads(responses_file.read_text(encoding="utf-8"))
 
         assert responses_obj is not None, (
             f"Failed to load responses.json @ {responses_file}"
@@ -98,10 +98,9 @@ def _migrate_response_datasets(path: Path) -> None:
         gendata_keys = responses_obj.get("gen_data", {}).get("keys", [])
 
         for ens in ensembles:
-            with open(ens / "index.json", encoding="utf-8") as f:
-                ens_file = json.load(f)
-                if ens_file["experiment_id"] != experiment_id:
-                    continue
+            ens_file = json.loads((ens / "index.json").read_text(encoding="utf-8"))
+            if ens_file["experiment_id"] != experiment_id:
+                continue
 
             real_dirs = [*ens.glob("realization-*")]
 

ert/storage/migration/to8.py

@@ -63,15 +63,13 @@ def _migrate_responses_from_netcdf_to_parquet(path: Path) -> None:
         ensembles = path.glob("ensembles/*")
 
         experiment_id = None
-        with open(experiment / "index.json", encoding="utf-8") as f:
-            exp_index = json.load(f)
-            experiment_id = exp_index["id"]
+        exp_index = json.loads((experiment / "index.json").read_text(encoding="utf-8"))
+        experiment_id = exp_index["id"]
 
         for ens in ensembles:
-            with open(ens / "index.json", encoding="utf-8") as f:
-                ens_file = json.load(f)
-                if ens_file["experiment_id"] != experiment_id:
-                    continue
+            ens_file = json.loads((ens / "index.json").read_text(encoding="utf-8"))
+            if ens_file["experiment_id"] != experiment_id:
+                continue
 
             real_dirs = [*ens.glob("realization-*")]
 
@@ -140,14 +138,14 @@ def _migrate_observations_to_grouped_parquet(path: Path) -> None:
 
         for response_type in ["gen_data", "summary"]:
             infos = [
-                _info for _info in obs_ds_infos if _info.response_type == response_type
+                info_ for info_ in obs_ds_infos if info_.response_type == response_type
             ]
             if len(infos) > 0:
-                concatd_df = pl.concat([_info.polars_df for _info in infos])
+                concatd_df = pl.concat([info_.polars_df for info_ in infos])
                 concatd_df.write_parquet(experiment / "observations" / response_type)
 
-                for _info in infos:
-                    os.remove(_info.original_ds_path)
+                for info_ in infos:
+                    os.remove(info_.original_ds_path)
 
 
 def migrate(path: Path) -> None:

ert/storage/migration/to9.py

@@ -39,10 +39,11 @@ def migrate(path: Path) -> None:
                 if not config.get("has_finalized_keys"):
                     # Read a sample response and write the keys
                     for ens in ensembles:
-                        with open(ens / "index.json", encoding="utf-8") as f_ensemble:
-                            ens_file = json.load(f_ensemble)
-                            if ens_file["experiment_id"] != experiment_id:
-                                continue
+                        ens_file = json.loads(
+                            (ens / "index.json").read_text(encoding="utf-8")
+                        )
+                        if ens_file["experiment_id"] != experiment_id:
+                            continue
 
                         real_dirs = [*ens.glob("realization-*")]
 

ert/storage/realization_storage_state.py

@@ -1,9 +1,9 @@
-from enum import Enum
+from enum import StrEnum, auto
 
 
-class RealizationStorageState(Enum):
-    UNDEFINED = 1
-    PARAMETERS_LOADED = 2
-    RESPONSES_LOADED = 4
-    FAILURE_IN_CURRENT = 8
-    FAILURE_IN_PARENT = 16
+class RealizationStorageState(StrEnum):
+    UNDEFINED = auto()
+    PARAMETERS_LOADED = auto()
+    RESPONSES_LOADED = auto()
+    FAILURE_IN_CURRENT = auto()
+    FAILURE_IN_PARENT = auto()

ert/substitutions.py

@@ -4,6 +4,7 @@ import logging
 import re
 from collections import UserDict
 from collections.abc import Mapping
+from typing import Self
 
 logger = logging.getLogger(__name__)
 _PATTERN = re.compile(r"<[^<>]+>")
@@ -25,44 +26,27 @@ class Substitutions(UserDict[str, str]):
         """
         return _substitute(self, to_substitute, context, max_iterations, warn_max_iter)
 
-    @staticmethod
-    def substitute_parameters(
-        to_substitute: str, parameter_values: Mapping[str, Mapping[str, str | float]]
-    ) -> str:
-        """Applies the substitution '<param_name>' to parameter value
-        Args:
-            parameter_values: Mapping from parameter name to parameter value
-            to_substitute: string to substitute magic strings in
-        Returns:
-            substituted string
-        """
-        for values in parameter_values.values():
-            for param_name, value in values.items():
-                if isinstance(value, (int, float)):
-                    formatted_value = f"{value:.6g}"
-                else:
-                    formatted_value = str(value)
-                to_substitute = to_substitute.replace(
-                    f"<{param_name}>", formatted_value
-                )
-        return to_substitute
-
-    def substitute_real_iter(
-        self, to_substitute: str, realization: int, iteration: int
-    ) -> str:
+    def real_iter_substituter(self, realization: int, iteration: int) -> Self:
         extra_data = {
             "<IENS>": str(realization),
             "<ITER>": str(iteration),
         }
 
-        model_id_key = f"<GEO_ID_{realization}_{iteration}>"
+        model_id_key = f"<REALIZATION_ID_{realization}_{iteration}>"
         if model_id_key in self:
-            extra_data["<GEO_ID>"] = self[model_id_key]
+            extra_data["<REALIZATION_ID>"] = self[model_id_key]
         sim_id_key = f"<SIM_DIR_{realization}_{iteration}>"
         if sim_id_key in self:
             extra_data["<SIM_DIR>"] = self[sim_id_key]
 
-        return Substitutions({**self, **extra_data}).substitute(to_substitute)
+        return type(self)({**self, **extra_data})
+
+    def substitute_real_iter(
+        self, to_substitute: str, realization: int, iteration: int
+    ) -> str:
+        return self.real_iter_substituter(realization, iteration).substitute(
+            to_substitute
+        )
 
     def _concise_representation(self) -> str:
         return (

ert/validation/active_range.py

@@ -47,19 +47,19 @@ class ActiveRange:
             raise ValueError(
                 f"Only digits, commas, dashes and spaces are allowed, got {rangestring}"
             )
-        for _range in rangestring.split(","):
-            if "-" in _range:
-                if len(_range.split("-")) != 2:
-                    raise ValueError(f"Invalid range specified, got {_range}")
-                realization_bounds = _range.split("-")
+        for range_ in rangestring.split(","):
+            if "-" in range_:
+                if len(range_.split("-")) != 2:
+                    raise ValueError(f"Invalid range specified, got {range_}")
+                realization_bounds = range_.split("-")
                 start = int(realization_bounds[0])
                 end = int(realization_bounds[1])
                 if end < start:
                     raise ValueError(
-                        f"Invalid direction in range specified, got {_range}"
+                        f"Invalid direction in range specified, got {range_}"
                     )
             else:
-                int(_range)
+                int(range_)
         return rangestring
 
     @classmethod

ert/validation/ensemble_realizations_argument.py

@@ -17,11 +17,13 @@ class EnsembleRealizationsArgument(RangeStringArgument):
     def __init__(
         self,
         ensemble: Callable[[], Union["Ensemble", None]],
-        max_value: int | None,
         required_realization_storage_states: Iterable["RealizationStorageState"],
         **kwargs: bool,
     ) -> None:
-        super().__init__(max_value, **kwargs)
+        ens = ensemble()
+        size = ens.ensemble_size if ens is not None else None
+        super().__init__(size, **kwargs)
+
         self.__ensemble_getter = ensemble
         self._required_realization_storage_states: Iterable[RealizationStorageState] = (
             required_realization_storage_states

ert/validation/rangestring.py

@@ -66,11 +66,11 @@ def rangestring_to_mask(rangestring: str, length: int) -> list[bool]:
         # An empty string means no active indecies. Note that an
         # IndexRange-typed instance being None means the opposite
         return mask
-    for _range in rangestring.split(","):
-        if "-" in _range:
-            if len(_range.strip().split("-")) != 2:
-                raise ValueError(f"Wrong range syntax {_range}")
-            start, end = map(int, _range.strip().split("-"))
+    for range_ in rangestring.split(","):
+        if "-" in range_:
+            if len(range_.strip().split("-")) != 2:
+                raise ValueError(f"Wrong range syntax {range_}")
+            start, end = map(int, range_.strip().split("-"))
             if end < start:
                 raise ValueError(f"Range {start}-{end} has invalid direction")
             if end + 1 > length:
@@ -78,12 +78,12 @@ def rangestring_to_mask(rangestring: str, length: int) -> list[bool]:
                     f"Range endpoint {end} is beyond the mask length {length}"
                 )
             mask[start : end + 1] = [True] * (end + 1 - start)
-        elif _range:
-            if int(_range) + 1 > length:
+        elif range_:
+            if int(range_) + 1 > length:
                 raise ValueError(
-                    f"Realization index {_range} is beyond the mask length {length}"
+                    f"Realization index {range_} is beyond the mask length {length}"
                 )
-            mask[int(_range)] = True
+            mask[int(range_)] = True
     return mask
 
 
@@ -102,14 +102,14 @@ def rangestring_to_list(rangestring: str) -> list[int]:
     result: set[int] = set()
     if not rangestring:
         return []
-    for _range in rangestring.split(","):
-        if "-" in _range:
-            if len(_range.strip().split("-")) != 2:
-                raise ValueError(f"Wrong range syntax {_range}")
-            start, end = map(int, _range.strip().split("-"))
+    for range_ in rangestring.split(","):
+        if "-" in range_:
+            if len(range_.strip().split("-")) != 2:
+                raise ValueError(f"Wrong range syntax {range_}")
+            start, end = map(int, range_.strip().split("-"))
             if end < start:
                 raise ValueError(f"Range {start}-{end} has invalid direction")
             result.update(range(start, end + 1))
-        elif _range:
-            result.add(int(_range))
+        elif range_:
+            result.add(int(range_))
     return list(result)

ert/workflow_runner.py

@@ -1,12 +1,14 @@
 from __future__ import annotations
 
 import logging
+import types
 from concurrent import futures
 from concurrent.futures import Future
 from typing import Any, Self
 
 from ert import ErtScript
 from ert.config import (
+    BaseErtScriptWorkflow,
     ErtScriptWorkflow,
     ExternalErtScript,
     Workflow,
@@ -43,8 +45,9 @@ class WorkflowJobRunner:
                 f"{self.job.max_args} arguments, {len(arguments)} given."
             )
 
-        if isinstance(self.job, ErtScriptWorkflow):
-            self.__script = self.job.ert_script()
+        if isinstance(self.job, BaseErtScriptWorkflow):
+            ert_script_class = self.job.load_ert_script_class()
+            self.__script = ert_script_class()
             # We let stop on fail either from class or config take precedence
             self.stop_on_fail = self.job.stop_on_fail or self.__script.stop_on_fail
 
@@ -125,7 +128,7 @@ class WorkflowRunner:
         self,
         exc_type: type[BaseException] | None,
         exc_value: BaseException | None,
-        traceback: Any,
+        traceback: types.TracebackType | None,
     ) -> None:
         self.wait()