ert 17.0.0__py3-none-any.whl → 19.0.0rc2__py3-none-any.whl

ert/storage/local_storage.py

@@ -5,14 +5,14 @@ import logging
 import os
 import re
 import shutil
+import types
 from collections.abc import Generator, MutableSequence
 from datetime import datetime
 from functools import cached_property
 from pathlib import Path
 from tempfile import NamedTemporaryFile
 from textwrap import dedent
-from types import TracebackType
-from typing import Any
+from typing import Any, Self
 from uuid import UUID, uuid4
 
 import polars as pl
@@ -20,6 +20,7 @@ import xarray as xr
 from filelock import FileLock, Timeout
 from pydantic import BaseModel, Field
 
+import ert.storage
 from ert.config import ErtConfig, ParameterConfig, ResponseConfig
 from ert.shared import __version__
 
@@ -30,7 +31,7 @@ from .realization_storage_state import RealizationStorageState
 
 logger = logging.getLogger(__name__)
 
-_LOCAL_STORAGE_VERSION = 15
+_LOCAL_STORAGE_VERSION = 21
 
 
 class _Migrations(BaseModel):
@@ -64,8 +65,7 @@ class LocalStorage(BaseMode):
         self,
         path: str | os.PathLike[str],
         mode: Mode,
-        *,
-        ignore_migration_check: bool = False,
+        stage_for_migration: bool = False,
     ) -> None:
         """
         Initializes the LocalStorage instance.
@@ -76,19 +76,22 @@ class LocalStorage(BaseMode):
             The file system path to the storage.
         mode : Mode
             The access mode for the storage (read/write).
-        ignore_migration_check : bool
-            If True, skips migration checks during initialization.
+        stage_for_migration : bool
+            Whether to avoid reloading storage to allow migration
         """
 
-        super().__init__(mode)
         self.path = Path(path).absolute()
+        super().__init__(mode)
+
+        if mode.can_write:
+            self._acquire_lock()
 
-        self._experiments: dict[UUID, LocalExperiment]
-        self._ensembles: dict[UUID, LocalEnsemble]
-        self._index: _Index
+        self._experiments: dict[UUID, LocalExperiment] = {}
+        self._ensembles: dict[UUID, LocalEnsemble] = {}
+        self._index: _Index = _Index()
 
         try:
-            version = _storage_version(self.path)
+            self.version = _storage_version(self.path)
         except FileNotFoundError as err:
             # No index json, will have a problem if other components of storage exists
             errors = []
@@ -100,28 +103,45 @@ class LocalStorage(BaseMode):
                 errors.append(f"ensemble path: {self.path / self.ENSEMBLES_PATH}")
             if errors:
                 raise ValueError(f"No index.json, but found: {errors}") from err
-            version = _LOCAL_STORAGE_VERSION
+            self.version = _LOCAL_STORAGE_VERSION
 
-        if version > _LOCAL_STORAGE_VERSION:
-            raise RuntimeError(
-                f"Cannot open storage '{self.path}': Storage version {version} "
-                f"is newer than the current version {_LOCAL_STORAGE_VERSION}, "
-                "upgrade ert to continue, or run with a different ENSPATH"
-            )
-        if self.can_write:
-            self._acquire_lock()
-            if version < _LOCAL_STORAGE_VERSION and not ignore_migration_check:
-                self._migrate(version)
-            self._index = self._load_index()
-            self._save_index()
-        elif version < _LOCAL_STORAGE_VERSION:
+        if self.check_migration_needed(Path(self.path)) and not self.can_write:
             raise RuntimeError(
                 f"Cannot open storage '{self.path}' in read-only mode: "
-                f"Storage version {version} is too old. Run ert to initiate migration."
+                f"Storage version {self.version} is too old. "
+                f"Run ert to initiate migration."
+            )
+
+        if not stage_for_migration:
+            self.reload()
+
+            if mode.can_write:
+                self._save_index()
+
+    @staticmethod
+    def check_migration_needed(storage_dir: Path) -> bool:
+        try:
+            version = _storage_version(storage_dir)
+        except FileNotFoundError:
+            version = _LOCAL_STORAGE_VERSION
+
+        if version > _LOCAL_STORAGE_VERSION:
+            raise ert.storage.ErtStorageException(
+                f"Cannot open storage '{storage_dir.absolute()}': Storage version "
+                f"{version} is newer than the current version {_LOCAL_STORAGE_VERSION}"
+                f", upgrade ert to continue, or run with a different ENSPATH"
             )
-        self.refresh()
 
-    def refresh(self) -> None:
+        return version < _LOCAL_STORAGE_VERSION
+
+    @staticmethod
+    def perform_migration(path: Path) -> None:
+        if LocalStorage.check_migration_needed(path):
+            with LocalStorage(path, Mode("w"), True) as storage:
+                storage._migrate(storage.version)
+                storage.reload()
+
+    def reload(self) -> None:
         """
         Reloads the index, experiments, and ensembles from the storage.
 
@@ -257,14 +277,14 @@ class LocalStorage(BaseMode):
     def _swap_path(self) -> Path:
         return self.path / self.SWAP_PATH
 
-    def __enter__(self) -> LocalStorage:
+    def __enter__(self) -> Self:
         return self
 
     def __exit__(
         self,
-        exception: Exception,
-        exception_type: type[Exception],
-        traceback: TracebackType,
+        exception: type[BaseException] | None,
+        exception_type: BaseException | None,
+        traceback: types.TracebackType | None,
     ) -> None:
         self.close()
 
@@ -283,22 +303,16 @@ class LocalStorage(BaseMode):
     def close(self) -> None:
         """
         Closes the storage, releasing any acquired locks and saving the index.
-
         This method should be called to cleanly close the storage, especially
         when it was opened in write mode. Failing to call this method may leave
         a lock file behind, which would interfere with subsequent access to
         the storage.
         """
-
         self._ensembles.clear()
         self._experiments.clear()
 
-        if not self.can_write:
-            return
-
-        self._save_index()
-
         if self.can_write:
+            self._save_index()
             self._release_lock()
 
     def _release_lock(self) -> None:
@@ -497,6 +511,12 @@ class LocalStorage(BaseMode):
             to13,
             to14,
             to15,
+            to16,
+            to17,
+            to18,
+            to19,
+            to20,
+            to21,
         )
 
         try:
@@ -526,7 +546,7 @@ class LocalStorage(BaseMode):
 
                 self._index = self._load_index()
 
-                logger.info("storage backed up for version less than 6")
+                logger.info("Storage backed up for version less than 5")
                 print(self._legacy_storage_migration_message(bkup_path, "14.6.*"))
                 return None
             elif version < _LOCAL_STORAGE_VERSION:
@@ -541,6 +561,12 @@ class LocalStorage(BaseMode):
                     12: to13,
                     13: to14,
                     14: to15,
+                    15: to16,
+                    16: to17,
+                    17: to18,
+                    18: to19,
+                    19: to20,
+                    20: to21,
                 }
                 for from_version in range(version, _LOCAL_STORAGE_VERSION):
                     migrations[from_version].migrate(self.path)
@@ -558,7 +584,7 @@ class LocalStorage(BaseMode):
         Get a unique experiment name
 
         If an experiment with the given name exists an _0 is appended
-        or _n+1 where n is the the largest postfix found for the given experiment name
+        or _n+1 where n is the largest postfix found for the given experiment name
         """
         if not experiment_name:
             return self.get_unique_experiment_name("default")

ert/storage/migration/to11.py

@@ -44,7 +44,7 @@ def migrate(path: Path) -> None:
                         array = ds.isel(realizations=0, drop=True)["values"]
                         realization = int(real_dir.name.split("-")[1])
 
-                        def parse_value(value: float | int | str) -> float | int | str:
+                        def parse_value(value: float | str) -> float | int | str:
                             if isinstance(value, float | int):
                                 return value
                             try:

ert/storage/migration/to16.py

@@ -0,0 +1,38 @@
+import json
+from pathlib import Path
+from typing import Any
+
+info = "Remove mask file from field config"
+
+
+def migrate_field_param(parameters_json: dict[str, Any]) -> dict[str, Any]:
+    new_configs = {}
+    for param_config in parameters_json.values():
+        if param_config["type"] == "field":
+            del param_config["mask_file"]
+
+        new_configs[param_config["name"]] = param_config
+    return new_configs
+
+
+def delete_mask_file(experiment: Path) -> None:
+    # delete grid mask file if it exists
+    grid_mask_file = experiment / "grid_mask.npy"
+    if grid_mask_file.exists():
+        grid_mask_file.unlink()
+
+
+def migrate_fields(experiment: Path) -> None:
+    with open(experiment / "parameter.json", encoding="utf-8") as fin:
+        parameters_json = json.load(fin)
+
+    new_parameter_configs = migrate_field_param(parameters_json)
+    Path(experiment / "parameter.json").write_text(
+        json.dumps(new_parameter_configs, indent=2), encoding="utf-8"
+    )
+
+
+def migrate(path: Path) -> None:
+    for experiment in path.glob("experiments/*"):
+        migrate_fields(experiment)
+        delete_mask_file(experiment)

ert/storage/migration/to17.py

@@ -0,0 +1,42 @@
+import json
+from pathlib import Path
+from typing import Any
+
+info = "Migrate realization error type from IntEnum to StrEnum"
+
+
+def migrate_realization_errors_json_content(
+    error_json: dict[str, Any],
+) -> dict[str, Any]:
+    int_to_str = {
+        1: "undefined",
+        2: "parameters_loaded",
+        4: "responses_loaded",
+        8: "failure_in_current",
+        16: "failure_in_parent",
+        # To cover cases  who ran with storage version 16
+        # and StrEnum _RealizationStorageState
+        "undefined": "undefined",
+        "parameters_loaded": "parameters_loaded",
+        "responses_loaded": "responses_loaded",
+        "failure_in_current": "failure_in_current",
+        "failure_in_parent": "failure_in_parent",
+    }
+    return error_json | {"type": int_to_str[error_json["type"]]}
+
+
+def migrate_realization_errors(path: Path) -> None:
+    for realization_error in path.glob("ensembles/*/realization-*/error.json"):
+        old_error_json_content = json.loads(
+            realization_error.read_text(encoding="utf-8")
+        )
+        realization_error.write_text(
+            json.dumps(
+                migrate_realization_errors_json_content(old_error_json_content),
+                indent=2,
+            )
+        )
+
+
+def migrate(path: Path) -> None:
+    migrate_realization_errors(path)

ert/storage/migration/to18.py

@@ -0,0 +1,11 @@
+from pathlib import Path
+
+info = (
+    "Added localization attributes to summary observations."
+    "Added RFT observations."
+    "No change to current storage, only additions. "
+)
+
+
+def migrate(path: Path) -> None:
+    pass

ert/storage/migration/to19.py

@@ -0,0 +1,34 @@
+import json
+from pathlib import Path
+from typing import Any
+
+info = "Add dimensionality attribute to parameters"
+
+
+def migrate_param(parameters_json: dict[str, Any]) -> dict[str, Any]:
+    new_configs = {}
+    for param_config in parameters_json.values():
+        if param_config["type"] == "surface":
+            param_config["dimensionality"] = 2
+        elif param_config["type"] == "field":
+            param_config["dimensionality"] = 3
+        else:
+            param_config["dimensionality"] = 1
+
+        new_configs[param_config["name"]] = param_config
+    return new_configs
+
+
+def migrate_parameters_for_experiment(experiment: Path) -> None:
+    with open(experiment / "parameter.json", encoding="utf-8") as fin:
+        parameters_json = json.load(fin)
+
+    new_parameter_configs = migrate_param(parameters_json)
+    Path(experiment / "parameter.json").write_text(
+        json.dumps(new_parameter_configs, indent=2), encoding="utf-8"
+    )
+
+
+def migrate(path: Path) -> None:
+    for experiment in path.glob("experiments/*"):
+        migrate_parameters_for_experiment(experiment)

ert/storage/migration/to20.py

@@ -0,0 +1,23 @@
+import json
+from pathlib import Path
+from typing import Any
+
+info = "Remove redundant .name attribute from responses."
+
+
+def config_without_name_attr(config: dict[str, Any]) -> dict[str, Any]:
+    new_json = {**config}
+    new_json.pop("name", None)
+
+    return new_json
+
+
+def migrate(path: Path) -> None:
+    for response_json_path in path.glob("experiments/*/responses.json"):
+        old_json = json.loads((response_json_path).read_text(encoding="utf-8"))
+        new_json = {
+            response_type: config_without_name_attr(config)
+            for response_type, config in old_json.items()
+        }
+
+        response_json_path.write_text(json.dumps(new_json, indent=2), encoding="utf-8")

ert/storage/migration/to21.py

@@ -0,0 +1,25 @@
+import json
+from pathlib import Path
+from typing import Any
+
+info = "Remove refcase from summary response configs"
+
+
+def config_without_refcase(summary_config: dict[str, Any]) -> dict[str, Any]:
+    new_json = {**summary_config}
+    new_json.pop("refcase", None)
+
+    return new_json
+
+
+def migrate(path: Path) -> None:
+    for response_json_path in path.glob("experiments/*/responses.json"):
+        old_json = json.loads((response_json_path).read_text(encoding="utf-8"))
+        new_json = {
+            response_type: config_without_refcase(response_config)
+            if response_config["type"] == "summary"
+            else response_config
+            for response_type, response_config in old_json.items()
+        }
+
+        response_json_path.write_text(json.dumps(new_json, indent=2), encoding="utf-8")

ert/storage/migration/to8.py

@@ -138,14 +138,14 @@ def _migrate_observations_to_grouped_parquet(path: Path) -> None:
 
         for response_type in ["gen_data", "summary"]:
             infos = [
-                _info for _info in obs_ds_infos if _info.response_type == response_type
+                info_ for info_ in obs_ds_infos if info_.response_type == response_type
             ]
             if len(infos) > 0:
-                concatd_df = pl.concat([_info.polars_df for _info in infos])
+                concatd_df = pl.concat([info_.polars_df for info_ in infos])
                 concatd_df.write_parquet(experiment / "observations" / response_type)
 
-                for _info in infos:
-                    os.remove(_info.original_ds_path)
+                for info_ in infos:
+                    os.remove(info_.original_ds_path)
 
 
 def migrate(path: Path) -> None:

ert/substitutions.py

@@ -4,6 +4,7 @@ import logging
 import re
 from collections import UserDict
 from collections.abc import Mapping
+from typing import Self
 
 logger = logging.getLogger(__name__)
 _PATTERN = re.compile(r"<[^<>]+>")
@@ -25,44 +26,27 @@ class Substitutions(UserDict[str, str]):
         """
         return _substitute(self, to_substitute, context, max_iterations, warn_max_iter)
 
-    @staticmethod
-    def substitute_parameters(
-        to_substitute: str, parameter_values: Mapping[str, Mapping[str, str | float]]
-    ) -> str:
-        """Applies the substitution '<param_name>' to parameter value
-        Args:
-            parameter_values: Mapping from parameter name to parameter value
-            to_substitute: string to substitute magic strings in
-        Returns:
-            substituted string
-        """
-        for values in parameter_values.values():
-            for param_name, value in values.items():
-                if isinstance(value, (int, float)):
-                    formatted_value = f"{value:.6g}"
-                else:
-                    formatted_value = str(value)
-                to_substitute = to_substitute.replace(
-                    f"<{param_name}>", formatted_value
-                )
-        return to_substitute
-
-    def substitute_real_iter(
-        self, to_substitute: str, realization: int, iteration: int
-    ) -> str:
+    def real_iter_substituter(self, realization: int, iteration: int) -> Self:
         extra_data = {
             "<IENS>": str(realization),
             "<ITER>": str(iteration),
         }
 
-        model_id_key = f"<GEO_ID_{realization}_{iteration}>"
+        model_id_key = f"<REALIZATION_ID_{realization}_{iteration}>"
         if model_id_key in self:
-            extra_data["<GEO_ID>"] = self[model_id_key]
+            extra_data["<REALIZATION_ID>"] = self[model_id_key]
         sim_id_key = f"<SIM_DIR_{realization}_{iteration}>"
         if sim_id_key in self:
             extra_data["<SIM_DIR>"] = self[sim_id_key]
 
-        return Substitutions({**self, **extra_data}).substitute(to_substitute)
+        return type(self)({**self, **extra_data})
+
+    def substitute_real_iter(
+        self, to_substitute: str, realization: int, iteration: int
+    ) -> str:
+        return self.real_iter_substituter(realization, iteration).substitute(
+            to_substitute
+        )
 
     def _concise_representation(self) -> str:
         return (

ert/validation/active_range.py

@@ -47,19 +47,19 @@ class ActiveRange:
             raise ValueError(
                 f"Only digits, commas, dashes and spaces are allowed, got {rangestring}"
             )
-        for _range in rangestring.split(","):
-            if "-" in _range:
-                if len(_range.split("-")) != 2:
-                    raise ValueError(f"Invalid range specified, got {_range}")
-                realization_bounds = _range.split("-")
+        for range_ in rangestring.split(","):
+            if "-" in range_:
+                if len(range_.split("-")) != 2:
+                    raise ValueError(f"Invalid range specified, got {range_}")
+                realization_bounds = range_.split("-")
                 start = int(realization_bounds[0])
                 end = int(realization_bounds[1])
                 if end < start:
                     raise ValueError(
-                        f"Invalid direction in range specified, got {_range}"
+                        f"Invalid direction in range specified, got {range_}"
                     )
             else:
-                int(_range)
+                int(range_)
         return rangestring
 
     @classmethod

ert/validation/rangestring.py

@@ -66,11 +66,11 @@ def rangestring_to_mask(rangestring: str, length: int) -> list[bool]:
         # An empty string means no active indecies. Note that an
         # IndexRange-typed instance being None means the opposite
         return mask
-    for _range in rangestring.split(","):
-        if "-" in _range:
-            if len(_range.strip().split("-")) != 2:
-                raise ValueError(f"Wrong range syntax {_range}")
-            start, end = map(int, _range.strip().split("-"))
+    for range_ in rangestring.split(","):
+        if "-" in range_:
+            if len(range_.strip().split("-")) != 2:
+                raise ValueError(f"Wrong range syntax {range_}")
+            start, end = map(int, range_.strip().split("-"))
             if end < start:
                 raise ValueError(f"Range {start}-{end} has invalid direction")
             if end + 1 > length:
@@ -78,12 +78,12 @@ def rangestring_to_mask(rangestring: str, length: int) -> list[bool]:
                     f"Range endpoint {end} is beyond the mask length {length}"
                 )
             mask[start : end + 1] = [True] * (end + 1 - start)
-        elif _range:
-            if int(_range) + 1 > length:
+        elif range_:
+            if int(range_) + 1 > length:
                 raise ValueError(
-                    f"Realization index {_range} is beyond the mask length {length}"
+                    f"Realization index {range_} is beyond the mask length {length}"
                 )
-            mask[int(_range)] = True
+            mask[int(range_)] = True
     return mask
 
 
@@ -102,14 +102,14 @@ def rangestring_to_list(rangestring: str) -> list[int]:
     result: set[int] = set()
     if not rangestring:
         return []
-    for _range in rangestring.split(","):
-        if "-" in _range:
-            if len(_range.strip().split("-")) != 2:
-                raise ValueError(f"Wrong range syntax {_range}")
-            start, end = map(int, _range.strip().split("-"))
+    for range_ in rangestring.split(","):
+        if "-" in range_:
+            if len(range_.strip().split("-")) != 2:
+                raise ValueError(f"Wrong range syntax {range_}")
+            start, end = map(int, range_.strip().split("-"))
             if end < start:
                 raise ValueError(f"Range {start}-{end} has invalid direction")
             result.update(range(start, end + 1))
-        elif _range:
-            result.add(int(_range))
+        elif range_:
+            result.add(int(range_))
     return list(result)

ert/workflow_runner.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import logging
+import types
 from concurrent import futures
 from concurrent.futures import Future
 from typing import Any, Self
@@ -127,7 +128,7 @@ class WorkflowRunner:
         self,
         exc_type: type[BaseException] | None,
         exc_value: BaseException | None,
-        traceback: Any,
+        traceback: types.TracebackType | None,
     ) -> None:
         self.wait()
 

{ert-17.0.0.dist-info → ert-19.0.0rc2.dist-info}/METADATA

@@ -1,15 +1,14 @@
 Metadata-Version: 2.4
 Name: ert
-Version: 17.0.0
+Version: 19.0.0rc2
 Summary: Ensemble based Reservoir Tool (ERT)
 Author-email: Equinor ASA <fg_sib-scout@equinor.com>
-License: GPL-3.0
+License-Expression: GPL-3.0-only
 Project-URL: Repository, https://github.com/equinor/ert
 Platform: all
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Environment :: Other Environment
 Classifier: Intended Audience :: Science/Research
-Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
 Classifier: Natural Language :: English
 Classifier: Programming Language :: Python
 Classifier: Programming Language :: Python :: 3.11
@@ -21,6 +20,7 @@ Requires-Python: <3.14,>=3.11
 Description-Content-Type: text/markdown
 License-File: COPYING
 Requires-Dist: aiohttp
+Requires-Dist: anyio
 Requires-Dist: colorama
 Requires-Dist: cryptography
 Requires-Dist: decorator
@@ -52,15 +52,15 @@ Requires-Dist: progressbar2
 Requires-Dist: psutil
 Requires-Dist: pyarrow
 Requires-Dist: pydantic>2
-Requires-Dist: pyqt6!=6.10.0
+Requires-Dist: pyqt6<6.10.0
 Requires-Dist: python-dateutil
 Requires-Dist: python-multipart
 Requires-Dist: pyyaml
 Requires-Dist: pyzmq
 Requires-Dist: requests
 Requires-Dist: resfo>=5.0.0
-Requires-Dist: ropt-dakota<0.25,>=0.24
-Requires-Dist: ropt[pandas]<0.25,>=0.24
+Requires-Dist: ropt-dakota<0.26,>=0.25
+Requires-Dist: ropt[pandas]<0.26,>=0.25
 Requires-Dist: ruamel.yaml
 Requires-Dist: scipy>=1.10.1
 Requires-Dist: seaborn
@@ -73,7 +73,7 @@ Requires-Dist: uvicorn>=0.17.0
 Requires-Dist: websockets
 Requires-Dist: xarray
 Requires-Dist: xtgeo>=3.3.0
-Requires-Dist: resfo-utilities>=0.3.0
+Requires-Dist: resfo-utilities>=0.5.0
 Provides-Extra: dev
 Requires-Dist: furo; extra == "dev"
 Requires-Dist: hypothesis!=6.102.0,!=6.112.3,>=6.85; extra == "dev"
@@ -94,6 +94,7 @@ Requires-Dist: pytest-rerunfailures!=16.0; extra == "dev"
 Requires-Dist: pytest-snapshot; extra == "dev"
 Requires-Dist: pytest-timeout; extra == "dev"
 Requires-Dist: pytest-xdist; extra == "dev"
+Requires-Dist: pytest-durations; extra == "dev"
 Requires-Dist: pytest>6; extra == "dev"
 Requires-Dist: resdata; extra == "dev"
 Requires-Dist: rust-just; extra == "dev"
@@ -105,7 +106,7 @@ Requires-Dist: sphinx-copybutton; extra == "dev"
 Requires-Dist: sphinxcontrib.datatemplates; extra == "dev"
 Requires-Dist: json-schema-for-humans; extra == "dev"
 Requires-Dist: xlsxwriter>=3.2.3; extra == "dev"
-Requires-Dist: resfo-utilities[testing]>=0.3.0; extra == "dev"
+Requires-Dist: resfo-utilities[testing]>=0.5.0; extra == "dev"
 Provides-Extra: style
 Requires-Dist: pre-commit; extra == "style"
 Provides-Extra: types