ert 17.1.7__py3-none-any.whl → 18.0.0__py3-none-any.whl

ert/plugins/hook_implementations/workflows/gen_data_rft_export.py

@@ -1,19 +1,23 @@
+from __future__ import annotations
+
 import contextlib
 import json
 import logging
 import os
 from collections.abc import Sequence
-from typing import Any
+from typing import TYPE_CHECKING, Any
 
 import numpy as np
 import pandas as pd
 import polars as pl
-from PyQt6.QtWidgets import QCheckBox, QWidget
 
 from ert.config import ErtPlugin
 from ert.plugins import CancelPluginException
 from ert.storage import Storage
 
+if TYPE_CHECKING:
+    from PyQt6.QtWidgets import QWidget
+
 logger = logging.getLogger(__name__)
 
 
@@ -240,6 +244,10 @@ class GenDataRFTCSVExportJob(ErtPlugin):
         list_edit = ListEditBox(ensemble_with_data_dict)
         list_edit.setObjectName("list_of_ensembles")
 
+        # Lazy load qt outside of gui to allow
+        # running of ert in cli without wm
+        from PyQt6.QtWidgets import QCheckBox  # noqa: PLC0415
+
         drop_const_columns_check = QCheckBox()
         drop_const_columns_check.setChecked(False)
         drop_const_columns_check.setObjectName("drop_const_columns_check")

ert/plugins/hook_specifications/__init__.py

@@ -5,7 +5,6 @@ from .forward_model_steps import (
 from .help_resources import help_links
 from .jobs import (
     ertscript_workflow,
-    installable_jobs,
     installable_workflow_jobs,
     job_documentation,
     legacy_ertscript_workflow,
@@ -20,7 +19,6 @@ __all__ = [
     "forward_model_configuration",
     "help_links",
     "installable_forward_model_steps",
-    "installable_jobs",
     "installable_workflow_jobs",
     "job_documentation",
     "legacy_ertscript_workflow",

ert/plugins/hook_specifications/jobs.py

@@ -9,15 +9,6 @@ if TYPE_CHECKING:
     from ert.plugins.plugin_response import PluginResponse
 
 
-@no_type_check
-@hook_specification
-def installable_jobs() -> PluginResponse[dict[str, str]]:
-    """
-    :return: dict with job names as keys and path to config as value
-    :rtype: PluginResponse with data as dict[str,str]
-    """
-
-
 @no_type_check
 @hook_specification(firstresult=True)
 def job_documentation(job_name: str) -> PluginResponse[dict[str, str] | None]:

ert/plugins/plugin_manager.py

@@ -5,7 +5,6 @@ import logging
 import warnings
 from argparse import ArgumentParser
 from collections.abc import Callable, Mapping, Sequence
-from pathlib import Path
 from typing import TYPE_CHECKING, Any, Literal, TypeVar, overload
 
 import pluggy
@@ -13,15 +12,14 @@ from pydantic import BaseModel, Field
 from typing_extensions import TypedDict
 
 from ert.config import (
-    ForwardModelStep,
     ForwardModelStepDocumentation,
     ForwardModelStepPlugin,
     KnownQueueOptions,
     LegacyWorkflowConfigs,
     LocalQueueOptions,
+    SiteInstalledForwardModelStep,
     WorkflowConfigs,
     WorkflowJob,
-    forward_model_step_from_config_contents,
     workflow_job_from_file,
 )
 from ert.trace import add_span_processor
@@ -251,32 +249,9 @@ class ErtPluginManager(pluggy.PluginManager):
             return merged_dict
         return {k: v[0] for k, v in merged_dict.items()}
 
-    def get_installable_jobs(self) -> Mapping[str, str]:
-        return ErtPluginManager._merge_dicts(self.hook.installable_jobs())
-
     def _get_config_workflow_jobs(self) -> dict[str, str]:
         return ErtPluginManager._merge_dicts(self.hook.installable_workflow_jobs())
 
-    def get_documentation_for_jobs(self) -> dict[str, Any]:
-        job_docs = {
-            k: {
-                "config_file": v[0],
-                "source_package": v[1].plugin_name,
-                "source_function_name": v[1].function_name,
-            }
-            for k, v in ErtPluginManager._merge_dicts(
-                self.hook.installable_jobs(), include_plugin_data=True
-            ).items()
-        }
-        for key, value in job_docs.items():
-            value.update(
-                ErtPluginManager._evaluate_job_doc_hook(
-                    self.hook.job_documentation,
-                    key,
-                )
-            )
-        return job_docs
-
     def get_documentation_for_forward_model_steps(
         self,
     ) -> dict[str, ForwardModelStepDocumentation]:
@@ -353,7 +328,7 @@ class ErtPluginManager(pluggy.PluginManager):
 
 
 class ErtRuntimePlugins(BaseModel):
-    installed_forward_model_steps: Mapping[str, ForwardModelStep] = Field(
+    installed_forward_model_steps: Mapping[str, SiteInstalledForwardModelStep] = Field(
         default_factory=dict
     )
     installed_workflow_jobs: Mapping[str, WorkflowJob] = Field(default_factory=dict)
@@ -380,12 +355,6 @@ def get_site_plugins(
         else {}
     )
 
-    for job_name, job_path in plugin_manager.get_installable_jobs().items():
-        fm_step = forward_model_step_from_config_contents(
-            Path(job_path).read_text(encoding="utf-8"), job_path, job_name
-        )
-        all_forward_model_steps[job_name] = fm_step
-
     all_workflow_jobs: dict[str, WorkflowJob] = dict[str, WorkflowJob](
         plugin_manager.get_ertscript_workflows().get_workflows()
     ) | dict[str, WorkflowJob](

ert/resources/shell_scripts/delete_directory.py

@@ -48,8 +48,8 @@ def delete_directory(path: str) -> None:
                     for file in files:
                         delete_file(os.path.join(root, file))
 
-                    for _dir in dirs:
-                        delete_empty_directory(os.path.join(root, _dir))
+                    for dir_ in dirs:
+                        delete_empty_directory(os.path.join(root, dir_))
 
         else:
             raise OSError(f"Entry:'{path}' is not a directory")

ert/run_models/__init__.py

@@ -1,6 +1,10 @@
-from .ensemble_experiment import EnsembleExperiment
-from .ensemble_information_filter import EnsembleInformationFilter
-from .ensemble_smoother import EnsembleSmoother
+from .ensemble_experiment import EnsembleExperiment, EnsembleExperimentConfig
+from .ensemble_information_filter import (
+    EnsembleInformationFilter,
+    EnsembleInformationFilterConfig,
+)
+from .ensemble_smoother import EnsembleSmoother, EnsembleSmootherConfig
+from .evaluate_ensemble import EvaluateEnsembleConfig
 from .event import (
     RunModelEvent,
     RunModelStatusEvent,
@@ -9,21 +13,29 @@ from .event import (
     RunModelUpdateEndEvent,
 )
 from .model_factory import create_model
-from .multiple_data_assimilation import MultipleDataAssimilation
+from .multiple_data_assimilation import (
+    MultipleDataAssimilation,
+    MultipleDataAssimilationConfig,
+)
 from .run_model import (
     ErtRunError,
     RunModel,
     RunModelAPI,
     StatusEvents,
 )
-from .single_test_run import SingleTestRun
+from .single_test_run import SingleTestRun, SingleTestRunConfig
 
 __all__ = [
     "EnsembleExperiment",
+    "EnsembleExperimentConfig",
     "EnsembleInformationFilter",
+    "EnsembleInformationFilterConfig",
     "EnsembleSmoother",
+    "EnsembleSmootherConfig",
     "ErtRunError",
+    "EvaluateEnsembleConfig",
     "MultipleDataAssimilation",
+    "MultipleDataAssimilationConfig",
     "RunModel",
     "RunModelAPI",
     "RunModelEvent",
@@ -32,6 +44,7 @@ __all__ = [
     "RunModelUpdateBeginEvent",
     "RunModelUpdateEndEvent",
     "SingleTestRun",
+    "SingleTestRunConfig",
     "StatusEvents",
     "create_model",
 ]

ert/run_models/_create_run_path.py

@@ -2,9 +2,11 @@ from __future__ import annotations
 
 import json
 import logging
+import math
 import os
 import time
 from collections.abc import Iterable, Mapping
+from copy import deepcopy
 from datetime import UTC, datetime
 from pathlib import Path
 from typing import TYPE_CHECKING, Any
@@ -22,6 +24,7 @@ from ert.config import (
     SurfaceConfig,
 )
 from ert.config.design_matrix import DESIGN_MATRIX_GROUP
+from ert.config.distribution import LogNormalSettings, LogUnifSettings
 from ert.config.ert_config import create_forward_model_json
 from ert.substitutions import Substitutions, substitute_runpath_name
 from ert.utils import log_duration
@@ -78,18 +81,18 @@ def _value_export_json(
     if len(values) == 0:
         return
 
-    # Hierarchical
-    json_out: dict[str, float | dict[str, float | str]] = {
-        key: dict(param_map.items()) for key, param_map in values.items()
-    }
+    # parameter file is {param: {"value": value}}
+    json_out: dict[str, dict[str, float | str]] = {}
+    for param_map in values.values():
+        for param, value in param_map.items():
+            json_out[param] = {"value": value}
 
     # Disallow NaN from being written: ERT produces the parameters and the only
     # way for the output to be NaN is if the input is invalid or if the sampling
     # function is buggy. Either way, that would be a bug and we can report it by
     # having json throw an error.
-    json.dump(
-        json_out, path.open("w"), allow_nan=False, indent=0, separators=(", ", " : ")
-    )
+    with path.open("w") as f:
+        json.dump(json_out, f, allow_nan=False, indent=0, separators=(", ", " : "))
 
 
 def _generate_parameter_files(
@@ -128,6 +131,7 @@ def _generate_parameter_files(
         df = fs._load_scalar_keys(keys=keys, realizations=iens, transformed=True)
         scalar_data = df.to_dicts()[0]
     exports: dict[str, dict[str, float | str]] = {}
+    log_exports: dict[str, dict[str, float | str]] = {}
     for param in parameter_configs:
         # For the first iteration we do not write the parameter
         # to run path, as we expect to read if after the forward
@@ -135,16 +139,34 @@ def _generate_parameter_files(
         if param.forward_init and iteration == 0:
             continue
         export_values: dict[str, dict[str, float | str]] | None = None
+        log_export_values: dict[str, dict[str, float | str]] | None = {}
         if param.name in scalar_data:
-            export_values = {param.group_name: {param.name: scalar_data[param.name]}}
+            scalar_value = scalar_data[param.name]
+            export_values = {param.group_name: {param.name: scalar_value}}
+            if isinstance(param, GenKwConfig) and isinstance(
+                param.distribution, (LogNormalSettings, LogUnifSettings)
+            ):
+                if isinstance(scalar_value, float) and scalar_value > 0:
+                    log_value = math.log10(scalar_value)
+                    log_export_values = {
+                        f"LOG10_{param.group_name}": {param.name: log_value}
+                    }
+                else:
+                    logger.warning(
+                        "Could not export the log10 value of "
+                        f"{scalar_value} as it is invalid"
+                    )
         else:
             export_values = param.write_to_runpath(Path(run_path), iens, fs)
         if export_values:
             for group, vals in export_values.items():
                 exports.setdefault(group, {}).update(vals)
+        if log_export_values:
+            for group, vals in log_export_values.items():
+                log_exports.setdefault(group, {}).update(vals)
         continue
 
-    _value_export_txt(run_path, export_base_name, exports)
+    _value_export_txt(run_path, export_base_name, exports | log_exports)
     _value_export_json(run_path, export_base_name, exports)
     return exports
 
@@ -167,13 +189,28 @@ def _manifest_to_json(ensemble: Ensemble, iens: int, iter_: int) -> dict[str, An
     # Add expected response files to manifest
     for response_config in ensemble.experiment.response_configuration.values():
         for input_file in response_config.expected_input_files:
-            manifest[f"{response_config.response_type}_{input_file}"] = (
-                substitute_runpath_name(input_file, iens, iter_)
+            manifest[f"{response_config.type}_{input_file}"] = substitute_runpath_name(
+                input_file, iens, iter_
             )
 
     return manifest
 
 
+def _make_param_substituter(
+    substituter: Substitutions,
+    param_data: Mapping[str, Mapping[str, str | float]],
+) -> Substitutions:
+    param_substituter = deepcopy(substituter)
+    for values in param_data.values():
+        for param_name, value in values.items():
+            if isinstance(value, (int, float)):
+                formatted_value = f"{value:.6g}"
+            else:
+                formatted_value = str(value)
+            param_substituter[f"<{param_name}>"] = formatted_value
+    return param_substituter
+
+
 @log_duration(logger, logging.INFO)
 def create_run_path(
     run_args: list[RunArg],
@@ -211,25 +248,21 @@ def create_run_path(
                 ensemble.iteration,
             )
             timings["generate_parameter_files"] += time.perf_counter() - start_time
+            real_iter_substituter = substituter.real_iter_substituter(
+                run_arg.iens, ensemble.iteration
+            )
+            param_substituter = _make_param_substituter(
+                real_iter_substituter, param_data
+            )
             for (
                 source_file_content,
                 target_file,
             ) in ensemble.experiment.templates_configuration:
                 start_time = time.perf_counter()
-                target_file = substituter.substitute_real_iter(
-                    target_file, run_arg.iens, ensemble.iteration
-                )
-                result = substituter.substitute_real_iter(
-                    source_file_content,
-                    run_arg.iens,
-                    ensemble.iteration,
-                )
+                target_file = real_iter_substituter.substitute(target_file)
                 timings["substitute_real_iter"] += time.perf_counter() - start_time
                 start_time = time.perf_counter()
-                result = substituter.substitute_parameters(
-                    result,
-                    param_data,
-                )
+                result = param_substituter.substitute(source_file_content)
                 timings["substitute_parameters"] += time.perf_counter() - start_time
                 start_time = time.perf_counter()
                 target = run_path / target_file

ert/run_models/ensemble_experiment.py

@@ -13,14 +13,22 @@ from ert.config import (
     ResponseConfig,
 )
 from ert.ensemble_evaluator import EvaluatorServerConfig
-from ert.run_models.initial_ensemble_run_model import InitialEnsembleRunModel
+from ert.run_models.initial_ensemble_run_model import (
+    InitialEnsembleRunModel,
+    InitialEnsembleRunModelConfig,
+)
 from ert.storage import Ensemble
 from ert.trace import tracer
 
 logger = logging.getLogger(__name__)
 
 
-class EnsembleExperiment(InitialEnsembleRunModel):
+class EnsembleExperimentConfig(InitialEnsembleRunModelConfig):
+    target_ensemble: str
+    supports_rerunning_failed_realizations: ClassVar[bool] = True
+
+
+class EnsembleExperiment(InitialEnsembleRunModel, EnsembleExperimentConfig):
     """
     This workflow will create a new experiment and a new ensemble from
     the user configuration.<br>It will never overwrite existing ensembles, and
@@ -28,8 +36,6 @@ class EnsembleExperiment(InitialEnsembleRunModel):
     """
 
     _ensemble_id: UUID | None = PrivateAttr(None)
-    supports_rerunning_failed_realizations: ClassVar[bool] = True
-    target_ensemble: str
 
     @property
     def _ensemble(self) -> Ensemble:

ert/run_models/ensemble_information_filter.py

@@ -7,11 +7,18 @@ from ert.run_models.ensemble_smoother import EnsembleSmoother
 from ert.storage import Ensemble
 
 from ..analysis import enif_update
+from .initial_ensemble_run_model import InitialEnsembleRunModelConfig
+from .update_run_model import UpdateRunModelConfig
 
 logger = logging.getLogger(__name__)
 
 
-class EnsembleInformationFilter(EnsembleSmoother):
+class EnsembleInformationFilterConfig(
+    InitialEnsembleRunModelConfig, UpdateRunModelConfig
+): ...
+
+
+class EnsembleInformationFilter(EnsembleSmoother, EnsembleInformationFilterConfig):
     def update_ensemble_parameters(
         self, prior: Ensemble, posterior: Ensemble, weight: float
     ) -> None:

ert/run_models/ensemble_smoother.py

@@ -14,8 +14,11 @@ from ert.config import (
     ResponseConfig,
 )
 from ert.ensemble_evaluator import EvaluatorServerConfig
-from ert.run_models.initial_ensemble_run_model import InitialEnsembleRunModel
-from ert.run_models.update_run_model import UpdateRunModel
+from ert.run_models.initial_ensemble_run_model import (
+    InitialEnsembleRunModel,
+    InitialEnsembleRunModelConfig,
+)
+from ert.run_models.update_run_model import UpdateRunModel, UpdateRunModelConfig
 from ert.storage import Ensemble
 from ert.trace import tracer
 
@@ -26,7 +29,10 @@ from .run_model import ErtRunError
 logger = logging.getLogger(__name__)
 
 
-class EnsembleSmoother(UpdateRunModel, InitialEnsembleRunModel):
+class EnsembleSmootherConfig(InitialEnsembleRunModelConfig, UpdateRunModelConfig): ...
+
+
+class EnsembleSmoother(InitialEnsembleRunModel, UpdateRunModel, EnsembleSmootherConfig):
     _total_iterations: int = PrivateAttr(default=2)
 
     @tracer.start_as_current_span(f"{__name__}.run_experiment")

ert/run_models/evaluate_ensemble.py

@@ -10,12 +10,17 @@ from ert.ensemble_evaluator import EvaluatorServerConfig
 from ert.trace import tracer
 
 from ..run_arg import create_run_arguments
-from .run_model import RunModel
+from .run_model import RunModel, RunModelConfig
 
 logger = logging.getLogger(__name__)
 
 
-class EvaluateEnsemble(RunModel):
+class EvaluateEnsembleConfig(RunModelConfig):
+    ensemble_id: str
+    supports_rerunning_failed_realizations: ClassVar[bool] = True
+
+
+class EvaluateEnsemble(RunModel, EvaluateEnsembleConfig):
     """
     This workflow will evaluate ensembles which have parameters, but no simulation
     has been performed, so there are no responses. This can be used in instances
@@ -24,14 +29,11 @@ class EvaluateEnsemble(RunModel):
     ensemble, and will not reflect any changes to the user configuration on disk.
     """
 
-    ensemble_id: str
-    supports_rerunning_failed_realizations: ClassVar[bool] = True
-
     def model_post_init(self, ctx: Any) -> None:
         super().model_post_init(ctx)
         try:
             ensemble = self._storage.get_ensemble(UUID(self.ensemble_id))
-            self.start_iteration = ensemble.iteration
+            self._start_iteration = ensemble.iteration
         except KeyError as err:
             raise ValueError(f"No ensemble: {self.ensemble_id}") from err
 

ert/run_models/event.py

@@ -6,6 +6,7 @@ from uuid import UUID
 
 from pydantic import BaseModel, ConfigDict, Field, TypeAdapter
 
+from _ert.events import EnsembleEvaluationWarning
 from ert.analysis import (
     AnalysisStatusEvent,
     AnalysisTimeEvent,
@@ -15,6 +16,7 @@ from ert.ensemble_evaluator.event import (
     EndEvent,
     FullSnapshotEvent,
     SnapshotUpdateEvent,
+    StartEvent,
     WarningEvent,
 )
 
@@ -90,17 +92,19 @@ StatusEvents = (
     AnalysisStatusEvent
     | AnalysisTimeEvent
     | EndEvent
-    | EverestStatusEvent
     | EverestBatchResultEvent
+    | EverestStatusEvent
     | FullSnapshotEvent
-    | SnapshotUpdateEvent
+    | RunModelDataEvent
     | RunModelErrorEvent
     | RunModelStatusEvent
     | RunModelTimeEvent
     | RunModelUpdateBeginEvent
-    | RunModelDataEvent
     | RunModelUpdateEndEvent
+    | SnapshotUpdateEvent
+    | StartEvent
     | WarningEvent
+    | EnsembleEvaluationWarning
 )