ert 16.0.9__py3-none-any.whl → 19.0.0rc2__py3-none-any.whl

ert/config/ensemble_config.py

@@ -7,30 +7,22 @@ from typing import Self
 
 from pydantic import BaseModel, Field, model_validator
 
-from .ext_param_config import ExtParamConfig
+from .everest_control import EverestControl
 from .field import Field as FieldConfig
-from .gen_data_config import GenDataConfig
 from .gen_kw_config import GenKwConfig
+from .known_response_types import KNOWN_ERT_RESPONSE_TYPES, KnownErtResponseTypes
 from .parameter_config import ParameterConfig
 from .parsing import ConfigDict, ConfigKeys, ConfigValidationError
 from .response_config import ResponseConfig
-from .summary_config import SummaryConfig
 from .surface_config import SurfaceConfig
 
-KnownResponseTypes = SummaryConfig | GenDataConfig
-
-_KNOWN_RESPONSE_TYPES = (
-    SummaryConfig,
-    GenDataConfig,
-)
-
 logger = logging.getLogger(__name__)
 
 
 class EnsembleConfig(BaseModel):
-    response_configs: dict[str, KnownResponseTypes] = Field(default_factory=dict)
+    response_configs: dict[str, KnownErtResponseTypes] = Field(default_factory=dict)
     parameter_configs: dict[
-        str, GenKwConfig | FieldConfig | SurfaceConfig | ExtParamConfig
+        str, GenKwConfig | FieldConfig | SurfaceConfig | EverestControl
     ] = Field(default_factory=dict)
 
     @model_validator(mode="after")
@@ -131,16 +123,16 @@ class EnsembleConfig(BaseModel):
             + [make_field(f) for f in field_list]
         )
         EnsembleConfig._check_for_duplicate_gen_kw_param_names(gen_kw_cfgs)
-        response_configs: list[KnownResponseTypes] = []
+        response_configs: list[KnownErtResponseTypes] = []
 
-        for config_cls in _KNOWN_RESPONSE_TYPES:
+        for config_cls in KNOWN_ERT_RESPONSE_TYPES:
             instance = config_cls.from_config_dict(config_dict)
 
             if instance is not None and instance.keys:
                 response_configs.append(instance)
 
         return cls(
-            response_configs={response.name: response for response in response_configs},
+            response_configs={response.type: response for response in response_configs},
             parameter_configs={
                 parameter.name: parameter for parameter in parameter_configs
             },
@@ -151,13 +143,13 @@ class EnsembleConfig(BaseModel):
             return self.parameter_configs[key]
         elif key in self.response_configs:
             return self.response_configs[key]
-        elif _config := next(
+        elif config := next(
             (c for c in self.response_configs.values() if key in c.keys), None
         ):
             # Only hit by blockfs migration
             # returns the same config for one call per
             # response type. Is later deduped before saving to json
-            return _config
+            return config
         else:
             raise KeyError(f"The key:{key} is not in the ensemble configuration")
 

ert/config/ert_config.py

@@ -25,6 +25,7 @@ from ._design_matrix_validator import DesignMatrixValidator
 from ._observations import (
     HistoryObservation,
     Observation,
+    RFTObservation,
     SummaryObservation,
     make_observations,
 )
@@ -36,9 +37,12 @@ from .forward_model_step import (
     ForwardModelStepJSON,
     ForwardModelStepValidationError,
     ForwardModelStepWarning,
+    SiteInstalledForwardModelStep,
+    SiteOrUserForwardModelStep,
+    UserInstalledForwardModelStep,
 )
 from .gen_data_config import GenDataConfig
-from .gen_kw_config import GenKwConfig
+from .gen_kw_config import DataSource, GenKwConfig
 from .model_config import ModelConfig
 from .parse_arg_types_list import parse_arg_types_list
 from .parsing import (
@@ -59,9 +63,11 @@ from .parsing import (
 from .parsing.observations_parser import ObservationDict
 from .queue_config import KnownQueueOptions, QueueConfig
 from .refcase import Refcase
+from .rft_config import RFTConfig
 from .workflow import Workflow
 from .workflow_fixtures import fixtures_per_hook
 from .workflow_job import (
+    BaseErtScriptWorkflow,
     ErtScriptLoadFailure,
     ErtScriptWorkflow,
     WorkflowJob,
@@ -132,6 +138,7 @@ def create_forward_model_json(
         env_pr_fm_step = {}
 
     context_substitutions = Substitutions(context)
+    real_iter_substituter = context_substitutions.real_iter_substituter(iens, itr)
 
     class Substituter:
         def __init__(self, fm_step: ForwardModelStep) -> None:
@@ -145,7 +152,7 @@ def create_forward_model_json(
             )
             self.copy_private_args = Substitutions(
                 {
-                    key: context_substitutions.substitute_real_iter(val, iens, itr)
+                    key: real_iter_substituter.substitute(val)
                     for key, val in fm_step.private_args.items()
                 }
             )
@@ -162,7 +169,7 @@ def create_forward_model_json(
             string = self.copy_private_args.substitute(
                 string, self.substitution_context_hint, 1, warn_max_iter=False
             )
-            return context_substitutions.substitute_real_iter(string, iens, itr)
+            return real_iter_substituter.substitute(string)
 
         def filter_env_dict(self, env_dict: dict[str, str]) -> dict[str, str] | None:
             substituted_dict = {}
@@ -323,63 +330,74 @@ def read_templates(config_dict: ConfigDict) -> list[tuple[str, str]]:
 
 def workflow_jobs_from_dict(
     content_dict: ConfigDict,
-    installed_workflows: dict[str, WorkflowJob] | None = None,
+    site_installed_workflows_jobs: dict[str, WorkflowJob] | None = None,
 ) -> dict[str, WorkflowJob]:
-    workflow_job_info = content_dict.get(ConfigKeys.LOAD_WORKFLOW_JOB, [])
-    workflow_job_dir_info = content_dict.get(ConfigKeys.WORKFLOW_JOB_DIRECTORY, [])
+    user_installed_workflow_job_info = content_dict.get(
+        ConfigKeys.LOAD_WORKFLOW_JOB, []
+    )
+    user_installed_workflow_job_dir_info = content_dict.get(
+        ConfigKeys.WORKFLOW_JOB_DIRECTORY, []
+    )
 
-    workflow_jobs = copy.copy(installed_workflows) if installed_workflows else {}
+    workflow_jobs = (
+        copy.copy(site_installed_workflows_jobs)
+        if site_installed_workflows_jobs
+        else {}
+    )
 
     errors: list[ErrorInfo | ConfigValidationError] = []
 
-    for workflow_job in workflow_job_info:
+    for user_workflow_job in user_installed_workflow_job_info:
         try:
             # workflow_job_from_file only throws error if a
             # non-readable file is provided.
             # Non-existing files are caught by the new parser
-            new_job = workflow_job_from_file(
-                config_file=workflow_job[0],
-                name=None if len(workflow_job) == 1 else workflow_job[1],
+            user_job = workflow_job_from_file(
+                config_file=user_workflow_job[0],
+                name=None if len(user_workflow_job) == 1 else user_workflow_job[1],
+                origin="user",
             )
-            name = new_job.name
+            name = user_job.name
             if name in workflow_jobs:
                 ConfigWarning.warn(
                     f"Duplicate workflow jobs with name {name!r}, choosing "
-                    f"{new_job.location()!r} over "
+                    f"{user_job.location()!r} over "
                     f"{workflow_jobs[name].location()!r}",
                     name,
                 )
-            workflow_jobs[name] = new_job
+            workflow_jobs[name] = user_job
         except ErtScriptLoadFailure as err:
             ConfigWarning.warn(
-                f"Loading workflow job {workflow_job[0]!r}"
+                f"Loading workflow job {user_workflow_job[0]!r}"
                 f" failed with '{err}'. It will not be loaded.",
-                workflow_job[0],
+                user_workflow_job[0],
             )
         except ConfigValidationError as err:
-            errors.append(ErrorInfo(message=str(err)).set_context(workflow_job[0]))
+            errors.append(ErrorInfo(message=str(err)).set_context(user_workflow_job[0]))
 
-    for job_path in workflow_job_dir_info:
-        for file_name in _get_files_in_directory(job_path, errors):
+    for user_job_path in user_installed_workflow_job_dir_info:
+        for user_job_file in _get_files_in_directory(user_job_path, errors):
             try:
-                new_job = workflow_job_from_file(config_file=file_name)
-                name = new_job.name
+                user_job = workflow_job_from_file(
+                    config_file=user_job_file, origin="user"
+                )
+                name = user_job.name
                 if name in workflow_jobs:
                     ConfigWarning.warn(
                         f"Duplicate workflow jobs with name {name!r}, choosing "
-                        f"{new_job.location()!r} over "
+                        f"{user_job.location()!r} over "
                         f"{workflow_jobs[name].location()!r}",
                         name,
                     )
-                workflow_jobs[name] = new_job
+                workflow_jobs[name] = user_job
             except ErtScriptLoadFailure as err:
                 ConfigWarning.warn(
-                    f"Loading workflow job {file_name!r}"
+                    f"Loading workflow job {user_job_file!r}"
                     f" failed with '{err}'. It will not be loaded.",
-                    file_name,
+                    user_job_file,
                 )
             except ConfigValidationError as err:
-                errors.append(ErrorInfo(message=str(err)).set_context(job_path))
+                errors.append(ErrorInfo(message=str(err)).set_context(user_job_path))
     if errors:
         raise ConfigValidationError.from_collected(errors)
 
@@ -387,13 +405,11 @@ def workflow_jobs_from_dict(
 
 
 def create_and_hook_workflows(
-    content_dict: ConfigDict,
+    hook_workflow_info: list[tuple[str, HookRuntime]],
+    workflow_info: list[tuple[str, str]],
     workflow_jobs: dict[str, WorkflowJob],
     substitutions: dict[str, str],
 ) -> tuple[dict[str, Workflow], defaultdict[HookRuntime, list[Workflow]]]:
-    hook_workflow_info = content_dict.get(ConfigKeys.HOOK_WORKFLOW, [])
-    workflow_info = content_dict.get(ConfigKeys.LOAD_WORKFLOW, [])
-
     workflows = {}
     hooked_workflows = defaultdict(list)
 
@@ -411,7 +427,7 @@ def create_and_hook_workflows(
             for job, args in workflow:
                 if isinstance(job, ErtScriptWorkflow):
                     try:
-                        job.ert_script.validate(args)
+                        job.load_ert_script_class().validate(args)
                     except ConfigValidationError as err:
                         errors.append(ErrorInfo(message=str(err)).set_context(work[0]))
                         continue
@@ -439,11 +455,11 @@ def create_and_hook_workflows(
         wf = workflows[hook_name]
         available_fixtures = fixtures_per_hook[mode]
         for job, _ in wf.cmd_list:
-            if not hasattr(job, "ert_script") or job.ert_script is None:
+            if not isinstance(job, BaseErtScriptWorkflow):
                 continue
 
-            assert isinstance(job, ErtScriptWorkflow)
-            ert_script_instance = job.ert_script()
+            ert_script_class = job.load_ert_script_class()
+            ert_script_instance = ert_script_class()
             requested_fixtures = ert_script_instance.requested_fixtures
 
             # Look for requested fixtures that are not available for the given
@@ -485,7 +501,7 @@ def create_and_hook_workflows(
 def workflows_from_dict(
     content_dict: ConfigDict,
     substitutions: dict[str, str],
-    installed_workflows: Mapping[str, WorkflowJob] | None = None,
+    site_installed_workflows_jobs: Mapping[str, WorkflowJob] | None = None,
 ) -> tuple[
     dict[str, WorkflowJob],
     dict[str, Workflow],
@@ -493,28 +509,31 @@ def workflows_from_dict(
 ]:
     workflow_jobs = workflow_jobs_from_dict(
         content_dict,
-        dict(copy.copy(installed_workflows)) if installed_workflows else {},
+        dict(copy.copy(site_installed_workflows_jobs))
+        if site_installed_workflows_jobs
+        else {},
     )
     workflows, hooked_workflows = create_and_hook_workflows(
-        content_dict, workflow_jobs, substitutions
+        content_dict.get(ConfigKeys.HOOK_WORKFLOW, []),
+        content_dict.get(ConfigKeys.LOAD_WORKFLOW, []),
+        workflow_jobs,
+        substitutions,
     )
     return workflow_jobs, workflows, hooked_workflows
 
 
 def installed_forward_model_steps_from_dict(
     config_dict: ConfigDict,
-) -> dict[str, ForwardModelStep]:
+) -> dict[str, UserInstalledForwardModelStep]:
     errors: list[ErrorInfo | ConfigValidationError] = []
-    fm_steps: dict[str, ForwardModelStep] = {}
+    fm_steps: dict[str, UserInstalledForwardModelStep] = {}
     for name, (fm_step_config_file, config_contents) in config_dict.get(
         ConfigKeys.INSTALL_JOB, []
     ):
         fm_step_config_file = path.abspath(fm_step_config_file)
         try:
             new_fm_step = forward_model_step_from_config_contents(
-                config_contents,
-                name=name,
-                config_file=fm_step_config_file,
+                config_contents, name=name, config_file=fm_step_config_file
             )
         except ConfigValidationError as e:
             errors.append(e)
@@ -677,6 +696,8 @@ def log_observation_keys(
 
 RESERVED_KEYWORDS = ["realization", "IENS", "ITER"]
 
+USER_CONFIG_SCHEMA = init_user_config_schema()
+
 
 class ErtConfig(BaseModel):
     DEFAULT_ENSPATH: ClassVar[str] = "storage"
@@ -688,6 +709,7 @@ class ErtConfig(BaseModel):
     QUEUE_OPTIONS: ClassVar[KnownQueueOptions | None] = None
     RESERVED_KEYWORDS: ClassVar[list[str]] = RESERVED_KEYWORDS
     ENV_VARS: ClassVar[dict[str, str]] = {}
+    PRIORITIZE_PRIVATE_IP_ADDRESS: ClassVar[bool] = False
 
     substitutions: dict[str, str] = Field(default_factory=dict)
     ensemble_config: EnsembleConfig = Field(default_factory=EnsembleConfig)
@@ -702,13 +724,11 @@ class ErtConfig(BaseModel):
         default_factory=lambda: defaultdict(lambda: cast(list[Workflow], []))
     )
     runpath_file: Path = Path(DEFAULT_RUNPATH_FILE)
+    prioritize_private_ip_address: bool = False
 
     ert_templates: list[tuple[str, str]] = Field(default_factory=list)
-    installed_forward_model_steps: dict[str, ForwardModelStep] = Field(
-        default_factory=dict
-    )
 
-    forward_model_steps: list[ForwardModelStep] = Field(default_factory=list)
+    forward_model_steps: list[SiteOrUserForwardModelStep] = Field(default_factory=list)
     runpath_config: ModelConfig = Field(default_factory=ModelConfig)
     user_config_file: str = "no_config"
     config_path: str = Field(init=False, default="")
@@ -721,6 +741,18 @@ class ErtConfig(BaseModel):
     @property
     def observations(self) -> dict[str, pl.DataFrame]:
         if self._observations is None:
+            has_rft_observations = any(
+                isinstance(o, RFTObservation) for o in self.observation_declarations
+            )
+            if (
+                has_rft_observations
+                and "rft" not in self.ensemble_config.response_configs
+            ):
+                self.ensemble_config.response_configs["rft"] = RFTConfig(
+                    input_files=[self.runpath_config.eclbase_format_string],
+                    data_to_read={},
+                    locations=[],
+                )
             computed = create_observation_dataframes(
                 self.observation_declarations,
                 self.refcase,
@@ -728,6 +760,10 @@ class ErtConfig(BaseModel):
                     GenDataConfig | None,
                     self.ensemble_config.response_configs.get("gen_data", None),
                 ),
+                cast(
+                    RFTConfig | None,
+                    self.ensemble_config.response_configs.get("rft", None),
+                ),
                 self.time_map,
                 self.history_source,
             )
@@ -779,6 +815,15 @@ class ErtConfig(BaseModel):
             )
         return self
 
+    @model_validator(mode="after")
+    def log_ensemble_config_contents(self) -> Self:
+        all_parameters = self.parameter_configurations_with_design_matrix
+        parameter_type_count = Counter(parameter.type for parameter in all_parameters)
+        logger.info(
+            f"EnsembleConfig contains parameters of type {dict(parameter_type_count)}"
+        )
+        return self
+
     def __eq__(self, other: object) -> bool:
         if not isinstance(other, ErtConfig):
             return False
@@ -805,7 +850,7 @@ class ErtConfig(BaseModel):
     def with_plugins(runtime_plugins: ErtRuntimePlugins) -> type[ErtConfig]:
         class ErtConfigWithPlugins(ErtConfig):
             PREINSTALLED_FORWARD_MODEL_STEPS: ClassVar[
-                Mapping[str, ForwardModelStep]
+                Mapping[str, SiteInstalledForwardModelStep]
             ] = runtime_plugins.installed_forward_model_steps
             PREINSTALLED_WORKFLOWS = dict(runtime_plugins.installed_workflow_jobs)
             ENV_PR_FM_STEP: ClassVar[dict[str, dict[str, Any]]] = (
@@ -815,6 +860,9 @@ class ErtConfig(BaseModel):
             )
             ENV_VARS = dict(runtime_plugins.environment_variables)
             QUEUE_OPTIONS = runtime_plugins.queue_options
+            PRIORITIZE_PRIVATE_IP_ADDRESS = (
+                runtime_plugins.prioritize_private_ip_address
+            )
 
         ErtConfigWithPlugins.model_rebuild()
         assert issubclass(ErtConfigWithPlugins, ErtConfig)
@@ -941,7 +989,9 @@ class ErtConfig(BaseModel):
             errors.append(e)
 
         try:
-            queue_config = QueueConfig.from_dict(config_dict, cls.QUEUE_OPTIONS)
+            queue_config = QueueConfig.from_dict(
+                config_dict, site_queue_options=cls.QUEUE_OPTIONS
+            )
 
             substitutions["<NUM_CPU>"] = str(queue_config.queue_options.num_cpu)
 
@@ -1011,11 +1061,30 @@ class ErtConfig(BaseModel):
                 if isinstance(cfg, GenKwConfig) and cfg.name in dm_params
             ]
             if overwrite_params:
-                ConfigWarning.warn(
-                    f"Parameters {overwrite_params} "
-                    "will be overridden by design matrix. This will cause "
-                    "updates to be turned off for these parameters."
-                )
+                param_sampled = [
+                    k
+                    for k in overwrite_params
+                    if analysis_config.design_matrix.parameter_priority[k]
+                    == DataSource.SAMPLED
+                ]
+                param_design = [
+                    k
+                    for k in overwrite_params
+                    if analysis_config.design_matrix.parameter_priority[k]
+                    == DataSource.DESIGN_MATRIX
+                ]
+                if param_sampled:
+                    ConfigWarning.warn(
+                        f"Parameters {param_sampled} "
+                        "are also defined in design matrix, but due to the sampled"
+                        " priority they will remain as such."
+                    )
+                if param_design:
+                    ConfigWarning.warn(
+                        f"Parameters {param_design} "
+                        "will be overridden by design matrix. This will cause "
+                        "updates to be turned off for these parameters."
+                    )
 
             if dm_errors:
                 raise ConfigValidationError.from_collected(dm_errors)
@@ -1046,6 +1115,19 @@ class ErtConfig(BaseModel):
             user_configured_.add(key)
             env_vars[key] = substituter.substitute(val)
 
+        prioritize_private_ip_address: bool = cls.PRIORITIZE_PRIVATE_IP_ADDRESS
+        if ConfigKeys.PRIORITIZE_PRIVATE_IP_ADDRESS in config_dict:
+            user_prioritize_private_ip_address = bool(
+                config_dict[ConfigKeys.PRIORITIZE_PRIVATE_IP_ADDRESS]
+            )
+            if prioritize_private_ip_address != user_prioritize_private_ip_address:
+                logger.warning(
+                    "PRIORITIZE_PRIVATE_IP_ADDRESS was overwritten by user: "
+                    f"{prioritize_private_ip_address} -> "
+                    f"{user_prioritize_private_ip_address}"
+                )
+                prioritize_private_ip_address = user_prioritize_private_ip_address
+
         try:
             refcase = Refcase.from_config_dict(config_dict)
             cls_config = cls(
@@ -1061,7 +1143,6 @@ class ErtConfig(BaseModel):
                 hooked_workflows=hooked_workflows,
                 runpath_file=Path(runpath_file),
                 ert_templates=read_templates(config_dict),
-                installed_forward_model_steps=installed_forward_model_steps,
                 forward_model_steps=cls._create_list_of_forward_model_steps_to_run(
                     installed_forward_model_steps,
                     substitutions,
@@ -1073,11 +1154,21 @@ class ErtConfig(BaseModel):
                 time_map=time_map,
                 history_source=history_source,
                 refcase=refcase,
+                prioritize_private_ip_address=prioritize_private_ip_address,
             )
 
             # The observations are created here because create_observation_dataframes
             # will perform additonal validation which needs the context in
             # obs_configs which is stripped by pydantic
+            has_rft_observations = any(
+                isinstance(o, RFTObservation) for o in obs_configs
+            )
+            if has_rft_observations and "rft" not in ensemble_config.response_configs:
+                ensemble_config.response_configs["rft"] = RFTConfig(
+                    input_files=[eclbase],
+                    data_to_read={},
+                    locations=[],
+                )
             cls_config._observations = create_observation_dataframes(
                 obs_configs,
                 refcase,
@@ -1085,6 +1176,10 @@ class ErtConfig(BaseModel):
                     GenDataConfig | None,
                     ensemble_config.response_configs.get("gen_data", None),
                 ),
+                cast(
+                    RFTConfig | None,
+                    ensemble_config.response_configs.get("rft", None),
+                ),
                 time_map,
                 history_source,
             )
@@ -1262,7 +1357,7 @@ class ErtConfig(BaseModel):
     @classmethod
     def _read_user_config_contents(cls, user_config: str, file_name: str) -> ConfigDict:
         return parse_contents(
-            user_config, file_name=file_name, schema=init_user_config_schema()
+            user_config, file_name=file_name, schema=USER_CONFIG_SCHEMA
         )
 
     @classmethod
@@ -1352,8 +1447,10 @@ def uppercase_subkeys_and_stringify_subvalues(
 
 
 def forward_model_step_from_config_contents(
-    config_contents: str, config_file: str, name: str | None = None
-) -> ForwardModelStep:
+    config_contents: str,
+    config_file: str,
+    name: str | None = None,
+) -> UserInstalledForwardModelStep:
     if name is None:
         name = os.path.basename(config_file)
 
@@ -1377,7 +1474,7 @@ def forward_model_step_from_config_contents(
     environment = {k: v for [k, v] in content_dict.get("ENV", [])}
     default_mapping = {k: v for [k, v] in content_dict.get("DEFAULT", [])}
 
-    return ForwardModelStep(
+    return UserInstalledForwardModelStep(
         name=name,
         executable=content_dict["EXECUTABLE"],
         stdin_file=content_dict.get("STDIN"),