ert 17.1.9__py3-none-any.whl → 18.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- _ert/events.py +19 -2
- ert/__main__.py +8 -7
- ert/analysis/_update_commons.py +12 -3
- ert/cli/main.py +6 -3
- ert/cli/monitor.py +7 -0
- ert/config/__init__.py +13 -3
- ert/config/_create_observation_dataframes.py +60 -12
- ert/config/_observations.py +14 -1
- ert/config/_read_summary.py +8 -6
- ert/config/ensemble_config.py +6 -14
- ert/config/ert_config.py +19 -13
- ert/config/{everest_objective_config.py → everest_response.py} +23 -12
- ert/config/ext_param_config.py +133 -1
- ert/config/field.py +12 -8
- ert/config/forward_model_step.py +108 -6
- ert/config/gen_data_config.py +2 -6
- ert/config/gen_kw_config.py +0 -9
- ert/config/known_response_types.py +14 -0
- ert/config/parameter_config.py +0 -17
- ert/config/parsing/config_keywords.py +1 -0
- ert/config/parsing/config_schema.py +12 -0
- ert/config/parsing/config_schema_deprecations.py +11 -0
- ert/config/parsing/config_schema_item.py +1 -1
- ert/config/queue_config.py +4 -4
- ert/config/response_config.py +0 -7
- ert/config/rft_config.py +230 -0
- ert/config/summary_config.py +2 -6
- ert/config/violations.py +0 -0
- ert/config/workflow_fixtures.py +2 -1
- ert/dark_storage/client/__init__.py +2 -2
- ert/dark_storage/client/_session.py +4 -4
- ert/dark_storage/client/client.py +2 -2
- ert/dark_storage/compute/misfits.py +7 -6
- ert/dark_storage/endpoints/compute/misfits.py +2 -2
- ert/dark_storage/endpoints/observations.py +4 -4
- ert/dark_storage/endpoints/responses.py +15 -1
- ert/ensemble_evaluator/__init__.py +8 -1
- ert/ensemble_evaluator/evaluator.py +81 -29
- ert/ensemble_evaluator/event.py +6 -0
- ert/ensemble_evaluator/snapshot.py +3 -1
- ert/ensemble_evaluator/state.py +1 -0
- ert/field_utils/__init__.py +8 -0
- ert/field_utils/field_utils.py +211 -1
- ert/gui/ertwidgets/__init__.py +23 -16
- ert/gui/ertwidgets/analysismoduleedit.py +2 -2
- ert/gui/ertwidgets/checklist.py +1 -1
- ert/gui/ertwidgets/create_experiment_dialog.py +3 -1
- ert/gui/ertwidgets/ensembleselector.py +2 -2
- ert/gui/ertwidgets/models/__init__.py +2 -0
- ert/gui/ertwidgets/models/activerealizationsmodel.py +2 -1
- ert/gui/ertwidgets/models/path_model.py +1 -1
- ert/gui/ertwidgets/models/targetensemblemodel.py +2 -1
- ert/gui/ertwidgets/models/text_model.py +1 -1
- ert/gui/ertwidgets/searchbox.py +13 -4
- ert/gui/{suggestor → ertwidgets/suggestor}/_suggestor_message.py +13 -4
- ert/gui/main.py +11 -6
- ert/gui/main_window.py +1 -2
- ert/gui/simulation/ensemble_experiment_panel.py +1 -1
- ert/gui/simulation/ensemble_information_filter_panel.py +1 -1
- ert/gui/simulation/ensemble_smoother_panel.py +1 -1
- ert/gui/simulation/evaluate_ensemble_panel.py +1 -1
- ert/gui/simulation/experiment_panel.py +1 -1
- ert/gui/simulation/manual_update_panel.py +31 -8
- ert/gui/simulation/multiple_data_assimilation_panel.py +12 -8
- ert/gui/simulation/run_dialog.py +25 -4
- ert/gui/simulation/single_test_run_panel.py +2 -2
- ert/gui/summarypanel.py +1 -1
- ert/gui/tools/load_results/load_results_panel.py +1 -1
- ert/gui/tools/manage_experiments/storage_info_widget.py +7 -7
- ert/gui/tools/manage_experiments/storage_widget.py +1 -2
- ert/gui/tools/plot/plot_api.py +13 -10
- ert/gui/tools/plot/plot_window.py +12 -0
- ert/gui/tools/plot/plottery/plot_config.py +2 -0
- ert/gui/tools/plot/plottery/plot_context.py +14 -0
- ert/gui/tools/plot/plottery/plots/ensemble.py +9 -2
- ert/gui/tools/plot/plottery/plots/statistics.py +59 -19
- ert/mode_definitions.py +2 -0
- ert/plugins/__init__.py +0 -1
- ert/plugins/hook_implementations/workflows/gen_data_rft_export.py +10 -2
- ert/plugins/hook_specifications/__init__.py +0 -2
- ert/plugins/hook_specifications/jobs.py +0 -9
- ert/plugins/plugin_manager.py +2 -33
- ert/resources/shell_scripts/delete_directory.py +2 -2
- ert/run_models/__init__.py +18 -5
- ert/run_models/_create_run_path.py +33 -21
- ert/run_models/ensemble_experiment.py +10 -4
- ert/run_models/ensemble_information_filter.py +8 -1
- ert/run_models/ensemble_smoother.py +9 -3
- ert/run_models/evaluate_ensemble.py +8 -6
- ert/run_models/event.py +7 -3
- ert/run_models/everest_run_model.py +155 -44
- ert/run_models/initial_ensemble_run_model.py +23 -22
- ert/run_models/manual_update.py +4 -2
- ert/run_models/manual_update_enif.py +37 -0
- ert/run_models/model_factory.py +81 -22
- ert/run_models/multiple_data_assimilation.py +21 -10
- ert/run_models/run_model.py +54 -34
- ert/run_models/single_test_run.py +7 -4
- ert/run_models/update_run_model.py +4 -2
- ert/runpaths.py +5 -6
- ert/sample_prior.py +9 -4
- ert/scheduler/driver.py +37 -0
- ert/scheduler/event.py +3 -1
- ert/scheduler/job.py +23 -13
- ert/scheduler/lsf_driver.py +6 -2
- ert/scheduler/openpbs_driver.py +7 -1
- ert/scheduler/scheduler.py +5 -0
- ert/scheduler/slurm_driver.py +6 -2
- ert/services/__init__.py +2 -2
- ert/services/_base_service.py +31 -15
- ert/services/ert_server.py +317 -0
- ert/shared/_doc_utils/ert_jobs.py +1 -4
- ert/shared/storage/connection.py +3 -3
- ert/shared/version.py +3 -3
- ert/storage/local_ensemble.py +25 -5
- ert/storage/local_experiment.py +6 -14
- ert/storage/local_storage.py +35 -30
- ert/storage/migration/to18.py +12 -0
- ert/storage/migration/to8.py +4 -4
- ert/substitutions.py +12 -28
- ert/validation/active_range.py +7 -7
- ert/validation/rangestring.py +16 -16
- {ert-17.1.9.dist-info → ert-18.0.0.dist-info}/METADATA +8 -7
- {ert-17.1.9.dist-info → ert-18.0.0.dist-info}/RECORD +160 -159
- everest/api/everest_data_api.py +1 -14
- everest/bin/config_branch_script.py +3 -6
- everest/bin/everconfigdump_script.py +1 -9
- everest/bin/everest_script.py +21 -11
- everest/bin/kill_script.py +2 -2
- everest/bin/monitor_script.py +2 -2
- everest/bin/utils.py +6 -3
- everest/config/__init__.py +4 -1
- everest/config/control_config.py +61 -2
- everest/config/control_variable_config.py +2 -1
- everest/config/everest_config.py +38 -16
- everest/config/forward_model_config.py +5 -3
- everest/config/install_data_config.py +7 -5
- everest/config/install_job_config.py +7 -3
- everest/config/install_template_config.py +3 -3
- everest/config/optimization_config.py +19 -6
- everest/config/output_constraint_config.py +8 -2
- everest/config/server_config.py +6 -49
- everest/config/utils.py +25 -105
- everest/config/validation_utils.py +10 -10
- everest/config_file_loader.py +13 -2
- everest/detached/everserver.py +7 -8
- everest/everest_storage.py +6 -10
- everest/gui/everest_client.py +0 -1
- everest/gui/main_window.py +2 -2
- everest/optimizer/everest2ropt.py +59 -32
- everest/optimizer/opt_model_transforms.py +12 -13
- everest/optimizer/utils.py +0 -29
- everest/strings.py +0 -5
- ert/config/everest_constraints_config.py +0 -95
- ert/services/storage_service.py +0 -127
- everest/config/sampler_config.py +0 -103
- everest/simulator/__init__.py +0 -88
- everest/simulator/everest_to_ert.py +0 -51
- /ert/gui/{suggestor → ertwidgets/suggestor}/__init__.py +0 -0
- /ert/gui/{suggestor → ertwidgets/suggestor}/_colors.py +0 -0
- /ert/gui/{suggestor → ertwidgets/suggestor}/suggestor.py +0 -0
- {ert-17.1.9.dist-info → ert-18.0.0.dist-info}/WHEEL +0 -0
- {ert-17.1.9.dist-info → ert-18.0.0.dist-info}/entry_points.txt +0 -0
- {ert-17.1.9.dist-info → ert-18.0.0.dist-info}/licenses/COPYING +0 -0
- {ert-17.1.9.dist-info → ert-18.0.0.dist-info}/top_level.txt +0 -0
everest/config/server_config.py
CHANGED
|
@@ -1,6 +1,4 @@
|
|
|
1
|
-
import json
|
|
2
1
|
import os
|
|
3
|
-
from pathlib import Path
|
|
4
2
|
from textwrap import dedent
|
|
5
3
|
from typing import Any
|
|
6
4
|
|
|
@@ -12,14 +10,9 @@ from ert.config.queue_config import (
|
|
|
12
10
|
SlurmQueueOptions,
|
|
13
11
|
TorqueQueueOptions,
|
|
14
12
|
)
|
|
15
|
-
from ert.dark_storage.client import
|
|
13
|
+
from ert.dark_storage.client import ErtClientConnectionInfo
|
|
16
14
|
|
|
17
|
-
from ..strings import
|
|
18
|
-
CERTIFICATE_DIR,
|
|
19
|
-
DETACHED_NODE_DIR,
|
|
20
|
-
HOSTFILE_NAME,
|
|
21
|
-
SESSION_DIR,
|
|
22
|
-
)
|
|
15
|
+
from ..strings import SESSION_DIR
|
|
23
16
|
from .simulator_config import check_removed_config
|
|
24
17
|
|
|
25
18
|
|
|
@@ -59,21 +52,11 @@ class ServerConfig(BaseModel):
|
|
|
59
52
|
check_removed_config(data.get("queue_system"))
|
|
60
53
|
return data
|
|
61
54
|
|
|
62
|
-
@staticmethod
|
|
63
|
-
def get_server_url(output_dir: str) -> str:
|
|
64
|
-
"""Return the url of the server.
|
|
65
|
-
|
|
66
|
-
If server_info are given, the url is generated using that info. Otherwise
|
|
67
|
-
server information are retrieved from the hostfile
|
|
68
|
-
"""
|
|
69
|
-
server_info = ServerConfig.get_server_info(output_dir)
|
|
70
|
-
return f"https://{server_info['host']}:{server_info['port']}/experiment_server"
|
|
71
|
-
|
|
72
55
|
@staticmethod
|
|
73
56
|
def get_server_context_from_conn_info(
|
|
74
|
-
conn_info:
|
|
57
|
+
conn_info: ErtClientConnectionInfo,
|
|
75
58
|
) -> tuple[str, str, tuple[str, str]]:
|
|
76
|
-
"""Get server connection context information from a
|
|
59
|
+
"""Get server connection context information from a ErtClientConnectionInfo.
|
|
77
60
|
|
|
78
61
|
Returns a tuple containing the server URL, certificate file path,
|
|
79
62
|
and authentication credentials.
|
|
@@ -83,7 +66,7 @@ class ServerConfig(BaseModel):
|
|
|
83
66
|
This should be refactored to use the ERT Storage Client class directly instead.
|
|
84
67
|
|
|
85
68
|
Args:
|
|
86
|
-
conn_info: An instance of the
|
|
69
|
+
conn_info: An instance of the ErtClientConnectionInfo
|
|
87
70
|
|
|
88
71
|
Returns:
|
|
89
72
|
tuple: A tuple containing:
|
|
@@ -102,34 +85,8 @@ class ServerConfig(BaseModel):
|
|
|
102
85
|
|
|
103
86
|
return url, cert_file, auth
|
|
104
87
|
|
|
105
|
-
@staticmethod
|
|
106
|
-
def get_server_info(output_dir: str) -> dict[str, Any]:
|
|
107
|
-
"""Load server information from the hostfile"""
|
|
108
|
-
host_file_path = Path(ServerConfig.get_hostfile_path(output_dir))
|
|
109
|
-
if not host_file_path.exists():
|
|
110
|
-
return {"host": None, "port": None, "cert": None, "auth": None}
|
|
111
|
-
|
|
112
|
-
data = json.loads(host_file_path.read_text(encoding="utf-8"))
|
|
113
|
-
|
|
114
|
-
if not all(k in data for k in ("host", "port", "cert", "auth")):
|
|
115
|
-
raise RuntimeError("Malformed hostfile")
|
|
116
|
-
return data
|
|
117
|
-
|
|
118
|
-
@staticmethod
|
|
119
|
-
def get_detached_node_dir(output_dir: str) -> str:
|
|
120
|
-
return os.path.join(os.path.abspath(output_dir), DETACHED_NODE_DIR)
|
|
121
|
-
|
|
122
|
-
@staticmethod
|
|
123
|
-
def get_hostfile_path(output_dir: str) -> str:
|
|
124
|
-
return os.path.join(ServerConfig.get_session_dir(output_dir), HOSTFILE_NAME)
|
|
125
|
-
|
|
126
88
|
@staticmethod
|
|
127
89
|
def get_session_dir(output_dir: str) -> str:
|
|
128
90
|
"""Return path to the session directory containing information about the
|
|
129
91
|
certificates and host information"""
|
|
130
|
-
return os.path.join(
|
|
131
|
-
|
|
132
|
-
@staticmethod
|
|
133
|
-
def get_certificate_dir(output_dir: str) -> str:
|
|
134
|
-
"""Return the path to certificate folder"""
|
|
135
|
-
return os.path.join(ServerConfig.get_session_dir(output_dir), CERTIFICATE_DIR)
|
|
92
|
+
return os.path.join(os.path.abspath(output_dir), SESSION_DIR)
|
everest/config/utils.py
CHANGED
|
@@ -1,108 +1,28 @@
|
|
|
1
|
-
from
|
|
2
|
-
from typing import Any, Literal
|
|
1
|
+
from ert.config import ExtParamConfig, SamplerConfig
|
|
3
2
|
|
|
4
|
-
from .control_config import ControlConfig
|
|
5
|
-
from .control_variable_config import ControlVariableGuessListConfig
|
|
6
|
-
from .sampler_config import SamplerConfig
|
|
7
3
|
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
def __init__(self, controls: list[ControlConfig]) -> None:
|
|
11
|
-
control_dicts = _get_control_dicts(controls)
|
|
12
|
-
self.names = [control["name"] for control in control_dicts]
|
|
13
|
-
self.types: list[Literal["real", "integer"]] = [
|
|
14
|
-
control["control_type"] for control in control_dicts
|
|
15
|
-
]
|
|
16
|
-
self.initial_guesses = [control["initial_guess"] for control in control_dicts]
|
|
17
|
-
self.lower_bounds = [control["min"] for control in control_dicts]
|
|
18
|
-
self.upper_bounds = [control["max"] for control in control_dicts]
|
|
19
|
-
self.scaled_ranges = [control["scaled_range"] for control in control_dicts]
|
|
20
|
-
self.enabled = [control["enabled"] for control in control_dicts]
|
|
21
|
-
self.perturbation_magnitudes = [
|
|
22
|
-
control["perturbation_magnitude"] for control in control_dicts
|
|
23
|
-
]
|
|
24
|
-
self.perturbation_types = [
|
|
25
|
-
control["perturbation_type"] for control in control_dicts
|
|
26
|
-
]
|
|
27
|
-
self.samplers, self.sampler_indices = _get_samplers(controls)
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
def _get_control_dicts(controls: list[ControlConfig]) -> list[dict[str, Any]]:
|
|
31
|
-
def _inject_defaults(control: ControlConfig, var_dict: dict[str, Any]) -> None:
|
|
32
|
-
for key in [
|
|
33
|
-
"type",
|
|
34
|
-
"initial_guess",
|
|
35
|
-
"control_type",
|
|
36
|
-
"enabled",
|
|
37
|
-
"min",
|
|
38
|
-
"max",
|
|
39
|
-
"perturbation_type",
|
|
40
|
-
"perturbation_magnitude",
|
|
41
|
-
"scaled_range",
|
|
42
|
-
]:
|
|
43
|
-
if var_dict.get(key) is None:
|
|
44
|
-
var_dict[key] = getattr(control, key)
|
|
45
|
-
|
|
46
|
-
control_dicts: list[dict[str, Any]] = []
|
|
47
|
-
for control in controls:
|
|
48
|
-
for variable in control.variables:
|
|
49
|
-
if isinstance(variable, ControlVariableGuessListConfig):
|
|
50
|
-
for index, guess in enumerate(variable.initial_guess, start=1):
|
|
51
|
-
var_dict = deepcopy(variable.model_dump())
|
|
52
|
-
var_dict["name"] = (control.name, variable.name, index)
|
|
53
|
-
var_dict["initial_guess"] = guess
|
|
54
|
-
_inject_defaults(control, var_dict)
|
|
55
|
-
control_dicts.append(var_dict)
|
|
56
|
-
else:
|
|
57
|
-
var_dict = deepcopy(variable.model_dump())
|
|
58
|
-
var_dict["name"] = (
|
|
59
|
-
(control.name, variable.name)
|
|
60
|
-
if variable.index is None
|
|
61
|
-
else (control.name, variable.name, variable.index)
|
|
62
|
-
)
|
|
63
|
-
_inject_defaults(control, var_dict)
|
|
64
|
-
control_dicts.append(var_dict)
|
|
65
|
-
return control_dicts
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
def _get_samplers(
|
|
69
|
-
controls: list[ControlConfig],
|
|
4
|
+
def get_samplers(
|
|
5
|
+
controls: list[ExtParamConfig],
|
|
70
6
|
) -> tuple[list[SamplerConfig | None], list[int]]:
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
samplers.append(None)
|
|
94
|
-
default_sampler_index = len(samplers) - 1
|
|
95
|
-
variable_sampler_index = default_sampler_index
|
|
96
|
-
|
|
97
|
-
if isinstance(variable, ControlVariableGuessListConfig):
|
|
98
|
-
sampler_indices.extend(
|
|
99
|
-
[variable_sampler_index] * len(variable.initial_guess)
|
|
100
|
-
)
|
|
101
|
-
else:
|
|
102
|
-
sampler_indices.append(variable_sampler_index)
|
|
103
|
-
|
|
104
|
-
return samplers, sampler_indices
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
def flatten_controls(controls: list[ControlConfig]) -> FlattenedControls:
|
|
108
|
-
return FlattenedControls(controls)
|
|
7
|
+
"""
|
|
8
|
+
Create a list of unique samplers, and a list mapping variable index
|
|
9
|
+
to sampler index. I.e., this points each control variable to
|
|
10
|
+
a sampler by index.
|
|
11
|
+
"""
|
|
12
|
+
flattened_samplers = [
|
|
13
|
+
sampler for control in controls for sampler in control.samplers
|
|
14
|
+
]
|
|
15
|
+
unique_samplers: list[SamplerConfig | None] = []
|
|
16
|
+
variable_to_unique_sampler_index: list[int] = []
|
|
17
|
+
for sampler in flattened_samplers:
|
|
18
|
+
try:
|
|
19
|
+
unique_sampler_index = next(
|
|
20
|
+
i for i, s in enumerate(unique_samplers) if s == sampler
|
|
21
|
+
)
|
|
22
|
+
except StopIteration:
|
|
23
|
+
unique_sampler_index = len(unique_samplers)
|
|
24
|
+
unique_samplers.append(sampler)
|
|
25
|
+
|
|
26
|
+
variable_to_unique_sampler_index.append(unique_sampler_index)
|
|
27
|
+
|
|
28
|
+
return unique_samplers, variable_to_unique_sampler_index
|
|
@@ -72,15 +72,15 @@ class InstallDataContext:
|
|
|
72
72
|
source = item.source
|
|
73
73
|
target = item.target
|
|
74
74
|
|
|
75
|
-
if "<
|
|
75
|
+
if "<REALIZATION_ID>" not in source:
|
|
76
76
|
self._set_symlink(source, target, None)
|
|
77
77
|
|
|
78
78
|
return self
|
|
79
79
|
|
|
80
80
|
def _set_symlink(self, source: str, target: str, realization: int | None) -> None:
|
|
81
81
|
if realization is not None:
|
|
82
|
-
source = source.replace("<
|
|
83
|
-
target = target.replace("<
|
|
82
|
+
source = source.replace("<REALIZATION_ID>", str(realization))
|
|
83
|
+
target = target.replace("<REALIZATION_ID>", str(realization))
|
|
84
84
|
|
|
85
85
|
tmp_target = Path(self._temp_dir.name) / Path(target)
|
|
86
86
|
if tmp_target.exists():
|
|
@@ -101,7 +101,7 @@ class InstallDataContext:
|
|
|
101
101
|
|
|
102
102
|
def add_links_for_realization(self, realization: int) -> None:
|
|
103
103
|
for data in self._install_data:
|
|
104
|
-
if data.source is not None and "<
|
|
104
|
+
if data.source is not None and "<REALIZATION_ID>" in data.source:
|
|
105
105
|
self._set_symlink(data.source, data.target, realization)
|
|
106
106
|
|
|
107
107
|
def __exit__(self, exc_type: Any, exc_value: Any, exc_tb: Any) -> None:
|
|
@@ -190,7 +190,7 @@ def unique_items(items: Sequence[T]) -> Sequence[T]:
|
|
|
190
190
|
|
|
191
191
|
|
|
192
192
|
def valid_range(range_value: tuple[float, float]) -> tuple[float, float]:
|
|
193
|
-
if range_value[0] >= range_value[1]:
|
|
193
|
+
if range_value is not None and range_value[0] >= range_value[1]:
|
|
194
194
|
raise ValueError("scaled_range must be a valid range [a, b], where a < b.")
|
|
195
195
|
return range_value
|
|
196
196
|
|
|
@@ -242,8 +242,8 @@ def as_abs_path(path: str, config_dir: str) -> str:
|
|
|
242
242
|
|
|
243
243
|
|
|
244
244
|
def expand_model_id_paths(path_source: str, realizations: list[int]) -> list[str]:
|
|
245
|
-
if "<
|
|
246
|
-
return [path_source.replace("<
|
|
245
|
+
if "<REALIZATION_ID>" in path_source:
|
|
246
|
+
return [path_source.replace("<REALIZATION_ID>", str(r)) for r in realizations]
|
|
247
247
|
return [path_source]
|
|
248
248
|
|
|
249
249
|
|
|
@@ -251,7 +251,8 @@ def check_path_exists(
|
|
|
251
251
|
path_source: str, config_path: Path | None, realizations: list[int]
|
|
252
252
|
) -> None:
|
|
253
253
|
"""Check if the given path exists. If the given path contains <CONFIG_PATH>
|
|
254
|
-
or
|
|
254
|
+
or REALIZATION_ID they will be expanded and all instances of expanded paths
|
|
255
|
+
need to exist.
|
|
255
256
|
"""
|
|
256
257
|
if not isinstance(path_source, str):
|
|
257
258
|
raise ValueError(
|
|
@@ -285,8 +286,7 @@ def check_writeable_path(path_source: str, config_path: Path) -> None:
|
|
|
285
286
|
if os.access(path, os.W_OK | os.X_OK):
|
|
286
287
|
break
|
|
287
288
|
elif os.path.isfile(path):
|
|
288
|
-
|
|
289
|
-
raise ValueError(f"File {path} exists, cannot create path {path_source}")
|
|
289
|
+
raise ValueError(f"{path} is a file, cannot create folders inside it")
|
|
290
290
|
parent = os.path.dirname(path)
|
|
291
291
|
if parent == path: # ie, if path is root
|
|
292
292
|
break
|
everest/config_file_loader.py
CHANGED
|
@@ -26,7 +26,7 @@ SUBSTITUTION_PATTERN = r"(r\{\{.*?\}\})"
|
|
|
26
26
|
|
|
27
27
|
# Jinja vars which should NOT be included in definitions portion of config.
|
|
28
28
|
ERT_CONFIG_TEMPLATES = {
|
|
29
|
-
"realization": "
|
|
29
|
+
"realization": "REALIZATION_ID",
|
|
30
30
|
"runpath_file": "RUNPATH_FILE",
|
|
31
31
|
}
|
|
32
32
|
|
|
@@ -67,7 +67,7 @@ def _get_definitions(
|
|
|
67
67
|
f"Internal key {key} specified by user as {defs[key]}. "
|
|
68
68
|
f"Overriding as {val}"
|
|
69
69
|
)
|
|
70
|
-
defs[key] = f"<{val}>" # ert uses <
|
|
70
|
+
defs[key] = f"<{val}>" # ert uses <REALIZATION_ID> as format
|
|
71
71
|
else:
|
|
72
72
|
logging.getLogger(EVEREST).warning("Empty configuration file provided!")
|
|
73
73
|
|
|
@@ -160,6 +160,17 @@ def yaml_file_to_substituted_config_dict(config_path: str) -> dict[str, Any]:
|
|
|
160
160
|
# Replace in definitions
|
|
161
161
|
config = jenv.from_string(txt).render(**definitions)
|
|
162
162
|
|
|
163
|
+
# Handle deprecated use of <GEO_ID>:
|
|
164
|
+
config, count = re.subn(
|
|
165
|
+
rf"<\s*{re.escape('GEO_ID')}\s*>", "<REALIZATION_ID>", config
|
|
166
|
+
)
|
|
167
|
+
if count > 0:
|
|
168
|
+
message = r"<GEO_ID> is deprecated, please replace with 'r{{realization}}'."
|
|
169
|
+
print(message)
|
|
170
|
+
logging.getLogger(EVEREST).warning(
|
|
171
|
+
"Deprecated key <GEO_ID> replaced by <REALIZATION_ID>."
|
|
172
|
+
)
|
|
173
|
+
|
|
163
174
|
# Load the config with definitions again as yaml
|
|
164
175
|
yaml = YAML(typ="safe", pure=True).load(config)
|
|
165
176
|
|
everest/detached/everserver.py
CHANGED
|
@@ -12,7 +12,7 @@ import yaml
|
|
|
12
12
|
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
|
|
13
13
|
|
|
14
14
|
from ert.plugins.plugin_manager import ErtPluginManager
|
|
15
|
-
from ert.services import
|
|
15
|
+
from ert.services import ErtServer
|
|
16
16
|
from ert.services._base_service import BaseServiceExit
|
|
17
17
|
from ert.storage import ExperimentStatus
|
|
18
18
|
from ert.storage.local_experiment import ExperimentState
|
|
@@ -31,7 +31,7 @@ logger = logging.getLogger(__name__)
|
|
|
31
31
|
|
|
32
32
|
|
|
33
33
|
def _configure_loggers(
|
|
34
|
-
|
|
34
|
+
log_dir: Path, logging_level: int, output_file: str | None
|
|
35
35
|
) -> None:
|
|
36
36
|
def make_handler_config(path: Path, log_level: int) -> dict[str, Any]:
|
|
37
37
|
makedirs_if_needed(str(path.parent))
|
|
@@ -46,7 +46,7 @@ def _configure_loggers(
|
|
|
46
46
|
"version": 1,
|
|
47
47
|
"handlers": {
|
|
48
48
|
"endpoint_log": make_handler_config(
|
|
49
|
-
|
|
49
|
+
log_dir / "everserver.log", logging_level
|
|
50
50
|
),
|
|
51
51
|
"everest_log": make_handler_config(log_dir / "everest.log", logging_level),
|
|
52
52
|
"forward_models_log": make_handler_config(
|
|
@@ -151,7 +151,6 @@ def main() -> None:
|
|
|
151
151
|
):
|
|
152
152
|
try:
|
|
153
153
|
_configure_loggers(
|
|
154
|
-
detached_dir=Path(ServerConfig.get_detached_node_dir(output_dir)),
|
|
155
154
|
log_dir=Path(output_dir) / OPTIMIZATION_LOG_DIR,
|
|
156
155
|
logging_level=options.logging_level,
|
|
157
156
|
output_file=log_file.name,
|
|
@@ -163,11 +162,11 @@ def main() -> None:
|
|
|
163
162
|
# Starting the server
|
|
164
163
|
server_path = os.path.abspath(ServerConfig.get_session_dir(output_dir))
|
|
165
164
|
status = ""
|
|
166
|
-
with
|
|
167
|
-
timeout=240, project=server_path, logging_config=log_file.name
|
|
165
|
+
with ErtServer.init_service(
|
|
166
|
+
timeout=240, project=Path(server_path), logging_config=log_file.name
|
|
168
167
|
) as server:
|
|
169
|
-
server.
|
|
170
|
-
with
|
|
168
|
+
server.fetch_connection_info()
|
|
169
|
+
with ErtServer.session(project=Path(server_path)) as client:
|
|
171
170
|
done = False
|
|
172
171
|
while not done:
|
|
173
172
|
response = client.get(
|
everest/everest_storage.py
CHANGED
|
@@ -12,8 +12,7 @@ import numpy as np
|
|
|
12
12
|
import polars as pl
|
|
13
13
|
from ropt.results import FunctionResults, GradientResults, Results
|
|
14
14
|
|
|
15
|
-
from ert.config import EverestObjectivesConfig
|
|
16
|
-
from everest.config.output_constraint_config import OutputConstraintConfig
|
|
15
|
+
from ert.config import EverestConstraintsConfig, EverestObjectivesConfig
|
|
17
16
|
from everest.strings import EVEREST
|
|
18
17
|
|
|
19
18
|
logger = logging.getLogger(__name__)
|
|
@@ -532,7 +531,7 @@ class EverestStorage:
|
|
|
532
531
|
self,
|
|
533
532
|
formatted_control_names: list[str],
|
|
534
533
|
objective_functions: EverestObjectivesConfig,
|
|
535
|
-
output_constraints:
|
|
534
|
+
output_constraints: EverestConstraintsConfig | None,
|
|
536
535
|
realizations: list[int],
|
|
537
536
|
) -> None:
|
|
538
537
|
controls = pl.DataFrame(
|
|
@@ -566,13 +565,7 @@ class EverestStorage:
|
|
|
566
565
|
)
|
|
567
566
|
|
|
568
567
|
nonlinear_constraints = (
|
|
569
|
-
pl.DataFrame(
|
|
570
|
-
{
|
|
571
|
-
"constraint_name": [
|
|
572
|
-
constraint.name for constraint in output_constraints
|
|
573
|
-
],
|
|
574
|
-
}
|
|
575
|
-
)
|
|
568
|
+
pl.DataFrame({"constraint_name": output_constraints.keys})
|
|
576
569
|
if output_constraints
|
|
577
570
|
else None
|
|
578
571
|
)
|
|
@@ -1001,6 +994,9 @@ class EverestStorage:
|
|
|
1001
994
|
def export_dataframes(
|
|
1002
995
|
self,
|
|
1003
996
|
) -> tuple[pl.DataFrame, pl.DataFrame, pl.DataFrame]:
|
|
997
|
+
if not self.data.batches:
|
|
998
|
+
return (pl.DataFrame(), pl.DataFrame(), pl.DataFrame())
|
|
999
|
+
|
|
1004
1000
|
batch_dfs_to_join = {} # type: ignore
|
|
1005
1001
|
realization_dfs_to_join = {} # type: ignore
|
|
1006
1002
|
perturbation_dfs_to_join = {} # type: ignore
|
everest/gui/everest_client.py
CHANGED
everest/gui/main_window.py
CHANGED
|
@@ -13,7 +13,7 @@ from PyQt6.QtWidgets import (
|
|
|
13
13
|
from ert.gui.ertnotifier import ErtNotifier
|
|
14
14
|
from ert.gui.simulation.run_dialog import RunDialog
|
|
15
15
|
from ert.plugins import ErtPluginManager
|
|
16
|
-
from ert.services import
|
|
16
|
+
from ert.services import ErtServer
|
|
17
17
|
from everest.config import ServerConfig
|
|
18
18
|
from everest.detached import wait_for_server
|
|
19
19
|
from everest.gui.everest_client import EverestClient
|
|
@@ -43,7 +43,7 @@ class EverestMainWindow(QMainWindow):
|
|
|
43
43
|
self.setCentralWidget(self.central_widget)
|
|
44
44
|
|
|
45
45
|
def run(self) -> None:
|
|
46
|
-
storage_client =
|
|
46
|
+
storage_client = ErtServer.session(
|
|
47
47
|
Path(ServerConfig.get_session_dir(self.output_dir))
|
|
48
48
|
)
|
|
49
49
|
wait_for_server(storage_client, 60)
|
|
@@ -3,34 +3,48 @@ from typing import Any
|
|
|
3
3
|
|
|
4
4
|
from ropt.enums import PerturbationType, VariableType
|
|
5
5
|
|
|
6
|
-
from ert.config import EverestObjectivesConfig
|
|
6
|
+
from ert.config import EverestConstraintsConfig, EverestObjectivesConfig, ExtParamConfig
|
|
7
7
|
from everest.config import (
|
|
8
|
-
ControlConfig,
|
|
9
8
|
InputConstraintConfig,
|
|
10
9
|
ModelConfig,
|
|
11
10
|
OptimizationConfig,
|
|
12
11
|
OutputConstraintConfig,
|
|
13
12
|
)
|
|
14
|
-
from everest.config.utils import
|
|
13
|
+
from everest.config.utils import get_samplers
|
|
15
14
|
|
|
16
15
|
|
|
17
16
|
def _parse_controls(
|
|
18
|
-
controls:
|
|
17
|
+
controls: list[ExtParamConfig], random_seed: int
|
|
19
18
|
) -> tuple[dict[str, Any], list[dict[str, Any]] | None]:
|
|
20
|
-
control_types = [
|
|
19
|
+
control_types = [
|
|
20
|
+
VariableType[type_.upper()]
|
|
21
|
+
for control in controls
|
|
22
|
+
for type_ in control.control_types
|
|
23
|
+
]
|
|
24
|
+
initial_guesses = [
|
|
25
|
+
initial_guess
|
|
26
|
+
for control in controls
|
|
27
|
+
for initial_guess in control.initial_guesses
|
|
28
|
+
]
|
|
29
|
+
samplers, sampler_indices = get_samplers(controls)
|
|
21
30
|
ropt_variables: dict[str, Any] = {
|
|
22
31
|
"types": None if all(item is None for item in control_types) else control_types,
|
|
23
|
-
"variable_count": len(
|
|
24
|
-
"lower_bounds": controls.
|
|
25
|
-
"upper_bounds": controls.
|
|
32
|
+
"variable_count": len(initial_guesses),
|
|
33
|
+
"lower_bounds": [min_ for control in controls for min_ in control.min],
|
|
34
|
+
"upper_bounds": [max_ for control in controls for max_ in control.max],
|
|
26
35
|
"perturbation_types": [
|
|
27
36
|
PerturbationType[perturbation_type.upper()]
|
|
28
|
-
for
|
|
37
|
+
for control in controls
|
|
38
|
+
for perturbation_type in control.perturbation_types
|
|
39
|
+
],
|
|
40
|
+
"perturbation_magnitudes": [
|
|
41
|
+
perturbation_magnitude
|
|
42
|
+
for control in controls
|
|
43
|
+
for perturbation_magnitude in control.perturbation_magnitudes
|
|
29
44
|
],
|
|
30
|
-
"
|
|
31
|
-
"mask": controls.enabled,
|
|
45
|
+
"mask": [enabled for control in controls for enabled in control.enabled],
|
|
32
46
|
"seed": random_seed,
|
|
33
|
-
"samplers":
|
|
47
|
+
"samplers": sampler_indices,
|
|
34
48
|
}
|
|
35
49
|
|
|
36
50
|
ropt_samplers = [
|
|
@@ -41,7 +55,7 @@ def _parse_controls(
|
|
|
41
55
|
"options": {} if sampler.options is None else sampler.options,
|
|
42
56
|
"shared": False if sampler.shared is None else sampler.shared,
|
|
43
57
|
}
|
|
44
|
-
for sampler in
|
|
58
|
+
for sampler in samplers
|
|
45
59
|
]
|
|
46
60
|
|
|
47
61
|
return ropt_variables, ropt_samplers
|
|
@@ -102,13 +116,13 @@ def _get_bounds(
|
|
|
102
116
|
|
|
103
117
|
def _parse_input_constraints(
|
|
104
118
|
input_constraints: list[InputConstraintConfig],
|
|
105
|
-
controls: list[
|
|
119
|
+
controls: list[ExtParamConfig],
|
|
106
120
|
) -> dict[str, Any]:
|
|
107
121
|
formatted_control_names = [
|
|
108
|
-
name for config in controls for name in config.
|
|
122
|
+
name for config in controls for name in config.input_keys
|
|
109
123
|
]
|
|
110
124
|
formatted_control_names_dotdash = [
|
|
111
|
-
name for config in controls for name in config.
|
|
125
|
+
name for config in controls for name in config.input_keys_dotdash
|
|
112
126
|
]
|
|
113
127
|
|
|
114
128
|
def _get_control_index(name: str) -> int:
|
|
@@ -143,13 +157,26 @@ def _parse_input_constraints(
|
|
|
143
157
|
|
|
144
158
|
|
|
145
159
|
def _parse_output_constraints(
|
|
146
|
-
output_constraints:
|
|
160
|
+
output_constraints: EverestConstraintsConfig | None,
|
|
147
161
|
) -> dict[str, Any]:
|
|
148
162
|
if output_constraints:
|
|
149
|
-
lower_bounds, upper_bounds = _get_bounds(output_constraints)
|
|
150
163
|
return {
|
|
151
|
-
"lower_bounds":
|
|
152
|
-
|
|
164
|
+
"lower_bounds": [
|
|
165
|
+
target if target is not None else lb
|
|
166
|
+
for lb, target in zip(
|
|
167
|
+
output_constraints.lower_bounds,
|
|
168
|
+
output_constraints.targets,
|
|
169
|
+
strict=False,
|
|
170
|
+
)
|
|
171
|
+
],
|
|
172
|
+
"upper_bounds": [
|
|
173
|
+
target if target is not None else ub
|
|
174
|
+
for ub, target in zip(
|
|
175
|
+
output_constraints.upper_bounds,
|
|
176
|
+
output_constraints.targets,
|
|
177
|
+
strict=False,
|
|
178
|
+
)
|
|
179
|
+
],
|
|
153
180
|
}
|
|
154
181
|
return {}
|
|
155
182
|
|
|
@@ -248,18 +275,16 @@ def _parse_optimization(
|
|
|
248
275
|
|
|
249
276
|
|
|
250
277
|
def everest2ropt(
|
|
251
|
-
controls: list[
|
|
278
|
+
controls: list[ExtParamConfig],
|
|
252
279
|
objective_functions: EverestObjectivesConfig,
|
|
253
280
|
input_constraints: list[InputConstraintConfig],
|
|
254
|
-
output_constraints:
|
|
281
|
+
output_constraints: EverestConstraintsConfig | None,
|
|
255
282
|
optimization: OptimizationConfig | None,
|
|
256
283
|
model: ModelConfig,
|
|
257
284
|
random_seed: int,
|
|
258
285
|
optimization_output_dir: str,
|
|
259
286
|
) -> tuple[dict[str, Any], list[float]]:
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
ropt_variables, ropt_samplers = _parse_controls(flattened_controls, random_seed)
|
|
287
|
+
ropt_variables, ropt_samplers = _parse_controls(controls, random_seed)
|
|
263
288
|
ropt_objectives, ropt_function_estimators = _parse_objectives(objective_functions)
|
|
264
289
|
ropt_linear_constraints = _parse_input_constraints(input_constraints, controls)
|
|
265
290
|
ropt_nonlinear_constraints = _parse_output_constraints(output_constraints)
|
|
@@ -293,13 +318,11 @@ def everest2ropt(
|
|
|
293
318
|
"realizations": ropt_realizations,
|
|
294
319
|
"optimizer": ropt_optimizer,
|
|
295
320
|
"names": {
|
|
296
|
-
"variable": [
|
|
297
|
-
name for config in controls for name in config.formatted_control_names
|
|
298
|
-
],
|
|
321
|
+
"variable": [name for config in controls for name in config.input_keys],
|
|
299
322
|
"objective": objective_functions.keys,
|
|
300
|
-
"nonlinear_constraint":
|
|
301
|
-
|
|
302
|
-
],
|
|
323
|
+
"nonlinear_constraint": output_constraints.keys
|
|
324
|
+
if output_constraints is not None
|
|
325
|
+
else [],
|
|
303
326
|
"realization": model.realizations,
|
|
304
327
|
},
|
|
305
328
|
}
|
|
@@ -316,4 +339,8 @@ def everest2ropt(
|
|
|
316
339
|
if ropt_samplers:
|
|
317
340
|
ropt_config["samplers"] = ropt_samplers
|
|
318
341
|
|
|
319
|
-
return ropt_config,
|
|
342
|
+
return ropt_config, [
|
|
343
|
+
initial_guess
|
|
344
|
+
for control in controls
|
|
345
|
+
for initial_guess in control.initial_guesses
|
|
346
|
+
]
|
|
@@ -11,14 +11,11 @@ from ropt.transforms.base import (
|
|
|
11
11
|
VariableTransform,
|
|
12
12
|
)
|
|
13
13
|
|
|
14
|
-
from ert.config import EverestObjectivesConfig
|
|
14
|
+
from ert.config import EverestConstraintsConfig, EverestObjectivesConfig, ExtParamConfig
|
|
15
15
|
from everest.config import (
|
|
16
|
-
ControlConfig,
|
|
17
16
|
InputConstraintConfig,
|
|
18
17
|
ModelConfig,
|
|
19
|
-
OutputConstraintConfig,
|
|
20
18
|
)
|
|
21
|
-
from everest.config.utils import FlattenedControls
|
|
22
19
|
|
|
23
20
|
|
|
24
21
|
class EverestOptModelTransforms(TypedDict):
|
|
@@ -429,19 +426,22 @@ class ConstraintScaler(NonLinearConstraintTransform):
|
|
|
429
426
|
|
|
430
427
|
|
|
431
428
|
def get_optimization_domain_transforms(
|
|
432
|
-
controls: list[
|
|
429
|
+
controls: list[ExtParamConfig],
|
|
433
430
|
objectives: EverestObjectivesConfig,
|
|
434
431
|
input_constraints: list[InputConstraintConfig] | None,
|
|
435
|
-
output_constraints:
|
|
432
|
+
output_constraints: EverestConstraintsConfig | None,
|
|
436
433
|
model: ModelConfig,
|
|
437
434
|
auto_scale: bool,
|
|
438
435
|
) -> EverestOptModelTransforms:
|
|
439
|
-
flattened_controls = FlattenedControls(controls)
|
|
440
436
|
control_scaler = ControlScaler(
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
437
|
+
[min_ for control in controls for min_ in control.min],
|
|
438
|
+
[max_ for control in controls for max_ in control.max],
|
|
439
|
+
[
|
|
440
|
+
scaled_range
|
|
441
|
+
for control in controls
|
|
442
|
+
for scaled_range in control.scaled_ranges
|
|
443
|
+
],
|
|
444
|
+
[type_ for control in controls for type_ in control.control_types],
|
|
445
445
|
auto_scale_input_constraints=auto_scale,
|
|
446
446
|
input_constraint_scales=(
|
|
447
447
|
None
|
|
@@ -472,8 +472,7 @@ def get_optimization_domain_transforms(
|
|
|
472
472
|
ConstraintScaler(
|
|
473
473
|
auto_scale=auto_scale,
|
|
474
474
|
scales=[
|
|
475
|
-
1.0 if
|
|
476
|
-
for constraint in output_constraints
|
|
475
|
+
1.0 if scale is None else scale for scale in output_constraints.scales
|
|
477
476
|
],
|
|
478
477
|
realization_weights=realization_weights,
|
|
479
478
|
)
|