ert 17.1.9__py3-none-any.whl → 18.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- _ert/events.py +19 -2
- ert/__main__.py +8 -7
- ert/analysis/_update_commons.py +12 -3
- ert/cli/main.py +6 -3
- ert/cli/monitor.py +7 -0
- ert/config/__init__.py +13 -3
- ert/config/_create_observation_dataframes.py +60 -12
- ert/config/_observations.py +14 -1
- ert/config/_read_summary.py +8 -6
- ert/config/ensemble_config.py +6 -14
- ert/config/ert_config.py +19 -13
- ert/config/{everest_objective_config.py → everest_response.py} +23 -12
- ert/config/ext_param_config.py +133 -1
- ert/config/field.py +12 -8
- ert/config/forward_model_step.py +108 -6
- ert/config/gen_data_config.py +2 -6
- ert/config/gen_kw_config.py +0 -9
- ert/config/known_response_types.py +14 -0
- ert/config/parameter_config.py +0 -17
- ert/config/parsing/config_keywords.py +1 -0
- ert/config/parsing/config_schema.py +12 -0
- ert/config/parsing/config_schema_deprecations.py +11 -0
- ert/config/parsing/config_schema_item.py +1 -1
- ert/config/queue_config.py +4 -4
- ert/config/response_config.py +0 -7
- ert/config/rft_config.py +230 -0
- ert/config/summary_config.py +2 -6
- ert/config/violations.py +0 -0
- ert/config/workflow_fixtures.py +2 -1
- ert/dark_storage/client/__init__.py +2 -2
- ert/dark_storage/client/_session.py +4 -4
- ert/dark_storage/client/client.py +2 -2
- ert/dark_storage/compute/misfits.py +7 -6
- ert/dark_storage/endpoints/compute/misfits.py +2 -2
- ert/dark_storage/endpoints/observations.py +4 -4
- ert/dark_storage/endpoints/responses.py +15 -1
- ert/ensemble_evaluator/__init__.py +8 -1
- ert/ensemble_evaluator/evaluator.py +81 -29
- ert/ensemble_evaluator/event.py +6 -0
- ert/ensemble_evaluator/snapshot.py +3 -1
- ert/ensemble_evaluator/state.py +1 -0
- ert/field_utils/__init__.py +8 -0
- ert/field_utils/field_utils.py +211 -1
- ert/gui/ertwidgets/__init__.py +23 -16
- ert/gui/ertwidgets/analysismoduleedit.py +2 -2
- ert/gui/ertwidgets/checklist.py +1 -1
- ert/gui/ertwidgets/create_experiment_dialog.py +3 -1
- ert/gui/ertwidgets/ensembleselector.py +2 -2
- ert/gui/ertwidgets/models/__init__.py +2 -0
- ert/gui/ertwidgets/models/activerealizationsmodel.py +2 -1
- ert/gui/ertwidgets/models/path_model.py +1 -1
- ert/gui/ertwidgets/models/targetensemblemodel.py +2 -1
- ert/gui/ertwidgets/models/text_model.py +1 -1
- ert/gui/ertwidgets/searchbox.py +13 -4
- ert/gui/{suggestor → ertwidgets/suggestor}/_suggestor_message.py +13 -4
- ert/gui/main.py +11 -6
- ert/gui/main_window.py +1 -2
- ert/gui/simulation/ensemble_experiment_panel.py +1 -1
- ert/gui/simulation/ensemble_information_filter_panel.py +1 -1
- ert/gui/simulation/ensemble_smoother_panel.py +1 -1
- ert/gui/simulation/evaluate_ensemble_panel.py +1 -1
- ert/gui/simulation/experiment_panel.py +1 -1
- ert/gui/simulation/manual_update_panel.py +31 -8
- ert/gui/simulation/multiple_data_assimilation_panel.py +12 -8
- ert/gui/simulation/run_dialog.py +25 -4
- ert/gui/simulation/single_test_run_panel.py +2 -2
- ert/gui/summarypanel.py +1 -1
- ert/gui/tools/load_results/load_results_panel.py +1 -1
- ert/gui/tools/manage_experiments/storage_info_widget.py +7 -7
- ert/gui/tools/manage_experiments/storage_widget.py +1 -2
- ert/gui/tools/plot/plot_api.py +13 -10
- ert/gui/tools/plot/plot_window.py +12 -0
- ert/gui/tools/plot/plottery/plot_config.py +2 -0
- ert/gui/tools/plot/plottery/plot_context.py +14 -0
- ert/gui/tools/plot/plottery/plots/ensemble.py +9 -2
- ert/gui/tools/plot/plottery/plots/statistics.py +59 -19
- ert/mode_definitions.py +2 -0
- ert/plugins/__init__.py +0 -1
- ert/plugins/hook_implementations/workflows/gen_data_rft_export.py +10 -2
- ert/plugins/hook_specifications/__init__.py +0 -2
- ert/plugins/hook_specifications/jobs.py +0 -9
- ert/plugins/plugin_manager.py +2 -33
- ert/resources/shell_scripts/delete_directory.py +2 -2
- ert/run_models/__init__.py +18 -5
- ert/run_models/_create_run_path.py +33 -21
- ert/run_models/ensemble_experiment.py +10 -4
- ert/run_models/ensemble_information_filter.py +8 -1
- ert/run_models/ensemble_smoother.py +9 -3
- ert/run_models/evaluate_ensemble.py +8 -6
- ert/run_models/event.py +7 -3
- ert/run_models/everest_run_model.py +155 -44
- ert/run_models/initial_ensemble_run_model.py +23 -22
- ert/run_models/manual_update.py +4 -2
- ert/run_models/manual_update_enif.py +37 -0
- ert/run_models/model_factory.py +81 -22
- ert/run_models/multiple_data_assimilation.py +21 -10
- ert/run_models/run_model.py +54 -34
- ert/run_models/single_test_run.py +7 -4
- ert/run_models/update_run_model.py +4 -2
- ert/runpaths.py +5 -6
- ert/sample_prior.py +9 -4
- ert/scheduler/driver.py +37 -0
- ert/scheduler/event.py +3 -1
- ert/scheduler/job.py +23 -13
- ert/scheduler/lsf_driver.py +6 -2
- ert/scheduler/openpbs_driver.py +7 -1
- ert/scheduler/scheduler.py +5 -0
- ert/scheduler/slurm_driver.py +6 -2
- ert/services/__init__.py +2 -2
- ert/services/_base_service.py +31 -15
- ert/services/ert_server.py +317 -0
- ert/shared/_doc_utils/ert_jobs.py +1 -4
- ert/shared/storage/connection.py +3 -3
- ert/shared/version.py +3 -3
- ert/storage/local_ensemble.py +25 -5
- ert/storage/local_experiment.py +6 -14
- ert/storage/local_storage.py +35 -30
- ert/storage/migration/to18.py +12 -0
- ert/storage/migration/to8.py +4 -4
- ert/substitutions.py +12 -28
- ert/validation/active_range.py +7 -7
- ert/validation/rangestring.py +16 -16
- {ert-17.1.9.dist-info → ert-18.0.0.dist-info}/METADATA +8 -7
- {ert-17.1.9.dist-info → ert-18.0.0.dist-info}/RECORD +160 -159
- everest/api/everest_data_api.py +1 -14
- everest/bin/config_branch_script.py +3 -6
- everest/bin/everconfigdump_script.py +1 -9
- everest/bin/everest_script.py +21 -11
- everest/bin/kill_script.py +2 -2
- everest/bin/monitor_script.py +2 -2
- everest/bin/utils.py +6 -3
- everest/config/__init__.py +4 -1
- everest/config/control_config.py +61 -2
- everest/config/control_variable_config.py +2 -1
- everest/config/everest_config.py +38 -16
- everest/config/forward_model_config.py +5 -3
- everest/config/install_data_config.py +7 -5
- everest/config/install_job_config.py +7 -3
- everest/config/install_template_config.py +3 -3
- everest/config/optimization_config.py +19 -6
- everest/config/output_constraint_config.py +8 -2
- everest/config/server_config.py +6 -49
- everest/config/utils.py +25 -105
- everest/config/validation_utils.py +10 -10
- everest/config_file_loader.py +13 -2
- everest/detached/everserver.py +7 -8
- everest/everest_storage.py +6 -10
- everest/gui/everest_client.py +0 -1
- everest/gui/main_window.py +2 -2
- everest/optimizer/everest2ropt.py +59 -32
- everest/optimizer/opt_model_transforms.py +12 -13
- everest/optimizer/utils.py +0 -29
- everest/strings.py +0 -5
- ert/config/everest_constraints_config.py +0 -95
- ert/services/storage_service.py +0 -127
- everest/config/sampler_config.py +0 -103
- everest/simulator/__init__.py +0 -88
- everest/simulator/everest_to_ert.py +0 -51
- /ert/gui/{suggestor → ertwidgets/suggestor}/__init__.py +0 -0
- /ert/gui/{suggestor → ertwidgets/suggestor}/_colors.py +0 -0
- /ert/gui/{suggestor → ertwidgets/suggestor}/suggestor.py +0 -0
- {ert-17.1.9.dist-info → ert-18.0.0.dist-info}/WHEEL +0 -0
- {ert-17.1.9.dist-info → ert-18.0.0.dist-info}/entry_points.txt +0 -0
- {ert-17.1.9.dist-info → ert-18.0.0.dist-info}/licenses/COPYING +0 -0
- {ert-17.1.9.dist-info → ert-18.0.0.dist-info}/top_level.txt +0 -0
_ert/events.py
CHANGED
|
@@ -41,10 +41,12 @@ class Id:
|
|
|
41
41
|
ENSEMBLE_SUCCEEDED_TYPE = Literal["ensemble.succeeded"]
|
|
42
42
|
ENSEMBLE_CANCELLED_TYPE = Literal["ensemble.cancelled"]
|
|
43
43
|
ENSEMBLE_FAILED_TYPE = Literal["ensemble.failed"]
|
|
44
|
+
ENSEMBLE_WARNING_TYPE = Literal["ensemble.warning"]
|
|
44
45
|
ENSEMBLE_STARTED: Final = "ensemble.started"
|
|
45
46
|
ENSEMBLE_SUCCEEDED: Final = "ensemble.succeeded"
|
|
46
47
|
ENSEMBLE_CANCELLED: Final = "ensemble.cancelled"
|
|
47
48
|
ENSEMBLE_FAILED: Final = "ensemble.failed"
|
|
49
|
+
ENSEMBLE_WARNING: Final = "ensemble.warning"
|
|
48
50
|
ENSEMBLE_TYPES = (
|
|
49
51
|
ENSEMBLE_STARTED_TYPE
|
|
50
52
|
| ENSEMBLE_FAILED_TYPE
|
|
@@ -170,6 +172,15 @@ class EnsembleCancelled(EnsembleBaseEvent):
|
|
|
170
172
|
event_type: Id.ENSEMBLE_CANCELLED_TYPE = Id.ENSEMBLE_CANCELLED
|
|
171
173
|
|
|
172
174
|
|
|
175
|
+
class EnsembleEvaluationWarning(EnsembleBaseEvent):
|
|
176
|
+
"""This event is to indicate that something unexpected happened while
|
|
177
|
+
running the ensemble, and that it might be stuck in an unresponsive state.
|
|
178
|
+
"""
|
|
179
|
+
|
|
180
|
+
event_type: Id.ENSEMBLE_WARNING_TYPE = Id.ENSEMBLE_WARNING
|
|
181
|
+
warning_message: str
|
|
182
|
+
|
|
183
|
+
|
|
173
184
|
class EESnapshot(EnsembleBaseEvent):
|
|
174
185
|
event_type: Id.EE_SNAPSHOT_TYPE = Id.EE_SNAPSHOT
|
|
175
186
|
snapshot: Any
|
|
@@ -200,9 +211,15 @@ RealizationEvent = (
|
|
|
200
211
|
| RealizationResubmit
|
|
201
212
|
)
|
|
202
213
|
|
|
203
|
-
EnsembleEvent =
|
|
214
|
+
EnsembleEvent = (
|
|
215
|
+
EnsembleStarted
|
|
216
|
+
| EnsembleSucceeded
|
|
217
|
+
| EnsembleFailed
|
|
218
|
+
| EnsembleCancelled
|
|
219
|
+
| EnsembleEvaluationWarning
|
|
220
|
+
)
|
|
204
221
|
|
|
205
|
-
EEEvent = EESnapshot | EESnapshotUpdate
|
|
222
|
+
EEEvent = EESnapshot | EESnapshotUpdate | EnsembleEvaluationWarning
|
|
206
223
|
|
|
207
224
|
SnapshotInputEvent = RealizationEvent | EnsembleEvent | FMEvent
|
|
208
225
|
|
ert/__main__.py
CHANGED
|
@@ -35,8 +35,9 @@ from ert.mode_definitions import (
|
|
|
35
35
|
)
|
|
36
36
|
from ert.namespace import Namespace
|
|
37
37
|
from ert.plugins import ErtRuntimePlugins, get_site_plugins, setup_site_logging
|
|
38
|
-
from ert.run_models.multiple_data_assimilation import
|
|
39
|
-
from ert.services import
|
|
38
|
+
from ert.run_models.multiple_data_assimilation import MultipleDataAssimilationConfig
|
|
39
|
+
from ert.services import ErtServer, WebvizErt
|
|
40
|
+
from ert.shared.status.utils import get_ert_memory_usage
|
|
40
41
|
from ert.shared.storage.command import add_parser_options as ert_api_add_parser_options
|
|
41
42
|
from ert.storage import ErtStorageException, ErtStoragePermissionError
|
|
42
43
|
from ert.trace import trace, tracer
|
|
@@ -53,9 +54,9 @@ logger = logging.getLogger(__name__)
|
|
|
53
54
|
|
|
54
55
|
|
|
55
56
|
def run_ert_storage(args: Namespace, _: ErtRuntimePlugins | None = None) -> None:
|
|
56
|
-
with
|
|
57
|
+
with ErtServer.start_server(
|
|
57
58
|
verbose=True,
|
|
58
|
-
project=ErtConfig.from_file(args.config).ens_path,
|
|
59
|
+
project=Path(ErtConfig.from_file(args.config).ens_path),
|
|
59
60
|
parent_pid=os.getpid(),
|
|
60
61
|
) as server:
|
|
61
62
|
server.wait()
|
|
@@ -80,7 +81,7 @@ def run_webviz_ert(args: Namespace, _: ErtRuntimePlugins | None = None) -> None:
|
|
|
80
81
|
kwargs["ert_config"] = os.path.basename(args.config)
|
|
81
82
|
kwargs["project"] = os.path.abspath(ens_path)
|
|
82
83
|
try:
|
|
83
|
-
with
|
|
84
|
+
with ErtServer.init_service(project=Path(ens_path).absolute()) as storage:
|
|
84
85
|
storage.wait_until_ready()
|
|
85
86
|
print(
|
|
86
87
|
"""
|
|
@@ -501,7 +502,7 @@ def get_ert_parser(parser: ArgumentParser | None = None) -> ArgumentParser:
|
|
|
501
502
|
es_mda_parser.add_argument(
|
|
502
503
|
"--weights",
|
|
503
504
|
type=valid_weights,
|
|
504
|
-
default=
|
|
505
|
+
default=MultipleDataAssimilationConfig.default_weights,
|
|
505
506
|
help="Example custom relative weights: '8,4,2,1'. This means multiple data "
|
|
506
507
|
"assimilation ensemble smoother will half the weight applied to the "
|
|
507
508
|
"observation errors from one iteration to the next across 4 iterations.",
|
|
@@ -598,7 +599,7 @@ def ert_parser(parser: ArgumentParser | None, args: Sequence[str]) -> Namespace:
|
|
|
598
599
|
def log_process_usage() -> None:
|
|
599
600
|
try:
|
|
600
601
|
usage = resource.getrusage(resource.RUSAGE_SELF)
|
|
601
|
-
max_rss =
|
|
602
|
+
max_rss = get_ert_memory_usage()
|
|
602
603
|
|
|
603
604
|
usage_dict: dict[str, int | float] = {
|
|
604
605
|
"User time": usage.ru_utime,
|
ert/analysis/_update_commons.py
CHANGED
|
@@ -61,10 +61,19 @@ def _copy_unupdated_parameters(
|
|
|
61
61
|
|
|
62
62
|
# Copy the non-updated parameter groups from source to target
|
|
63
63
|
# for each active realization
|
|
64
|
+
complete_df: pl.DataFrame | None = None
|
|
64
65
|
for parameter_group in not_updated_parameter_groups:
|
|
65
|
-
source_ensemble.
|
|
66
|
-
|
|
67
|
-
|
|
66
|
+
data = source_ensemble.load_parameters(parameter_group, iens_active_index)
|
|
67
|
+
if isinstance(data, pl.DataFrame):
|
|
68
|
+
if complete_df is None:
|
|
69
|
+
complete_df = data
|
|
70
|
+
else:
|
|
71
|
+
complete_df = complete_df.join(data, on="realization")
|
|
72
|
+
else:
|
|
73
|
+
target_ensemble.save_parameters(dataset=data)
|
|
74
|
+
|
|
75
|
+
if complete_df is not None:
|
|
76
|
+
target_ensemble.save_parameters(complete_df)
|
|
68
77
|
|
|
69
78
|
|
|
70
79
|
def _expand_wildcards(
|
ert/cli/main.py
CHANGED
|
@@ -93,6 +93,7 @@ def run_cli(args: Namespace, runtime_plugins: ErtRuntimePlugins | None = None) -
|
|
|
93
93
|
return
|
|
94
94
|
|
|
95
95
|
status_queue: queue.SimpleQueue[StatusEvents] = queue.SimpleQueue()
|
|
96
|
+
using_local_queuesystem: bool = True
|
|
96
97
|
try:
|
|
97
98
|
with use_runtime_plugins(get_site_plugins()):
|
|
98
99
|
model = create_model(
|
|
@@ -100,6 +101,9 @@ def run_cli(args: Namespace, runtime_plugins: ErtRuntimePlugins | None = None) -
|
|
|
100
101
|
args,
|
|
101
102
|
status_queue,
|
|
102
103
|
)
|
|
104
|
+
using_local_queuesystem = (
|
|
105
|
+
ert_config.queue_config.queue_system == QueueSystem.LOCAL
|
|
106
|
+
)
|
|
103
107
|
except ValueError as e:
|
|
104
108
|
raise ErtCliError(f"{args.mode} was not valid, failed with: {e}") from e
|
|
105
109
|
|
|
@@ -116,17 +120,16 @@ def run_cli(args: Namespace, runtime_plugins: ErtRuntimePlugins | None = None) -
|
|
|
116
120
|
f"and DESIGN_MATRIX ({model.active_realizations.count(True)})"
|
|
117
121
|
)
|
|
118
122
|
|
|
119
|
-
if args.port_range is None and
|
|
123
|
+
if args.port_range is None and using_local_queuesystem:
|
|
120
124
|
# This is within the range for ephemeral ports as defined by
|
|
121
125
|
# most unix flavors https://en.wikipedia.org/wiki/Ephemeral_port
|
|
122
126
|
args.port_range = range(49152, 51819)
|
|
123
127
|
|
|
124
|
-
use_ipc_protocol = model.queue_system == QueueSystem.LOCAL
|
|
125
128
|
evaluator_server_config = EvaluatorServerConfig(
|
|
126
129
|
port_range=None
|
|
127
130
|
if args.port_range is None
|
|
128
131
|
else (min(args.port_range), max(args.port_range) + 1),
|
|
129
|
-
use_ipc_protocol=
|
|
132
|
+
use_ipc_protocol=using_local_queuesystem,
|
|
130
133
|
)
|
|
131
134
|
|
|
132
135
|
if model.check_if_runpath_exists():
|
ert/cli/monitor.py
CHANGED
|
@@ -9,6 +9,7 @@ from typing import TextIO
|
|
|
9
9
|
import humanize
|
|
10
10
|
from tqdm import tqdm
|
|
11
11
|
|
|
12
|
+
from _ert.events import EnsembleEvaluationWarning
|
|
12
13
|
from ert.ensemble_evaluator import (
|
|
13
14
|
EndEvent,
|
|
14
15
|
EnsembleSnapshot,
|
|
@@ -19,6 +20,7 @@ from ert.ensemble_evaluator import identifiers as ids
|
|
|
19
20
|
from ert.ensemble_evaluator.state import (
|
|
20
21
|
COLOR_FAILED,
|
|
21
22
|
COLOR_FINISHED,
|
|
23
|
+
COLOR_WARNING,
|
|
22
24
|
FORWARD_MODEL_STATE_FAILURE,
|
|
23
25
|
REAL_STATE_TO_COLOR,
|
|
24
26
|
)
|
|
@@ -96,6 +98,11 @@ class Monitor:
|
|
|
96
98
|
| RunModelErrorEvent() as event
|
|
97
99
|
):
|
|
98
100
|
event.write_as_csv(output_path)
|
|
101
|
+
case EnsembleEvaluationWarning(warning_message=msg):
|
|
102
|
+
print(
|
|
103
|
+
self._colorize(msg, color=COLOR_WARNING),
|
|
104
|
+
file=self._out,
|
|
105
|
+
)
|
|
99
106
|
|
|
100
107
|
def _print_step_errors(self) -> None:
|
|
101
108
|
failed_steps: dict[str | None, int] = {}
|
ert/config/__init__.py
CHANGED
|
@@ -11,9 +11,8 @@ from .ensemble_config import EnsembleConfig
|
|
|
11
11
|
from .ert_config import ErtConfig, forward_model_step_from_config_contents
|
|
12
12
|
from .ert_plugin import ErtPlugin
|
|
13
13
|
from .ert_script import ErtScript
|
|
14
|
-
from .
|
|
15
|
-
from .
|
|
16
|
-
from .ext_param_config import ExtParamConfig
|
|
14
|
+
from .everest_response import EverestConstraintsConfig, EverestObjectivesConfig
|
|
15
|
+
from .ext_param_config import ExtParamConfig, SamplerConfig
|
|
17
16
|
from .external_ert_script import ExternalErtScript
|
|
18
17
|
from .field import Field, field_transform
|
|
19
18
|
from .forward_model_step import (
|
|
@@ -23,9 +22,13 @@ from .forward_model_step import (
|
|
|
23
22
|
ForwardModelStepPlugin,
|
|
24
23
|
ForwardModelStepValidationError,
|
|
25
24
|
ForwardModelStepWarning,
|
|
25
|
+
SiteInstalledForwardModelStep,
|
|
26
|
+
SiteOrUserForwardModelStep,
|
|
27
|
+
UserInstalledForwardModelStep,
|
|
26
28
|
)
|
|
27
29
|
from .gen_data_config import GenDataConfig
|
|
28
30
|
from .gen_kw_config import DataSource, GenKwConfig, PriorDict
|
|
31
|
+
from .known_response_types import KnownResponseTypes
|
|
29
32
|
from .lint_file import lint_file
|
|
30
33
|
from .model_config import ModelConfig
|
|
31
34
|
from .parameter_config import ParameterCardinality, ParameterConfig, ParameterMetadata
|
|
@@ -45,6 +48,7 @@ from .queue_config import (
|
|
|
45
48
|
QueueConfig,
|
|
46
49
|
)
|
|
47
50
|
from .response_config import InvalidResponseFile, ResponseConfig, ResponseMetadata
|
|
51
|
+
from .rft_config import RFTConfig
|
|
48
52
|
from .summary_config import SummaryConfig
|
|
49
53
|
from .surface_config import SurfaceConfig
|
|
50
54
|
from .workflow import Workflow
|
|
@@ -105,6 +109,7 @@ __all__ = [
|
|
|
105
109
|
"InversionTypeES",
|
|
106
110
|
"KnownQueueOptions",
|
|
107
111
|
"KnownQueueOptionsAdapter",
|
|
112
|
+
"KnownResponseTypes",
|
|
108
113
|
"LegacyWorkflowConfigs",
|
|
109
114
|
"LocalQueueOptions",
|
|
110
115
|
"ModelConfig",
|
|
@@ -125,10 +130,15 @@ __all__ = [
|
|
|
125
130
|
"PriorDict",
|
|
126
131
|
"QueueConfig",
|
|
127
132
|
"QueueSystem",
|
|
133
|
+
"RFTConfig",
|
|
128
134
|
"ResponseConfig",
|
|
129
135
|
"ResponseMetadata",
|
|
136
|
+
"SamplerConfig",
|
|
137
|
+
"SiteInstalledForwardModelStep",
|
|
138
|
+
"SiteOrUserForwardModelStep",
|
|
130
139
|
"SummaryConfig",
|
|
131
140
|
"SurfaceConfig",
|
|
141
|
+
"UserInstalledForwardModelStep",
|
|
132
142
|
"WarningInfo",
|
|
133
143
|
"Workflow",
|
|
134
144
|
"WorkflowConfigs",
|
|
@@ -34,6 +34,7 @@ if TYPE_CHECKING:
|
|
|
34
34
|
|
|
35
35
|
|
|
36
36
|
DEFAULT_TIME_DELTA = timedelta(seconds=30)
|
|
37
|
+
DEFAULT_LOCATION_RANGE_M = 3000
|
|
37
38
|
|
|
38
39
|
|
|
39
40
|
def create_observation_dataframes(
|
|
@@ -286,6 +287,43 @@ def _get_restart(
|
|
|
286
287
|
) from err
|
|
287
288
|
|
|
288
289
|
|
|
290
|
+
def _has_localization(summary_dict: SummaryObservation) -> bool:
|
|
291
|
+
return any(
|
|
292
|
+
[
|
|
293
|
+
summary_dict.location_x is not None,
|
|
294
|
+
summary_dict.location_y is not None,
|
|
295
|
+
summary_dict.location_range is not None,
|
|
296
|
+
]
|
|
297
|
+
)
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
def _validate_localization_values(summary_dict: SummaryObservation) -> None:
|
|
301
|
+
"""The user must provide LOCATION_X and LOCATION_Y to use localization, while
|
|
302
|
+
unprovided LOCATION_RANGE should default to some value.
|
|
303
|
+
|
|
304
|
+
This method assumes the summary dict contains at least one LOCATION key.
|
|
305
|
+
"""
|
|
306
|
+
if summary_dict.location_x is None or summary_dict.location_y is None:
|
|
307
|
+
loc_values = {
|
|
308
|
+
"LOCATION_X": summary_dict.location_x,
|
|
309
|
+
"LOCATION_Y": summary_dict.location_y,
|
|
310
|
+
"LOCATION_RANGE": summary_dict.location_range,
|
|
311
|
+
}
|
|
312
|
+
provided_loc_values = {k: v for k, v in loc_values.items() if v is not None}
|
|
313
|
+
|
|
314
|
+
provided_loc_values_string = ", ".join(
|
|
315
|
+
key.upper() for key in provided_loc_values
|
|
316
|
+
)
|
|
317
|
+
raise ObservationConfigError.with_context(
|
|
318
|
+
f"Localization for observation {summary_dict.name} is misconfigured.\n"
|
|
319
|
+
f"Only {provided_loc_values_string} were provided. To enable "
|
|
320
|
+
f"localization for an observation, ensure that both LOCATION_X and "
|
|
321
|
+
f"LOCATION_Y are defined - or remove LOCATION keywords to disable "
|
|
322
|
+
f"localization.",
|
|
323
|
+
summary_dict,
|
|
324
|
+
)
|
|
325
|
+
|
|
326
|
+
|
|
289
327
|
def _handle_summary_observation(
|
|
290
328
|
summary_dict: SummaryObservation,
|
|
291
329
|
obs_key: str,
|
|
@@ -323,15 +361,23 @@ def _handle_summary_observation(
|
|
|
323
361
|
"Observation uncertainty must be strictly > 0", summary_key
|
|
324
362
|
) from None
|
|
325
363
|
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
)
|
|
364
|
+
data_dict = {
|
|
365
|
+
"response_key": [summary_key],
|
|
366
|
+
"observation_key": [obs_key],
|
|
367
|
+
"time": pl.Series([date]).dt.cast_time_unit("ms"),
|
|
368
|
+
"observations": pl.Series([value], dtype=pl.Float32),
|
|
369
|
+
"std": pl.Series([std_dev], dtype=pl.Float32),
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
if _has_localization(summary_dict):
|
|
373
|
+
_validate_localization_values(summary_dict)
|
|
374
|
+
data_dict["location_x"] = summary_dict.location_x
|
|
375
|
+
data_dict["location_y"] = summary_dict.location_y
|
|
376
|
+
data_dict["location_range"] = (
|
|
377
|
+
summary_dict.location_range or DEFAULT_LOCATION_RANGE_M
|
|
378
|
+
)
|
|
379
|
+
|
|
380
|
+
return pl.DataFrame(data_dict)
|
|
335
381
|
|
|
336
382
|
|
|
337
383
|
def _handle_general_observation(
|
|
@@ -439,9 +485,11 @@ def _handle_general_observation(
|
|
|
439
485
|
raise ObservationConfigError.with_context(
|
|
440
486
|
f"Values ({values}), error ({stds}) and "
|
|
441
487
|
f"index list ({indices}) must be of equal length",
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
488
|
+
(
|
|
489
|
+
general_observation.obs_file
|
|
490
|
+
if general_observation.obs_file is not None
|
|
491
|
+
else ""
|
|
492
|
+
),
|
|
445
493
|
)
|
|
446
494
|
|
|
447
495
|
if np.any(stds <= 0):
|
ert/config/_observations.py
CHANGED
|
@@ -90,10 +90,13 @@ class _SummaryValues:
|
|
|
90
90
|
name: str
|
|
91
91
|
value: float
|
|
92
92
|
key: str #: The :term:`summary key` in the summary response
|
|
93
|
+
location_x: float | None = None
|
|
94
|
+
location_y: float | None = None
|
|
95
|
+
location_range: float | None = None
|
|
93
96
|
|
|
94
97
|
|
|
95
98
|
@dataclass
|
|
96
|
-
class SummaryObservation(ObservationDate,
|
|
99
|
+
class SummaryObservation(ObservationDate, _SummaryValues, ObservationError):
|
|
97
100
|
@classmethod
|
|
98
101
|
def from_obs_dict(cls, directory: str, observation_dict: ObservationDict) -> Self:
|
|
99
102
|
error_mode = ErrorModes.ABS
|
|
@@ -101,6 +104,7 @@ class SummaryObservation(ObservationDate, ObservationError, _SummaryValues):
|
|
|
101
104
|
|
|
102
105
|
date_dict: ObservationDate = ObservationDate()
|
|
103
106
|
float_values: dict[str, float] = {"ERROR_MIN": 0.1}
|
|
107
|
+
localization_values: dict[str, float] = {}
|
|
104
108
|
for key, value in observation_dict.items():
|
|
105
109
|
match key:
|
|
106
110
|
case "type" | "name":
|
|
@@ -121,6 +125,12 @@ class SummaryObservation(ObservationDate, ObservationError, _SummaryValues):
|
|
|
121
125
|
summary_key = value
|
|
122
126
|
case "DATE":
|
|
123
127
|
date_dict.date = value
|
|
128
|
+
case "LOCATION_X":
|
|
129
|
+
localization_values["x"] = validate_float(value, key)
|
|
130
|
+
case "LOCATION_Y":
|
|
131
|
+
localization_values["y"] = validate_float(value, key)
|
|
132
|
+
case "LOCATION_RANGE":
|
|
133
|
+
localization_values["range"] = validate_float(value, key)
|
|
124
134
|
case _:
|
|
125
135
|
raise _unknown_key_error(str(key), observation_dict["name"])
|
|
126
136
|
if "VALUE" not in float_values:
|
|
@@ -137,6 +147,9 @@ class SummaryObservation(ObservationDate, ObservationError, _SummaryValues):
|
|
|
137
147
|
error_min=float_values["ERROR_MIN"],
|
|
138
148
|
key=summary_key,
|
|
139
149
|
value=float_values["VALUE"],
|
|
150
|
+
location_x=localization_values.get("x"),
|
|
151
|
+
location_y=localization_values.get("y"),
|
|
152
|
+
location_range=localization_values.get("range"),
|
|
140
153
|
**date_dict.__dict__,
|
|
141
154
|
)
|
|
142
155
|
|
ert/config/_read_summary.py
CHANGED
|
@@ -12,6 +12,7 @@ import warnings
|
|
|
12
12
|
from collections.abc import Callable, Sequence
|
|
13
13
|
from datetime import datetime, timedelta
|
|
14
14
|
from enum import Enum, auto
|
|
15
|
+
from functools import lru_cache
|
|
15
16
|
|
|
16
17
|
import numpy as np
|
|
17
18
|
import numpy.typing as npt
|
|
@@ -83,29 +84,30 @@ class DateUnit(Enum):
|
|
|
83
84
|
raise InvalidResponseFile(f"Unknown date unit {val}")
|
|
84
85
|
|
|
85
86
|
|
|
87
|
+
@lru_cache
|
|
86
88
|
def _fetch_keys_to_matcher(fetch_keys: Sequence[str]) -> Callable[[str], bool]:
|
|
87
89
|
"""
|
|
88
90
|
Transform the list of keys (with * used as repeated wildcard) into
|
|
89
91
|
a matcher.
|
|
90
92
|
|
|
91
|
-
>>> match = _fetch_keys_to_matcher(
|
|
93
|
+
>>> match = _fetch_keys_to_matcher(("",))
|
|
92
94
|
>>> match("FOPR")
|
|
93
95
|
False
|
|
94
96
|
|
|
95
|
-
>>> match = _fetch_keys_to_matcher(
|
|
97
|
+
>>> match = _fetch_keys_to_matcher(("*",))
|
|
96
98
|
>>> match("FOPR"), match("FO*")
|
|
97
99
|
(True, True)
|
|
98
100
|
|
|
99
101
|
|
|
100
|
-
>>> match = _fetch_keys_to_matcher(
|
|
102
|
+
>>> match = _fetch_keys_to_matcher(("F*PR",))
|
|
101
103
|
>>> match("WOPR"), match("FOPR"), match("FGPR"), match("SOIL")
|
|
102
104
|
(False, True, True, False)
|
|
103
105
|
|
|
104
|
-
>>> match = _fetch_keys_to_matcher(
|
|
106
|
+
>>> match = _fetch_keys_to_matcher(("WGOR:*",))
|
|
105
107
|
>>> match("FOPR"), match("WGOR:OP1"), match("WGOR:OP2"), match("WGOR")
|
|
106
108
|
(False, True, True, False)
|
|
107
109
|
|
|
108
|
-
>>> match = _fetch_keys_to_matcher(
|
|
110
|
+
>>> match = _fetch_keys_to_matcher(("FOPR", "FGPR"))
|
|
109
111
|
>>> match("FOPR"), match("FGPR"), match("WGOR:OP2"), match("WGOR")
|
|
110
112
|
(True, True, False, False)
|
|
111
113
|
"""
|
|
@@ -138,7 +140,7 @@ def _read_spec(
|
|
|
138
140
|
date_index = None
|
|
139
141
|
date_unit_str = None
|
|
140
142
|
|
|
141
|
-
should_load_key = _fetch_keys_to_matcher(fetch_keys)
|
|
143
|
+
should_load_key = _fetch_keys_to_matcher(tuple(fetch_keys))
|
|
142
144
|
|
|
143
145
|
for i, kw in enumerate(keywords):
|
|
144
146
|
try:
|
ert/config/ensemble_config.py
CHANGED
|
@@ -9,26 +9,18 @@ from pydantic import BaseModel, Field, model_validator
|
|
|
9
9
|
|
|
10
10
|
from .ext_param_config import ExtParamConfig
|
|
11
11
|
from .field import Field as FieldConfig
|
|
12
|
-
from .gen_data_config import GenDataConfig
|
|
13
12
|
from .gen_kw_config import GenKwConfig
|
|
13
|
+
from .known_response_types import KNOWN_ERT_RESPONSE_TYPES, KnownErtResponseTypes
|
|
14
14
|
from .parameter_config import ParameterConfig
|
|
15
15
|
from .parsing import ConfigDict, ConfigKeys, ConfigValidationError
|
|
16
16
|
from .response_config import ResponseConfig
|
|
17
|
-
from .summary_config import SummaryConfig
|
|
18
17
|
from .surface_config import SurfaceConfig
|
|
19
18
|
|
|
20
|
-
KnownResponseTypes = SummaryConfig | GenDataConfig
|
|
21
|
-
|
|
22
|
-
_KNOWN_RESPONSE_TYPES = (
|
|
23
|
-
SummaryConfig,
|
|
24
|
-
GenDataConfig,
|
|
25
|
-
)
|
|
26
|
-
|
|
27
19
|
logger = logging.getLogger(__name__)
|
|
28
20
|
|
|
29
21
|
|
|
30
22
|
class EnsembleConfig(BaseModel):
|
|
31
|
-
response_configs: dict[str,
|
|
23
|
+
response_configs: dict[str, KnownErtResponseTypes] = Field(default_factory=dict)
|
|
32
24
|
parameter_configs: dict[
|
|
33
25
|
str, GenKwConfig | FieldConfig | SurfaceConfig | ExtParamConfig
|
|
34
26
|
] = Field(default_factory=dict)
|
|
@@ -131,9 +123,9 @@ class EnsembleConfig(BaseModel):
|
|
|
131
123
|
+ [make_field(f) for f in field_list]
|
|
132
124
|
)
|
|
133
125
|
EnsembleConfig._check_for_duplicate_gen_kw_param_names(gen_kw_cfgs)
|
|
134
|
-
response_configs: list[
|
|
126
|
+
response_configs: list[KnownErtResponseTypes] = []
|
|
135
127
|
|
|
136
|
-
for config_cls in
|
|
128
|
+
for config_cls in KNOWN_ERT_RESPONSE_TYPES:
|
|
137
129
|
instance = config_cls.from_config_dict(config_dict)
|
|
138
130
|
|
|
139
131
|
if instance is not None and instance.keys:
|
|
@@ -151,13 +143,13 @@ class EnsembleConfig(BaseModel):
|
|
|
151
143
|
return self.parameter_configs[key]
|
|
152
144
|
elif key in self.response_configs:
|
|
153
145
|
return self.response_configs[key]
|
|
154
|
-
elif
|
|
146
|
+
elif config := next(
|
|
155
147
|
(c for c in self.response_configs.values() if key in c.keys), None
|
|
156
148
|
):
|
|
157
149
|
# Only hit by blockfs migration
|
|
158
150
|
# returns the same config for one call per
|
|
159
151
|
# response type. Is later deduped before saving to json
|
|
160
|
-
return
|
|
152
|
+
return config
|
|
161
153
|
else:
|
|
162
154
|
raise KeyError(f"The key:{key} is not in the ensemble configuration")
|
|
163
155
|
|
ert/config/ert_config.py
CHANGED
|
@@ -36,6 +36,9 @@ from .forward_model_step import (
|
|
|
36
36
|
ForwardModelStepJSON,
|
|
37
37
|
ForwardModelStepValidationError,
|
|
38
38
|
ForwardModelStepWarning,
|
|
39
|
+
SiteInstalledForwardModelStep,
|
|
40
|
+
SiteOrUserForwardModelStep,
|
|
41
|
+
UserInstalledForwardModelStep,
|
|
39
42
|
)
|
|
40
43
|
from .gen_data_config import GenDataConfig
|
|
41
44
|
from .gen_kw_config import DataSource, GenKwConfig
|
|
@@ -133,6 +136,7 @@ def create_forward_model_json(
|
|
|
133
136
|
env_pr_fm_step = {}
|
|
134
137
|
|
|
135
138
|
context_substitutions = Substitutions(context)
|
|
139
|
+
real_iter_substituter = context_substitutions.real_iter_substituter(iens, itr)
|
|
136
140
|
|
|
137
141
|
class Substituter:
|
|
138
142
|
def __init__(self, fm_step: ForwardModelStep) -> None:
|
|
@@ -146,7 +150,7 @@ def create_forward_model_json(
|
|
|
146
150
|
)
|
|
147
151
|
self.copy_private_args = Substitutions(
|
|
148
152
|
{
|
|
149
|
-
key:
|
|
153
|
+
key: real_iter_substituter.substitute(val)
|
|
150
154
|
for key, val in fm_step.private_args.items()
|
|
151
155
|
}
|
|
152
156
|
)
|
|
@@ -163,7 +167,7 @@ def create_forward_model_json(
|
|
|
163
167
|
string = self.copy_private_args.substitute(
|
|
164
168
|
string, self.substitution_context_hint, 1, warn_max_iter=False
|
|
165
169
|
)
|
|
166
|
-
return
|
|
170
|
+
return real_iter_substituter.substitute(string)
|
|
167
171
|
|
|
168
172
|
def filter_env_dict(self, env_dict: dict[str, str]) -> dict[str, str] | None:
|
|
169
173
|
substituted_dict = {}
|
|
@@ -518,18 +522,16 @@ def workflows_from_dict(
|
|
|
518
522
|
|
|
519
523
|
def installed_forward_model_steps_from_dict(
|
|
520
524
|
config_dict: ConfigDict,
|
|
521
|
-
) -> dict[str,
|
|
525
|
+
) -> dict[str, UserInstalledForwardModelStep]:
|
|
522
526
|
errors: list[ErrorInfo | ConfigValidationError] = []
|
|
523
|
-
fm_steps: dict[str,
|
|
527
|
+
fm_steps: dict[str, UserInstalledForwardModelStep] = {}
|
|
524
528
|
for name, (fm_step_config_file, config_contents) in config_dict.get(
|
|
525
529
|
ConfigKeys.INSTALL_JOB, []
|
|
526
530
|
):
|
|
527
531
|
fm_step_config_file = path.abspath(fm_step_config_file)
|
|
528
532
|
try:
|
|
529
533
|
new_fm_step = forward_model_step_from_config_contents(
|
|
530
|
-
config_contents,
|
|
531
|
-
name=name,
|
|
532
|
-
config_file=fm_step_config_file,
|
|
534
|
+
config_contents, name=name, config_file=fm_step_config_file
|
|
533
535
|
)
|
|
534
536
|
except ConfigValidationError as e:
|
|
535
537
|
errors.append(e)
|
|
@@ -692,6 +694,8 @@ def log_observation_keys(
|
|
|
692
694
|
|
|
693
695
|
RESERVED_KEYWORDS = ["realization", "IENS", "ITER"]
|
|
694
696
|
|
|
697
|
+
USER_CONFIG_SCHEMA = init_user_config_schema()
|
|
698
|
+
|
|
695
699
|
|
|
696
700
|
class ErtConfig(BaseModel):
|
|
697
701
|
DEFAULT_ENSPATH: ClassVar[str] = "storage"
|
|
@@ -720,7 +724,7 @@ class ErtConfig(BaseModel):
|
|
|
720
724
|
|
|
721
725
|
ert_templates: list[tuple[str, str]] = Field(default_factory=list)
|
|
722
726
|
|
|
723
|
-
forward_model_steps: list[
|
|
727
|
+
forward_model_steps: list[SiteOrUserForwardModelStep] = Field(default_factory=list)
|
|
724
728
|
runpath_config: ModelConfig = Field(default_factory=ModelConfig)
|
|
725
729
|
user_config_file: str = "no_config"
|
|
726
730
|
config_path: str = Field(init=False, default="")
|
|
@@ -826,7 +830,7 @@ class ErtConfig(BaseModel):
|
|
|
826
830
|
def with_plugins(runtime_plugins: ErtRuntimePlugins) -> type[ErtConfig]:
|
|
827
831
|
class ErtConfigWithPlugins(ErtConfig):
|
|
828
832
|
PREINSTALLED_FORWARD_MODEL_STEPS: ClassVar[
|
|
829
|
-
Mapping[str,
|
|
833
|
+
Mapping[str, SiteInstalledForwardModelStep]
|
|
830
834
|
] = runtime_plugins.installed_forward_model_steps
|
|
831
835
|
PREINSTALLED_WORKFLOWS = dict(runtime_plugins.installed_workflow_jobs)
|
|
832
836
|
ENV_PR_FM_STEP: ClassVar[dict[str, dict[str, Any]]] = (
|
|
@@ -1303,7 +1307,7 @@ class ErtConfig(BaseModel):
|
|
|
1303
1307
|
@classmethod
|
|
1304
1308
|
def _read_user_config_contents(cls, user_config: str, file_name: str) -> ConfigDict:
|
|
1305
1309
|
return parse_contents(
|
|
1306
|
-
user_config, file_name=file_name, schema=
|
|
1310
|
+
user_config, file_name=file_name, schema=USER_CONFIG_SCHEMA
|
|
1307
1311
|
)
|
|
1308
1312
|
|
|
1309
1313
|
@classmethod
|
|
@@ -1393,8 +1397,10 @@ def uppercase_subkeys_and_stringify_subvalues(
|
|
|
1393
1397
|
|
|
1394
1398
|
|
|
1395
1399
|
def forward_model_step_from_config_contents(
|
|
1396
|
-
config_contents: str,
|
|
1397
|
-
|
|
1400
|
+
config_contents: str,
|
|
1401
|
+
config_file: str,
|
|
1402
|
+
name: str | None = None,
|
|
1403
|
+
) -> UserInstalledForwardModelStep:
|
|
1398
1404
|
if name is None:
|
|
1399
1405
|
name = os.path.basename(config_file)
|
|
1400
1406
|
|
|
@@ -1418,7 +1424,7 @@ def forward_model_step_from_config_contents(
|
|
|
1418
1424
|
environment = {k: v for [k, v] in content_dict.get("ENV", [])}
|
|
1419
1425
|
default_mapping = {k: v for [k, v] in content_dict.get("DEFAULT", [])}
|
|
1420
1426
|
|
|
1421
|
-
return
|
|
1427
|
+
return UserInstalledForwardModelStep(
|
|
1422
1428
|
name=name,
|
|
1423
1429
|
executable=content_dict["EXECUTABLE"],
|
|
1424
1430
|
stdin_file=content_dict.get("STDIN"),
|
|
@@ -11,13 +11,15 @@ from .response_config import InvalidResponseFile, ResponseConfig, ResponseMetada
|
|
|
11
11
|
from .responses_index import responses_index
|
|
12
12
|
|
|
13
13
|
|
|
14
|
-
class
|
|
15
|
-
|
|
16
|
-
|
|
14
|
+
class EverestResponse(ResponseConfig):
|
|
15
|
+
"""Base class for Everest response configurations."""
|
|
16
|
+
|
|
17
17
|
has_finalized_keys: bool = True
|
|
18
|
-
weights: list[float | None]
|
|
19
18
|
scales: list[float | None]
|
|
20
|
-
|
|
19
|
+
|
|
20
|
+
@property
|
|
21
|
+
def primary_key(self) -> list[str]:
|
|
22
|
+
return []
|
|
21
23
|
|
|
22
24
|
@property
|
|
23
25
|
def metadata(self) -> list[ResponseMetadata]:
|
|
@@ -52,7 +54,6 @@ class EverestObjectivesConfig(ResponseConfig):
|
|
|
52
54
|
)
|
|
53
55
|
|
|
54
56
|
errors = []
|
|
55
|
-
|
|
56
57
|
run_path_ = Path(run_path)
|
|
57
58
|
datasets_per_name = []
|
|
58
59
|
|
|
@@ -86,13 +87,23 @@ class EverestObjectivesConfig(ResponseConfig):
|
|
|
86
87
|
combined = pl.concat(datasets_per_name)
|
|
87
88
|
return combined
|
|
88
89
|
|
|
89
|
-
@property
|
|
90
|
-
def response_type(self) -> str:
|
|
91
|
-
return "everest_objectives"
|
|
92
90
|
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
91
|
+
class EverestConstraintsConfig(EverestResponse):
|
|
92
|
+
type: Literal["everest_constraints"] = "everest_constraints"
|
|
93
|
+
name: str = "everest_constraints"
|
|
94
|
+
targets: list[float | None]
|
|
95
|
+
upper_bounds: list[float]
|
|
96
|
+
lower_bounds: list[float]
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
responses_index.add_response_type(EverestConstraintsConfig)
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
class EverestObjectivesConfig(EverestResponse):
|
|
103
|
+
type: Literal["everest_objectives"] = "everest_objectives"
|
|
104
|
+
name: str = "everest_objectives"
|
|
105
|
+
weights: list[float | None]
|
|
106
|
+
objective_types: list[Literal["mean", "stddev"]]
|
|
96
107
|
|
|
97
108
|
|
|
98
109
|
responses_index.add_response_type(EverestObjectivesConfig)
|