ert 17.0.0__py3-none-any.whl → 19.0.0rc2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- _ert/events.py +19 -2
- _ert/forward_model_runner/client.py +6 -2
- ert/__main__.py +28 -13
- ert/analysis/_enif_update.py +8 -4
- ert/analysis/_es_update.py +19 -6
- ert/analysis/_update_commons.py +16 -6
- ert/cli/main.py +13 -6
- ert/cli/monitor.py +7 -0
- ert/config/__init__.py +15 -6
- ert/config/_create_observation_dataframes.py +117 -20
- ert/config/_get_num_cpu.py +1 -1
- ert/config/_observations.py +91 -2
- ert/config/_read_summary.py +8 -6
- ert/config/design_matrix.py +51 -24
- ert/config/distribution.py +1 -1
- ert/config/ensemble_config.py +9 -17
- ert/config/ert_config.py +103 -19
- ert/config/everest_control.py +234 -0
- ert/config/{everest_objective_config.py → everest_response.py} +24 -15
- ert/config/field.py +96 -84
- ert/config/forward_model_step.py +122 -17
- ert/config/gen_data_config.py +5 -10
- ert/config/gen_kw_config.py +5 -35
- ert/config/known_response_types.py +14 -0
- ert/config/parameter_config.py +1 -33
- ert/config/parsing/_option_dict.py +10 -2
- ert/config/parsing/config_keywords.py +2 -0
- ert/config/parsing/config_schema.py +23 -3
- ert/config/parsing/config_schema_deprecations.py +3 -14
- ert/config/parsing/config_schema_item.py +26 -11
- ert/config/parsing/context_values.py +3 -3
- ert/config/parsing/file_context_token.py +1 -1
- ert/config/parsing/observations_parser.py +6 -2
- ert/config/parsing/queue_system.py +9 -0
- ert/config/parsing/schema_item_type.py +1 -0
- ert/config/queue_config.py +4 -5
- ert/config/response_config.py +0 -8
- ert/config/rft_config.py +275 -0
- ert/config/summary_config.py +3 -8
- ert/config/surface_config.py +59 -16
- ert/config/workflow_fixtures.py +2 -1
- ert/dark_storage/client/__init__.py +2 -2
- ert/dark_storage/client/_session.py +4 -4
- ert/dark_storage/client/client.py +2 -2
- ert/dark_storage/common.py +1 -1
- ert/dark_storage/compute/misfits.py +11 -7
- ert/dark_storage/endpoints/compute/misfits.py +6 -4
- ert/dark_storage/endpoints/experiment_server.py +12 -9
- ert/dark_storage/endpoints/experiments.py +2 -2
- ert/dark_storage/endpoints/observations.py +8 -6
- ert/dark_storage/endpoints/parameters.py +2 -18
- ert/dark_storage/endpoints/responses.py +24 -5
- ert/dark_storage/json_schema/experiment.py +1 -1
- ert/data/_measured_data.py +6 -5
- ert/ensemble_evaluator/__init__.py +8 -1
- ert/ensemble_evaluator/config.py +2 -1
- ert/ensemble_evaluator/evaluator.py +81 -29
- ert/ensemble_evaluator/event.py +6 -0
- ert/ensemble_evaluator/snapshot.py +3 -1
- ert/ensemble_evaluator/state.py +1 -0
- ert/field_utils/__init__.py +8 -0
- ert/field_utils/field_utils.py +212 -3
- ert/field_utils/roff_io.py +1 -1
- ert/gui/__init__.py +5 -2
- ert/gui/ertnotifier.py +1 -1
- ert/gui/ertwidgets/__init__.py +23 -16
- ert/gui/ertwidgets/analysismoduleedit.py +2 -2
- ert/gui/ertwidgets/checklist.py +1 -1
- ert/gui/ertwidgets/create_experiment_dialog.py +3 -1
- ert/gui/ertwidgets/ensembleselector.py +2 -2
- ert/gui/ertwidgets/models/__init__.py +2 -0
- ert/gui/ertwidgets/models/activerealizationsmodel.py +2 -1
- ert/gui/ertwidgets/models/path_model.py +1 -1
- ert/gui/ertwidgets/models/targetensemblemodel.py +2 -1
- ert/gui/ertwidgets/models/text_model.py +1 -1
- ert/gui/ertwidgets/pathchooser.py +0 -3
- ert/gui/ertwidgets/searchbox.py +13 -4
- ert/gui/{suggestor → ertwidgets/suggestor}/_suggestor_message.py +13 -4
- ert/gui/{suggestor → ertwidgets/suggestor}/suggestor.py +63 -30
- ert/gui/main.py +37 -8
- ert/gui/main_window.py +1 -7
- ert/gui/simulation/ensemble_experiment_panel.py +1 -1
- ert/gui/simulation/ensemble_information_filter_panel.py +1 -1
- ert/gui/simulation/ensemble_smoother_panel.py +1 -1
- ert/gui/simulation/evaluate_ensemble_panel.py +1 -1
- ert/gui/simulation/experiment_panel.py +16 -3
- ert/gui/simulation/manual_update_panel.py +31 -8
- ert/gui/simulation/multiple_data_assimilation_panel.py +12 -8
- ert/gui/simulation/run_dialog.py +27 -20
- ert/gui/simulation/single_test_run_panel.py +2 -2
- ert/gui/summarypanel.py +20 -1
- ert/gui/tools/load_results/load_results_panel.py +1 -1
- ert/gui/tools/manage_experiments/export_dialog.py +136 -0
- ert/gui/tools/manage_experiments/storage_info_widget.py +121 -16
- ert/gui/tools/manage_experiments/storage_widget.py +1 -2
- ert/gui/tools/plot/plot_api.py +37 -25
- ert/gui/tools/plot/plot_widget.py +10 -2
- ert/gui/tools/plot/plot_window.py +38 -18
- ert/gui/tools/plot/plottery/plot_config.py +2 -0
- ert/gui/tools/plot/plottery/plot_context.py +14 -0
- ert/gui/tools/plot/plottery/plots/__init__.py +2 -0
- ert/gui/tools/plot/plottery/plots/cesp.py +3 -1
- ert/gui/tools/plot/plottery/plots/distribution.py +6 -1
- ert/gui/tools/plot/plottery/plots/ensemble.py +12 -3
- ert/gui/tools/plot/plottery/plots/gaussian_kde.py +12 -2
- ert/gui/tools/plot/plottery/plots/histogram.py +3 -1
- ert/gui/tools/plot/plottery/plots/misfits.py +436 -0
- ert/gui/tools/plot/plottery/plots/observations.py +18 -4
- ert/gui/tools/plot/plottery/plots/statistics.py +62 -20
- ert/gui/tools/plot/plottery/plots/std_dev.py +3 -1
- ert/mode_definitions.py +2 -0
- ert/plugins/__init__.py +0 -1
- ert/plugins/hook_implementations/workflows/csv_export.py +2 -3
- ert/plugins/hook_implementations/workflows/gen_data_rft_export.py +10 -2
- ert/plugins/hook_specifications/__init__.py +0 -2
- ert/plugins/hook_specifications/jobs.py +0 -9
- ert/plugins/plugin_manager.py +6 -33
- ert/resources/forward_models/run_reservoirsimulator.py +8 -3
- ert/resources/shell_scripts/delete_directory.py +2 -2
- ert/run_models/__init__.py +18 -5
- ert/run_models/_create_run_path.py +131 -37
- ert/run_models/ensemble_experiment.py +10 -4
- ert/run_models/ensemble_information_filter.py +8 -1
- ert/run_models/ensemble_smoother.py +9 -3
- ert/run_models/evaluate_ensemble.py +8 -6
- ert/run_models/event.py +7 -3
- ert/run_models/everest_run_model.py +159 -46
- ert/run_models/initial_ensemble_run_model.py +25 -24
- ert/run_models/manual_update.py +6 -3
- ert/run_models/manual_update_enif.py +37 -0
- ert/run_models/model_factory.py +81 -21
- ert/run_models/multiple_data_assimilation.py +22 -11
- ert/run_models/run_model.py +64 -55
- ert/run_models/single_test_run.py +7 -4
- ert/run_models/update_run_model.py +4 -2
- ert/runpaths.py +5 -6
- ert/sample_prior.py +9 -4
- ert/scheduler/driver.py +37 -0
- ert/scheduler/event.py +3 -1
- ert/scheduler/job.py +23 -13
- ert/scheduler/lsf_driver.py +6 -2
- ert/scheduler/openpbs_driver.py +7 -1
- ert/scheduler/scheduler.py +5 -0
- ert/scheduler/slurm_driver.py +6 -2
- ert/services/__init__.py +2 -2
- ert/services/_base_service.py +37 -20
- ert/services/ert_server.py +317 -0
- ert/shared/_doc_utils/__init__.py +4 -2
- ert/shared/_doc_utils/ert_jobs.py +1 -4
- ert/shared/net_utils.py +43 -18
- ert/shared/storage/connection.py +3 -3
- ert/shared/version.py +3 -3
- ert/storage/__init__.py +2 -0
- ert/storage/local_ensemble.py +38 -12
- ert/storage/local_experiment.py +8 -16
- ert/storage/local_storage.py +68 -42
- ert/storage/migration/to11.py +1 -1
- ert/storage/migration/to16.py +38 -0
- ert/storage/migration/to17.py +42 -0
- ert/storage/migration/to18.py +11 -0
- ert/storage/migration/to19.py +34 -0
- ert/storage/migration/to20.py +23 -0
- ert/storage/migration/to21.py +25 -0
- ert/storage/migration/to8.py +4 -4
- ert/substitutions.py +12 -28
- ert/validation/active_range.py +7 -7
- ert/validation/rangestring.py +16 -16
- ert/workflow_runner.py +2 -1
- {ert-17.0.0.dist-info → ert-19.0.0rc2.dist-info}/METADATA +9 -8
- {ert-17.0.0.dist-info → ert-19.0.0rc2.dist-info}/RECORD +208 -205
- {ert-17.0.0.dist-info → ert-19.0.0rc2.dist-info}/WHEEL +1 -1
- everest/api/everest_data_api.py +14 -1
- everest/bin/config_branch_script.py +3 -6
- everest/bin/everconfigdump_script.py +1 -9
- everest/bin/everest_script.py +21 -11
- everest/bin/everlint_script.py +0 -2
- everest/bin/kill_script.py +2 -2
- everest/bin/monitor_script.py +2 -2
- everest/bin/utils.py +8 -4
- everest/bin/visualization_script.py +6 -14
- everest/config/__init__.py +4 -1
- everest/config/control_config.py +81 -6
- everest/config/control_variable_config.py +4 -3
- everest/config/everest_config.py +75 -42
- everest/config/forward_model_config.py +5 -3
- everest/config/install_data_config.py +7 -5
- everest/config/install_job_config.py +7 -3
- everest/config/install_template_config.py +3 -3
- everest/config/optimization_config.py +19 -6
- everest/config/output_constraint_config.py +8 -2
- everest/config/server_config.py +6 -49
- everest/config/utils.py +25 -105
- everest/config/validation_utils.py +17 -11
- everest/config_file_loader.py +13 -4
- everest/detached/client.py +3 -3
- everest/detached/everserver.py +7 -8
- everest/everest_storage.py +6 -12
- everest/gui/everest_client.py +2 -3
- everest/gui/main_window.py +2 -2
- everest/optimizer/everest2ropt.py +59 -32
- everest/optimizer/opt_model_transforms.py +12 -13
- everest/optimizer/utils.py +0 -29
- everest/strings.py +0 -5
- ert/config/everest_constraints_config.py +0 -95
- ert/config/ext_param_config.py +0 -106
- ert/gui/tools/export/__init__.py +0 -3
- ert/gui/tools/export/export_panel.py +0 -83
- ert/gui/tools/export/export_tool.py +0 -69
- ert/gui/tools/export/exporter.py +0 -36
- ert/services/storage_service.py +0 -127
- everest/config/sampler_config.py +0 -103
- everest/simulator/__init__.py +0 -88
- everest/simulator/everest_to_ert.py +0 -51
- /ert/gui/{suggestor → ertwidgets/suggestor}/__init__.py +0 -0
- /ert/gui/{suggestor → ertwidgets/suggestor}/_colors.py +0 -0
- {ert-17.0.0.dist-info → ert-19.0.0rc2.dist-info}/entry_points.txt +0 -0
- {ert-17.0.0.dist-info → ert-19.0.0rc2.dist-info}/licenses/COPYING +0 -0
- {ert-17.0.0.dist-info → ert-19.0.0rc2.dist-info}/top_level.txt +0 -0
|
@@ -111,7 +111,7 @@ class CSVExportJob(ErtScript):
|
|
|
111
111
|
summary_data = pl.DataFrame({})
|
|
112
112
|
|
|
113
113
|
if not summary_data.is_empty():
|
|
114
|
-
pivoted_summary = summary_data.pivot(
|
|
114
|
+
pivoted_summary = summary_data.pivot( # noqa: PD010
|
|
115
115
|
index=["realization", "time"], on="response_key", values="values"
|
|
116
116
|
).to_pandas()
|
|
117
117
|
|
|
@@ -123,8 +123,7 @@ class CSVExportJob(ErtScript):
|
|
|
123
123
|
# Reset index to make 'Realization' a regular column
|
|
124
124
|
ensemble_data = ensemble_data.reset_index()
|
|
125
125
|
|
|
126
|
-
ensemble_data =
|
|
127
|
-
ensemble_data,
|
|
126
|
+
ensemble_data = ensemble_data.merge(
|
|
128
127
|
pivoted_summary,
|
|
129
128
|
on="Realization",
|
|
130
129
|
how="left",
|
|
@@ -1,19 +1,23 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
1
3
|
import contextlib
|
|
2
4
|
import json
|
|
3
5
|
import logging
|
|
4
6
|
import os
|
|
5
7
|
from collections.abc import Sequence
|
|
6
|
-
from typing import Any
|
|
8
|
+
from typing import TYPE_CHECKING, Any
|
|
7
9
|
|
|
8
10
|
import numpy as np
|
|
9
11
|
import pandas as pd
|
|
10
12
|
import polars as pl
|
|
11
|
-
from PyQt6.QtWidgets import QCheckBox, QWidget
|
|
12
13
|
|
|
13
14
|
from ert.config import ErtPlugin
|
|
14
15
|
from ert.plugins import CancelPluginException
|
|
15
16
|
from ert.storage import Storage
|
|
16
17
|
|
|
18
|
+
if TYPE_CHECKING:
|
|
19
|
+
from PyQt6.QtWidgets import QWidget
|
|
20
|
+
|
|
17
21
|
logger = logging.getLogger(__name__)
|
|
18
22
|
|
|
19
23
|
|
|
@@ -240,6 +244,10 @@ class GenDataRFTCSVExportJob(ErtPlugin):
|
|
|
240
244
|
list_edit = ListEditBox(ensemble_with_data_dict)
|
|
241
245
|
list_edit.setObjectName("list_of_ensembles")
|
|
242
246
|
|
|
247
|
+
# Lazy load qt outside of gui to allow
|
|
248
|
+
# running of ert in cli without wm
|
|
249
|
+
from PyQt6.QtWidgets import QCheckBox # noqa: PLC0415
|
|
250
|
+
|
|
243
251
|
drop_const_columns_check = QCheckBox()
|
|
244
252
|
drop_const_columns_check.setChecked(False)
|
|
245
253
|
drop_const_columns_check.setObjectName("drop_const_columns_check")
|
|
@@ -5,7 +5,6 @@ from .forward_model_steps import (
|
|
|
5
5
|
from .help_resources import help_links
|
|
6
6
|
from .jobs import (
|
|
7
7
|
ertscript_workflow,
|
|
8
|
-
installable_jobs,
|
|
9
8
|
installable_workflow_jobs,
|
|
10
9
|
job_documentation,
|
|
11
10
|
legacy_ertscript_workflow,
|
|
@@ -20,7 +19,6 @@ __all__ = [
|
|
|
20
19
|
"forward_model_configuration",
|
|
21
20
|
"help_links",
|
|
22
21
|
"installable_forward_model_steps",
|
|
23
|
-
"installable_jobs",
|
|
24
22
|
"installable_workflow_jobs",
|
|
25
23
|
"job_documentation",
|
|
26
24
|
"legacy_ertscript_workflow",
|
|
@@ -9,15 +9,6 @@ if TYPE_CHECKING:
|
|
|
9
9
|
from ert.plugins.plugin_response import PluginResponse
|
|
10
10
|
|
|
11
11
|
|
|
12
|
-
@no_type_check
|
|
13
|
-
@hook_specification
|
|
14
|
-
def installable_jobs() -> PluginResponse[dict[str, str]]:
|
|
15
|
-
"""
|
|
16
|
-
:return: dict with job names as keys and path to config as value
|
|
17
|
-
:rtype: PluginResponse with data as dict[str,str]
|
|
18
|
-
"""
|
|
19
|
-
|
|
20
|
-
|
|
21
12
|
@no_type_check
|
|
22
13
|
@hook_specification(firstresult=True)
|
|
23
14
|
def job_documentation(job_name: str) -> PluginResponse[dict[str, str] | None]:
|
ert/plugins/plugin_manager.py
CHANGED
|
@@ -5,7 +5,6 @@ import logging
|
|
|
5
5
|
import warnings
|
|
6
6
|
from argparse import ArgumentParser
|
|
7
7
|
from collections.abc import Callable, Mapping, Sequence
|
|
8
|
-
from pathlib import Path
|
|
9
8
|
from typing import TYPE_CHECKING, Any, Literal, TypeVar, overload
|
|
10
9
|
|
|
11
10
|
import pluggy
|
|
@@ -13,15 +12,14 @@ from pydantic import BaseModel, Field
|
|
|
13
12
|
from typing_extensions import TypedDict
|
|
14
13
|
|
|
15
14
|
from ert.config import (
|
|
16
|
-
ForwardModelStep,
|
|
17
15
|
ForwardModelStepDocumentation,
|
|
18
16
|
ForwardModelStepPlugin,
|
|
19
17
|
KnownQueueOptions,
|
|
20
18
|
LegacyWorkflowConfigs,
|
|
21
19
|
LocalQueueOptions,
|
|
20
|
+
SiteInstalledForwardModelStep,
|
|
22
21
|
WorkflowConfigs,
|
|
23
22
|
WorkflowJob,
|
|
24
|
-
forward_model_step_from_config_contents,
|
|
25
23
|
workflow_job_from_file,
|
|
26
24
|
)
|
|
27
25
|
from ert.trace import add_span_processor
|
|
@@ -251,32 +249,9 @@ class ErtPluginManager(pluggy.PluginManager):
|
|
|
251
249
|
return merged_dict
|
|
252
250
|
return {k: v[0] for k, v in merged_dict.items()}
|
|
253
251
|
|
|
254
|
-
def get_installable_jobs(self) -> Mapping[str, str]:
|
|
255
|
-
return ErtPluginManager._merge_dicts(self.hook.installable_jobs())
|
|
256
|
-
|
|
257
252
|
def _get_config_workflow_jobs(self) -> dict[str, str]:
|
|
258
253
|
return ErtPluginManager._merge_dicts(self.hook.installable_workflow_jobs())
|
|
259
254
|
|
|
260
|
-
def get_documentation_for_jobs(self) -> dict[str, Any]:
|
|
261
|
-
job_docs = {
|
|
262
|
-
k: {
|
|
263
|
-
"config_file": v[0],
|
|
264
|
-
"source_package": v[1].plugin_name,
|
|
265
|
-
"source_function_name": v[1].function_name,
|
|
266
|
-
}
|
|
267
|
-
for k, v in ErtPluginManager._merge_dicts(
|
|
268
|
-
self.hook.installable_jobs(), include_plugin_data=True
|
|
269
|
-
).items()
|
|
270
|
-
}
|
|
271
|
-
for key, value in job_docs.items():
|
|
272
|
-
value.update(
|
|
273
|
-
ErtPluginManager._evaluate_job_doc_hook(
|
|
274
|
-
self.hook.job_documentation,
|
|
275
|
-
key,
|
|
276
|
-
)
|
|
277
|
-
)
|
|
278
|
-
return job_docs
|
|
279
|
-
|
|
280
255
|
def get_documentation_for_forward_model_steps(
|
|
281
256
|
self,
|
|
282
257
|
) -> dict[str, ForwardModelStepDocumentation]:
|
|
@@ -353,7 +328,7 @@ class ErtPluginManager(pluggy.PluginManager):
|
|
|
353
328
|
|
|
354
329
|
|
|
355
330
|
class ErtRuntimePlugins(BaseModel):
|
|
356
|
-
installed_forward_model_steps: Mapping[str,
|
|
331
|
+
installed_forward_model_steps: Mapping[str, SiteInstalledForwardModelStep] = Field(
|
|
357
332
|
default_factory=dict
|
|
358
333
|
)
|
|
359
334
|
installed_workflow_jobs: Mapping[str, WorkflowJob] = Field(default_factory=dict)
|
|
@@ -363,6 +338,7 @@ class ErtRuntimePlugins(BaseModel):
|
|
|
363
338
|
environment_variables: Mapping[str, str] = Field(default_factory=dict)
|
|
364
339
|
env_pr_fm_step: Mapping[str, Mapping[str, Any]] = Field(default_factory=dict)
|
|
365
340
|
help_links: dict[str, str] = Field(default_factory=dict)
|
|
341
|
+
prioritize_private_ip_address: bool = False
|
|
366
342
|
|
|
367
343
|
|
|
368
344
|
def get_site_plugins(
|
|
@@ -380,12 +356,6 @@ def get_site_plugins(
|
|
|
380
356
|
else {}
|
|
381
357
|
)
|
|
382
358
|
|
|
383
|
-
for job_name, job_path in plugin_manager.get_installable_jobs().items():
|
|
384
|
-
fm_step = forward_model_step_from_config_contents(
|
|
385
|
-
Path(job_path).read_text(encoding="utf-8"), job_path, job_name
|
|
386
|
-
)
|
|
387
|
-
all_forward_model_steps[job_name] = fm_step
|
|
388
|
-
|
|
389
359
|
all_workflow_jobs: dict[str, WorkflowJob] = dict[str, WorkflowJob](
|
|
390
360
|
plugin_manager.get_ertscript_workflows().get_workflows()
|
|
391
361
|
) | dict[str, WorkflowJob](
|
|
@@ -417,6 +387,9 @@ def get_site_plugins(
|
|
|
417
387
|
),
|
|
418
388
|
env_pr_fm_step=plugin_manager.get_forward_model_configuration(),
|
|
419
389
|
help_links=plugin_manager.get_help_links(),
|
|
390
|
+
prioritize_private_ip_address=site_configurations.prioritize_private_ip_address
|
|
391
|
+
if site_configurations
|
|
392
|
+
else False,
|
|
420
393
|
)
|
|
421
394
|
|
|
422
395
|
return runtime_plugins
|
|
@@ -8,10 +8,9 @@ import subprocess
|
|
|
8
8
|
import sys
|
|
9
9
|
import time
|
|
10
10
|
from argparse import ArgumentParser
|
|
11
|
-
from collections import namedtuple
|
|
12
11
|
from pathlib import Path
|
|
13
12
|
from random import random
|
|
14
|
-
from typing import Literal, get_args
|
|
13
|
+
from typing import Literal, NamedTuple, get_args
|
|
15
14
|
|
|
16
15
|
import resfo
|
|
17
16
|
|
|
@@ -43,7 +42,13 @@ class EclError(RuntimeError):
|
|
|
43
42
|
|
|
44
43
|
|
|
45
44
|
Simulators = Literal["flow", "eclipse", "e300"]
|
|
46
|
-
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class EclipseResult(NamedTuple):
|
|
48
|
+
errors: int
|
|
49
|
+
bugs: int
|
|
50
|
+
|
|
51
|
+
|
|
47
52
|
body_sub_pattern = r"(\s^\s@.+$)*"
|
|
48
53
|
date_sub_pattern = r"\s+AT TIME\s+(?P<Days>\d+\.\d+)\s+DAYS\s+\((?P<Date>(.+)):\s*$"
|
|
49
54
|
error_pattern_e100 = (
|
|
@@ -48,8 +48,8 @@ def delete_directory(path: str) -> None:
|
|
|
48
48
|
for file in files:
|
|
49
49
|
delete_file(os.path.join(root, file))
|
|
50
50
|
|
|
51
|
-
for
|
|
52
|
-
delete_empty_directory(os.path.join(root,
|
|
51
|
+
for dir_ in dirs:
|
|
52
|
+
delete_empty_directory(os.path.join(root, dir_))
|
|
53
53
|
|
|
54
54
|
else:
|
|
55
55
|
raise OSError(f"Entry:'{path}' is not a directory")
|
ert/run_models/__init__.py
CHANGED
|
@@ -1,6 +1,10 @@
|
|
|
1
|
-
from .ensemble_experiment import EnsembleExperiment
|
|
2
|
-
from .ensemble_information_filter import
|
|
3
|
-
|
|
1
|
+
from .ensemble_experiment import EnsembleExperiment, EnsembleExperimentConfig
|
|
2
|
+
from .ensemble_information_filter import (
|
|
3
|
+
EnsembleInformationFilter,
|
|
4
|
+
EnsembleInformationFilterConfig,
|
|
5
|
+
)
|
|
6
|
+
from .ensemble_smoother import EnsembleSmoother, EnsembleSmootherConfig
|
|
7
|
+
from .evaluate_ensemble import EvaluateEnsembleConfig
|
|
4
8
|
from .event import (
|
|
5
9
|
RunModelEvent,
|
|
6
10
|
RunModelStatusEvent,
|
|
@@ -9,21 +13,29 @@ from .event import (
|
|
|
9
13
|
RunModelUpdateEndEvent,
|
|
10
14
|
)
|
|
11
15
|
from .model_factory import create_model
|
|
12
|
-
from .multiple_data_assimilation import
|
|
16
|
+
from .multiple_data_assimilation import (
|
|
17
|
+
MultipleDataAssimilation,
|
|
18
|
+
MultipleDataAssimilationConfig,
|
|
19
|
+
)
|
|
13
20
|
from .run_model import (
|
|
14
21
|
ErtRunError,
|
|
15
22
|
RunModel,
|
|
16
23
|
RunModelAPI,
|
|
17
24
|
StatusEvents,
|
|
18
25
|
)
|
|
19
|
-
from .single_test_run import SingleTestRun
|
|
26
|
+
from .single_test_run import SingleTestRun, SingleTestRunConfig
|
|
20
27
|
|
|
21
28
|
__all__ = [
|
|
22
29
|
"EnsembleExperiment",
|
|
30
|
+
"EnsembleExperimentConfig",
|
|
23
31
|
"EnsembleInformationFilter",
|
|
32
|
+
"EnsembleInformationFilterConfig",
|
|
24
33
|
"EnsembleSmoother",
|
|
34
|
+
"EnsembleSmootherConfig",
|
|
25
35
|
"ErtRunError",
|
|
36
|
+
"EvaluateEnsembleConfig",
|
|
26
37
|
"MultipleDataAssimilation",
|
|
38
|
+
"MultipleDataAssimilationConfig",
|
|
27
39
|
"RunModel",
|
|
28
40
|
"RunModelAPI",
|
|
29
41
|
"RunModelEvent",
|
|
@@ -32,6 +44,7 @@ __all__ = [
|
|
|
32
44
|
"RunModelUpdateBeginEvent",
|
|
33
45
|
"RunModelUpdateEndEvent",
|
|
34
46
|
"SingleTestRun",
|
|
47
|
+
"SingleTestRunConfig",
|
|
35
48
|
"StatusEvents",
|
|
36
49
|
"create_model",
|
|
37
50
|
]
|
|
@@ -2,8 +2,12 @@ from __future__ import annotations
|
|
|
2
2
|
|
|
3
3
|
import json
|
|
4
4
|
import logging
|
|
5
|
+
import math
|
|
5
6
|
import os
|
|
7
|
+
import time
|
|
8
|
+
from collections import defaultdict
|
|
6
9
|
from collections.abc import Iterable, Mapping
|
|
10
|
+
from copy import deepcopy
|
|
7
11
|
from datetime import UTC, datetime
|
|
8
12
|
from pathlib import Path
|
|
9
13
|
from typing import TYPE_CHECKING, Any
|
|
@@ -12,13 +16,16 @@ import orjson
|
|
|
12
16
|
|
|
13
17
|
from _ert.utils import file_safe_timestamp
|
|
14
18
|
from ert.config import (
|
|
15
|
-
|
|
19
|
+
EverestControl,
|
|
16
20
|
Field,
|
|
17
21
|
ForwardModelStep,
|
|
18
22
|
GenKwConfig,
|
|
23
|
+
ParameterCardinality,
|
|
19
24
|
ParameterConfig,
|
|
20
25
|
SurfaceConfig,
|
|
21
26
|
)
|
|
27
|
+
from ert.config.design_matrix import DESIGN_MATRIX_GROUP
|
|
28
|
+
from ert.config.distribution import LogNormalSettings, LogUnifSettings
|
|
22
29
|
from ert.config.ert_config import create_forward_model_json
|
|
23
30
|
from ert.substitutions import Substitutions, substitute_runpath_name
|
|
24
31
|
from ert.utils import log_duration
|
|
@@ -53,10 +60,17 @@ def _value_export_txt(
|
|
|
53
60
|
with path.open("w") as f:
|
|
54
61
|
for key, param_map in values.items():
|
|
55
62
|
for param, value in param_map.items():
|
|
56
|
-
if isinstance(value,
|
|
57
|
-
|
|
63
|
+
if isinstance(value, int):
|
|
64
|
+
value_str = str(value)
|
|
65
|
+
elif isinstance(value, float):
|
|
66
|
+
value_str = f"{value:g}"
|
|
58
67
|
else:
|
|
59
|
-
|
|
68
|
+
value_str = str(value)
|
|
69
|
+
|
|
70
|
+
if key == DESIGN_MATRIX_GROUP:
|
|
71
|
+
print(f"{param} {value_str}", file=f)
|
|
72
|
+
else:
|
|
73
|
+
print(f"{key}:{param} {value_str}", file=f)
|
|
60
74
|
|
|
61
75
|
|
|
62
76
|
def _value_export_json(
|
|
@@ -70,18 +84,18 @@ def _value_export_json(
|
|
|
70
84
|
if len(values) == 0:
|
|
71
85
|
return
|
|
72
86
|
|
|
73
|
-
#
|
|
74
|
-
json_out: dict[str,
|
|
75
|
-
|
|
76
|
-
|
|
87
|
+
# parameter file is {param: {"value": value}}
|
|
88
|
+
json_out: dict[str, dict[str, float | str]] = {}
|
|
89
|
+
for param_map in values.values():
|
|
90
|
+
for param, value in param_map.items():
|
|
91
|
+
json_out[param] = {"value": value}
|
|
77
92
|
|
|
78
93
|
# Disallow NaN from being written: ERT produces the parameters and the only
|
|
79
94
|
# way for the output to be NaN is if the input is invalid or if the sampling
|
|
80
95
|
# function is buggy. Either way, that would be a bug and we can report it by
|
|
81
96
|
# having json throw an error.
|
|
82
|
-
|
|
83
|
-
json_out,
|
|
84
|
-
)
|
|
97
|
+
with path.open("w") as f:
|
|
98
|
+
json.dump(json_out, f, allow_nan=False, indent=0, separators=(", ", " : "))
|
|
85
99
|
|
|
86
100
|
|
|
87
101
|
def _generate_parameter_files(
|
|
@@ -91,7 +105,7 @@ def _generate_parameter_files(
|
|
|
91
105
|
iens: int,
|
|
92
106
|
fs: Ensemble,
|
|
93
107
|
iteration: int,
|
|
94
|
-
) -> Mapping[str, Mapping[str, float | str]]:
|
|
108
|
+
) -> tuple[Mapping[str, Mapping[str, float | str]], Mapping[str, float]]:
|
|
95
109
|
"""
|
|
96
110
|
Generate parameter files that are placed in each runtime directory for
|
|
97
111
|
forward-model jobs to consume.
|
|
@@ -106,10 +120,25 @@ def _generate_parameter_files(
|
|
|
106
120
|
fs: Ensemble from which to load parameter data
|
|
107
121
|
|
|
108
122
|
Returns:
|
|
109
|
-
Returns the union of parameters returned by
|
|
110
|
-
parameter_config
|
|
123
|
+
Returns a tuple containing: the union of parameters returned by
|
|
124
|
+
write_to_runpath for each parameter_config, and a dict with
|
|
125
|
+
timings/durations for each parameter type.
|
|
111
126
|
"""
|
|
127
|
+
# preload scalar parameters for this realization
|
|
128
|
+
keys = [
|
|
129
|
+
p.name
|
|
130
|
+
for p in parameter_configs
|
|
131
|
+
if p.cardinality == ParameterCardinality.multiple_configs_per_ensemble_dataset
|
|
132
|
+
]
|
|
133
|
+
export_timings: defaultdict[str, float] = defaultdict(float)
|
|
134
|
+
scalar_data: dict[str, float | str] = {}
|
|
135
|
+
if keys:
|
|
136
|
+
start_time = time.perf_counter()
|
|
137
|
+
df = fs.load_scalar_keys(keys=keys, realizations=iens, transformed=True)
|
|
138
|
+
scalar_data = df.to_dicts()[0]
|
|
139
|
+
export_timings["load_scalar_keys"] = time.perf_counter() - start_time
|
|
112
140
|
exports: dict[str, dict[str, float | str]] = {}
|
|
141
|
+
log_exports: dict[str, dict[str, float | str]] = {}
|
|
113
142
|
|
|
114
143
|
for param in parameter_configs:
|
|
115
144
|
# For the first iteration we do not write the parameter
|
|
@@ -117,15 +146,42 @@ def _generate_parameter_files(
|
|
|
117
146
|
# model has completed.
|
|
118
147
|
if param.forward_init and iteration == 0:
|
|
119
148
|
continue
|
|
120
|
-
|
|
149
|
+
start_time = time.perf_counter()
|
|
150
|
+
export_values: dict[str, dict[str, float | str]] | None = None
|
|
151
|
+
log_export_values: dict[str, dict[str, float | str]] | None = {}
|
|
152
|
+
if param.name in scalar_data:
|
|
153
|
+
scalar_value = scalar_data[param.name]
|
|
154
|
+
export_values = {param.group_name: {param.name: scalar_value}}
|
|
155
|
+
if isinstance(param, GenKwConfig) and isinstance(
|
|
156
|
+
param.distribution, (LogNormalSettings, LogUnifSettings)
|
|
157
|
+
):
|
|
158
|
+
if isinstance(scalar_value, float) and scalar_value > 0:
|
|
159
|
+
log_value = math.log10(scalar_value)
|
|
160
|
+
log_export_values = {
|
|
161
|
+
f"LOG10_{param.group_name}": {param.name: log_value}
|
|
162
|
+
}
|
|
163
|
+
else:
|
|
164
|
+
logger.warning(
|
|
165
|
+
"Could not export the log10 value of "
|
|
166
|
+
f"{scalar_value} as it is invalid"
|
|
167
|
+
)
|
|
168
|
+
else:
|
|
169
|
+
export_values = param.write_to_runpath(Path(run_path), iens, fs)
|
|
121
170
|
if export_values:
|
|
122
171
|
for group, vals in export_values.items():
|
|
123
172
|
exports.setdefault(group, {}).update(vals)
|
|
173
|
+
if log_export_values:
|
|
174
|
+
for group, vals in log_export_values.items():
|
|
175
|
+
log_exports.setdefault(group, {}).update(vals)
|
|
176
|
+
export_timings[param.type] += time.perf_counter() - start_time
|
|
124
177
|
continue
|
|
125
|
-
|
|
126
|
-
_value_export_txt(run_path, export_base_name, exports)
|
|
178
|
+
start_time = time.perf_counter()
|
|
179
|
+
_value_export_txt(run_path, export_base_name, exports | log_exports)
|
|
180
|
+
export_timings["value_export_txt"] = time.perf_counter() - start_time
|
|
181
|
+
start_time = time.perf_counter()
|
|
127
182
|
_value_export_json(run_path, export_base_name, exports)
|
|
128
|
-
|
|
183
|
+
export_timings["value_export_json"] = time.perf_counter() - start_time
|
|
184
|
+
return (exports, dict(export_timings))
|
|
129
185
|
|
|
130
186
|
|
|
131
187
|
def _manifest_to_json(ensemble: Ensemble, iens: int, iter_: int) -> dict[str, Any]:
|
|
@@ -134,7 +190,7 @@ def _manifest_to_json(ensemble: Ensemble, iens: int, iter_: int) -> dict[str, An
|
|
|
134
190
|
for param_config in ensemble.experiment.parameter_configuration.values():
|
|
135
191
|
assert isinstance(
|
|
136
192
|
param_config,
|
|
137
|
-
|
|
193
|
+
EverestControl | GenKwConfig | Field | SurfaceConfig,
|
|
138
194
|
)
|
|
139
195
|
if param_config.forward_init and ensemble.iteration == 0:
|
|
140
196
|
assert not isinstance(param_config, GenKwConfig)
|
|
@@ -146,13 +202,32 @@ def _manifest_to_json(ensemble: Ensemble, iens: int, iter_: int) -> dict[str, An
|
|
|
146
202
|
# Add expected response files to manifest
|
|
147
203
|
for response_config in ensemble.experiment.response_configuration.values():
|
|
148
204
|
for input_file in response_config.expected_input_files:
|
|
149
|
-
manifest[f"{response_config.
|
|
150
|
-
|
|
205
|
+
manifest[f"{response_config.type}_{input_file}"] = substitute_runpath_name(
|
|
206
|
+
input_file, iens, iter_
|
|
151
207
|
)
|
|
152
208
|
|
|
153
209
|
return manifest
|
|
154
210
|
|
|
155
211
|
|
|
212
|
+
def _make_param_substituter(
|
|
213
|
+
substituter: Substitutions,
|
|
214
|
+
param_data: Mapping[str, Mapping[str, str | float]],
|
|
215
|
+
) -> Substitutions:
|
|
216
|
+
param_substituter = deepcopy(substituter)
|
|
217
|
+
for values in param_data.values():
|
|
218
|
+
for param_name, value in values.items():
|
|
219
|
+
if isinstance(value, int):
|
|
220
|
+
formatted_value = str(value)
|
|
221
|
+
elif isinstance(value, float):
|
|
222
|
+
formatted_value = f"{value:g}"
|
|
223
|
+
else:
|
|
224
|
+
formatted_value = str(value)
|
|
225
|
+
|
|
226
|
+
param_substituter[f"<{param_name}>"] = formatted_value
|
|
227
|
+
|
|
228
|
+
return param_substituter
|
|
229
|
+
|
|
230
|
+
|
|
156
231
|
@log_duration(logger, logging.INFO)
|
|
157
232
|
def create_run_path(
|
|
158
233
|
run_args: list[RunArg],
|
|
@@ -169,13 +244,19 @@ def create_run_path(
|
|
|
169
244
|
if context_env is None:
|
|
170
245
|
context_env = {}
|
|
171
246
|
runpaths.set_ert_ensemble(ensemble.name)
|
|
172
|
-
|
|
247
|
+
timings = {
|
|
248
|
+
"generate_parameter_files": 0.0,
|
|
249
|
+
"substitute_parameters": 0.0,
|
|
250
|
+
"substitute_real_iter": 0.0,
|
|
251
|
+
"result_file_to_target": 0.0,
|
|
252
|
+
}
|
|
173
253
|
substituter = Substitutions(substitutions)
|
|
174
254
|
for run_arg in run_args:
|
|
175
255
|
run_path = Path(run_arg.runpath)
|
|
176
256
|
if run_arg.active:
|
|
177
257
|
run_path.mkdir(parents=True, exist_ok=True)
|
|
178
|
-
|
|
258
|
+
start_time = time.perf_counter()
|
|
259
|
+
(param_data, detailed_parameter_timings) = _generate_parameter_files(
|
|
179
260
|
ensemble.experiment.parameter_configuration.values(),
|
|
180
261
|
parameters_file,
|
|
181
262
|
run_path,
|
|
@@ -183,22 +264,29 @@ def create_run_path(
|
|
|
183
264
|
ensemble,
|
|
184
265
|
ensemble.iteration,
|
|
185
266
|
)
|
|
267
|
+
for parameter_type, duration in detailed_parameter_timings.items():
|
|
268
|
+
if parameter_type not in timings:
|
|
269
|
+
timings[parameter_type] = 0.0
|
|
270
|
+
timings[parameter_type] += duration
|
|
271
|
+
|
|
272
|
+
timings["generate_parameter_files"] += time.perf_counter() - start_time
|
|
273
|
+
real_iter_substituter = substituter.real_iter_substituter(
|
|
274
|
+
run_arg.iens, ensemble.iteration
|
|
275
|
+
)
|
|
276
|
+
param_substituter = _make_param_substituter(
|
|
277
|
+
real_iter_substituter, param_data
|
|
278
|
+
)
|
|
186
279
|
for (
|
|
187
280
|
source_file_content,
|
|
188
281
|
target_file,
|
|
189
282
|
) in ensemble.experiment.templates_configuration:
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
)
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
)
|
|
198
|
-
result = substituter.substitute_parameters(
|
|
199
|
-
result,
|
|
200
|
-
param_data,
|
|
201
|
-
)
|
|
283
|
+
start_time = time.perf_counter()
|
|
284
|
+
target_file = real_iter_substituter.substitute(target_file)
|
|
285
|
+
timings["substitute_real_iter"] += time.perf_counter() - start_time
|
|
286
|
+
start_time = time.perf_counter()
|
|
287
|
+
result = param_substituter.substitute(source_file_content)
|
|
288
|
+
timings["substitute_parameters"] += time.perf_counter() - start_time
|
|
289
|
+
start_time = time.perf_counter()
|
|
202
290
|
target = run_path / target_file
|
|
203
291
|
if not target.parent.exists():
|
|
204
292
|
os.makedirs(
|
|
@@ -206,10 +294,13 @@ def create_run_path(
|
|
|
206
294
|
exist_ok=True,
|
|
207
295
|
)
|
|
208
296
|
target.write_text(result)
|
|
297
|
+
timings["result_file_to_target"] += time.perf_counter() - start_time
|
|
209
298
|
|
|
210
299
|
path = run_path / "jobs.json"
|
|
300
|
+
start_time = time.perf_counter()
|
|
211
301
|
_backup_if_existing(path)
|
|
212
|
-
|
|
302
|
+
timings["backup_if_existing"] = time.perf_counter() - start_time
|
|
303
|
+
start_time = time.perf_counter()
|
|
213
304
|
forward_model_output = create_forward_model_json(
|
|
214
305
|
context=substitutions,
|
|
215
306
|
forward_model_steps=forward_model_steps,
|
|
@@ -226,12 +317,15 @@ def create_run_path(
|
|
|
226
317
|
option=orjson.OPT_NON_STR_KEYS | orjson.OPT_INDENT_2,
|
|
227
318
|
)
|
|
228
319
|
)
|
|
320
|
+
timings["jobs_to_json"] = time.perf_counter() - start_time
|
|
229
321
|
# Write MANIFEST file to runpath use to avoid NFS sync issues
|
|
322
|
+
start_time = time.perf_counter()
|
|
230
323
|
data = _manifest_to_json(ensemble, run_arg.iens, run_arg.itr)
|
|
231
324
|
Path(run_path / "manifest.json").write_bytes(
|
|
232
325
|
orjson.dumps(data, option=orjson.OPT_NON_STR_KEYS | orjson.OPT_INDENT_2)
|
|
233
326
|
)
|
|
234
|
-
|
|
327
|
+
timings["manifest_to_json"] = time.perf_counter() - start_time
|
|
328
|
+
logger.info(f"_create_run_path durations: {timings}")
|
|
235
329
|
runpaths.write_runpath_list(
|
|
236
330
|
[ensemble.iteration], [real.iens for real in run_args if real.active]
|
|
237
331
|
)
|
|
@@ -13,14 +13,22 @@ from ert.config import (
|
|
|
13
13
|
ResponseConfig,
|
|
14
14
|
)
|
|
15
15
|
from ert.ensemble_evaluator import EvaluatorServerConfig
|
|
16
|
-
from ert.run_models.initial_ensemble_run_model import
|
|
16
|
+
from ert.run_models.initial_ensemble_run_model import (
|
|
17
|
+
InitialEnsembleRunModel,
|
|
18
|
+
InitialEnsembleRunModelConfig,
|
|
19
|
+
)
|
|
17
20
|
from ert.storage import Ensemble
|
|
18
21
|
from ert.trace import tracer
|
|
19
22
|
|
|
20
23
|
logger = logging.getLogger(__name__)
|
|
21
24
|
|
|
22
25
|
|
|
23
|
-
class
|
|
26
|
+
class EnsembleExperimentConfig(InitialEnsembleRunModelConfig):
|
|
27
|
+
target_ensemble: str
|
|
28
|
+
supports_rerunning_failed_realizations: ClassVar[bool] = True
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class EnsembleExperiment(InitialEnsembleRunModel, EnsembleExperimentConfig):
|
|
24
32
|
"""
|
|
25
33
|
This workflow will create a new experiment and a new ensemble from
|
|
26
34
|
the user configuration.<br>It will never overwrite existing ensembles, and
|
|
@@ -28,8 +36,6 @@ class EnsembleExperiment(InitialEnsembleRunModel):
|
|
|
28
36
|
"""
|
|
29
37
|
|
|
30
38
|
_ensemble_id: UUID | None = PrivateAttr(None)
|
|
31
|
-
supports_rerunning_failed_realizations: ClassVar[bool] = True
|
|
32
|
-
target_ensemble: str
|
|
33
39
|
|
|
34
40
|
@property
|
|
35
41
|
def _ensemble(self) -> Ensemble:
|
|
@@ -7,11 +7,18 @@ from ert.run_models.ensemble_smoother import EnsembleSmoother
|
|
|
7
7
|
from ert.storage import Ensemble
|
|
8
8
|
|
|
9
9
|
from ..analysis import enif_update
|
|
10
|
+
from .initial_ensemble_run_model import InitialEnsembleRunModelConfig
|
|
11
|
+
from .update_run_model import UpdateRunModelConfig
|
|
10
12
|
|
|
11
13
|
logger = logging.getLogger(__name__)
|
|
12
14
|
|
|
13
15
|
|
|
14
|
-
class
|
|
16
|
+
class EnsembleInformationFilterConfig(
|
|
17
|
+
InitialEnsembleRunModelConfig, UpdateRunModelConfig
|
|
18
|
+
): ...
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class EnsembleInformationFilter(EnsembleSmoother, EnsembleInformationFilterConfig):
|
|
15
22
|
def update_ensemble_parameters(
|
|
16
23
|
self, prior: Ensemble, posterior: Ensemble, weight: float
|
|
17
24
|
) -> None:
|
|
@@ -14,8 +14,11 @@ from ert.config import (
|
|
|
14
14
|
ResponseConfig,
|
|
15
15
|
)
|
|
16
16
|
from ert.ensemble_evaluator import EvaluatorServerConfig
|
|
17
|
-
from ert.run_models.initial_ensemble_run_model import
|
|
18
|
-
|
|
17
|
+
from ert.run_models.initial_ensemble_run_model import (
|
|
18
|
+
InitialEnsembleRunModel,
|
|
19
|
+
InitialEnsembleRunModelConfig,
|
|
20
|
+
)
|
|
21
|
+
from ert.run_models.update_run_model import UpdateRunModel, UpdateRunModelConfig
|
|
19
22
|
from ert.storage import Ensemble
|
|
20
23
|
from ert.trace import tracer
|
|
21
24
|
|
|
@@ -26,7 +29,10 @@ from .run_model import ErtRunError
|
|
|
26
29
|
logger = logging.getLogger(__name__)
|
|
27
30
|
|
|
28
31
|
|
|
29
|
-
class
|
|
32
|
+
class EnsembleSmootherConfig(InitialEnsembleRunModelConfig, UpdateRunModelConfig): ...
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class EnsembleSmoother(InitialEnsembleRunModel, UpdateRunModel, EnsembleSmootherConfig):
|
|
30
36
|
_total_iterations: int = PrivateAttr(default=2)
|
|
31
37
|
|
|
32
38
|
@tracer.start_as_current_span(f"{__name__}.run_experiment")
|