ert 16.0.9__py3-none-any.whl → 19.0.0rc2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- _ert/events.py +19 -2
- _ert/forward_model_runner/client.py +6 -2
- _ert/forward_model_runner/fm_dispatch.py +9 -6
- _ert/forward_model_runner/reporting/event.py +1 -0
- _ert/forward_model_runner/runner.py +1 -2
- _ert/utils.py +12 -0
- ert/__main__.py +58 -38
- ert/analysis/_enif_update.py +8 -4
- ert/analysis/_es_update.py +19 -6
- ert/analysis/_update_commons.py +16 -6
- ert/base_model_context.py +1 -1
- ert/cli/main.py +17 -12
- ert/cli/monitor.py +7 -0
- ert/config/__init__.py +17 -6
- ert/config/_create_observation_dataframes.py +118 -21
- ert/config/_get_num_cpu.py +1 -1
- ert/config/_observations.py +91 -2
- ert/config/_read_summary.py +74 -328
- ert/config/design_matrix.py +62 -23
- ert/config/distribution.py +1 -1
- ert/config/ensemble_config.py +9 -17
- ert/config/ert_config.py +155 -58
- ert/config/everest_control.py +234 -0
- ert/config/{everest_constraints_config.py → everest_response.py} +27 -15
- ert/config/field.py +99 -90
- ert/config/forward_model_step.py +122 -17
- ert/config/gen_data_config.py +5 -10
- ert/config/gen_kw_config.py +11 -41
- ert/config/known_response_types.py +14 -0
- ert/config/parameter_config.py +1 -33
- ert/config/parsing/_option_dict.py +10 -2
- ert/config/parsing/config_errors.py +1 -1
- ert/config/parsing/config_keywords.py +2 -1
- ert/config/parsing/config_schema.py +23 -11
- ert/config/parsing/config_schema_deprecations.py +3 -3
- ert/config/parsing/config_schema_item.py +26 -11
- ert/config/parsing/context_values.py +3 -3
- ert/config/parsing/file_context_token.py +1 -1
- ert/config/parsing/observations_parser.py +6 -2
- ert/config/parsing/queue_system.py +9 -0
- ert/config/parsing/schema_item_type.py +1 -0
- ert/config/queue_config.py +42 -50
- ert/config/response_config.py +0 -8
- ert/config/rft_config.py +275 -0
- ert/config/summary_config.py +3 -8
- ert/config/surface_config.py +73 -26
- ert/config/workflow_fixtures.py +2 -1
- ert/config/workflow_job.py +135 -54
- ert/dark_storage/client/__init__.py +2 -2
- ert/dark_storage/client/_session.py +4 -4
- ert/dark_storage/client/client.py +2 -2
- ert/dark_storage/common.py +12 -3
- ert/dark_storage/compute/misfits.py +11 -7
- ert/dark_storage/endpoints/compute/misfits.py +6 -4
- ert/dark_storage/endpoints/ensembles.py +4 -0
- ert/dark_storage/endpoints/experiment_server.py +30 -24
- ert/dark_storage/endpoints/experiments.py +2 -2
- ert/dark_storage/endpoints/observations.py +8 -6
- ert/dark_storage/endpoints/parameters.py +4 -12
- ert/dark_storage/endpoints/responses.py +24 -5
- ert/dark_storage/json_schema/ensemble.py +3 -0
- ert/dark_storage/json_schema/experiment.py +1 -1
- ert/data/_measured_data.py +6 -5
- ert/ensemble_evaluator/__init__.py +8 -1
- ert/ensemble_evaluator/config.py +2 -1
- ert/ensemble_evaluator/evaluator.py +81 -29
- ert/ensemble_evaluator/event.py +6 -0
- ert/ensemble_evaluator/snapshot.py +3 -1
- ert/ensemble_evaluator/state.py +1 -0
- ert/field_utils/__init__.py +8 -0
- ert/field_utils/field_utils.py +228 -15
- ert/field_utils/grdecl_io.py +1 -1
- ert/field_utils/roff_io.py +1 -1
- ert/gui/__init__.py +5 -2
- ert/gui/ertnotifier.py +1 -1
- ert/gui/ertwidgets/__init__.py +23 -16
- ert/gui/ertwidgets/analysismoduleedit.py +2 -2
- ert/gui/ertwidgets/checklist.py +1 -1
- ert/gui/ertwidgets/closabledialog.py +2 -0
- ert/gui/ertwidgets/copyablelabel.py +2 -0
- ert/gui/ertwidgets/create_experiment_dialog.py +3 -1
- ert/gui/ertwidgets/ensembleselector.py +2 -2
- ert/gui/ertwidgets/listeditbox.py +2 -0
- ert/gui/ertwidgets/models/__init__.py +2 -0
- ert/gui/ertwidgets/models/activerealizationsmodel.py +5 -1
- ert/gui/ertwidgets/models/path_model.py +1 -1
- ert/gui/ertwidgets/models/targetensemblemodel.py +5 -1
- ert/gui/ertwidgets/models/text_model.py +4 -1
- ert/gui/ertwidgets/pathchooser.py +0 -3
- ert/gui/ertwidgets/searchbox.py +17 -4
- ert/gui/ertwidgets/stringbox.py +2 -0
- ert/gui/{suggestor → ertwidgets/suggestor}/_suggestor_message.py +13 -4
- ert/gui/{suggestor → ertwidgets/suggestor}/suggestor.py +63 -30
- ert/gui/main.py +41 -13
- ert/gui/main_window.py +3 -7
- ert/gui/model/fm_step_list.py +3 -0
- ert/gui/model/real_list.py +1 -0
- ert/gui/model/snapshot.py +1 -0
- ert/gui/simulation/combobox_with_description.py +3 -0
- ert/gui/simulation/ensemble_experiment_panel.py +8 -2
- ert/gui/simulation/ensemble_information_filter_panel.py +7 -2
- ert/gui/simulation/ensemble_smoother_panel.py +8 -2
- ert/gui/simulation/evaluate_ensemble_panel.py +17 -7
- ert/gui/simulation/experiment_panel.py +18 -6
- ert/gui/simulation/manual_update_panel.py +35 -10
- ert/gui/simulation/multiple_data_assimilation_panel.py +13 -9
- ert/gui/simulation/run_dialog.py +47 -20
- ert/gui/simulation/single_test_run_panel.py +6 -3
- ert/gui/simulation/view/progress_widget.py +2 -0
- ert/gui/simulation/view/realization.py +5 -1
- ert/gui/simulation/view/update.py +2 -0
- ert/gui/summarypanel.py +20 -1
- ert/gui/tools/event_viewer/panel.py +3 -4
- ert/gui/tools/event_viewer/tool.py +2 -0
- ert/gui/tools/load_results/load_results_panel.py +1 -1
- ert/gui/tools/load_results/load_results_tool.py +2 -0
- ert/gui/tools/manage_experiments/export_dialog.py +136 -0
- ert/gui/tools/manage_experiments/manage_experiments_panel.py +2 -0
- ert/gui/tools/manage_experiments/storage_info_widget.py +121 -16
- ert/gui/tools/manage_experiments/storage_widget.py +4 -3
- ert/gui/tools/plot/customize/color_chooser.py +5 -2
- ert/gui/tools/plot/customize/customize_plot_dialog.py +2 -0
- ert/gui/tools/plot/customize/default_customization_view.py +4 -0
- ert/gui/tools/plot/customize/limits_customization_view.py +3 -0
- ert/gui/tools/plot/customize/statistics_customization_view.py +3 -0
- ert/gui/tools/plot/customize/style_chooser.py +2 -0
- ert/gui/tools/plot/customize/style_customization_view.py +3 -0
- ert/gui/tools/plot/data_type_keys_widget.py +2 -0
- ert/gui/tools/plot/data_type_proxy_model.py +3 -0
- ert/gui/tools/plot/plot_api.py +50 -28
- ert/gui/tools/plot/plot_ensemble_selection_widget.py +17 -10
- ert/gui/tools/plot/plot_widget.py +15 -2
- ert/gui/tools/plot/plot_window.py +41 -19
- ert/gui/tools/plot/plottery/plot_config.py +2 -0
- ert/gui/tools/plot/plottery/plot_context.py +14 -0
- ert/gui/tools/plot/plottery/plots/__init__.py +2 -0
- ert/gui/tools/plot/plottery/plots/cesp.py +3 -1
- ert/gui/tools/plot/plottery/plots/distribution.py +6 -1
- ert/gui/tools/plot/plottery/plots/ensemble.py +13 -5
- ert/gui/tools/plot/plottery/plots/gaussian_kde.py +12 -2
- ert/gui/tools/plot/plottery/plots/histogram.py +3 -1
- ert/gui/tools/plot/plottery/plots/misfits.py +436 -0
- ert/gui/tools/plot/plottery/plots/observations.py +18 -4
- ert/gui/tools/plot/plottery/plots/statistics.py +62 -20
- ert/gui/tools/plot/plottery/plots/std_dev.py +3 -1
- ert/gui/tools/plot/widgets/clearable_line_edit.py +9 -0
- ert/gui/tools/plot/widgets/filter_popup.py +2 -0
- ert/gui/tools/plot/widgets/filterable_kw_list_model.py +3 -0
- ert/gui/tools/plugins/plugin.py +1 -1
- ert/gui/tools/plugins/plugins_tool.py +2 -0
- ert/gui/tools/plugins/process_job_dialog.py +3 -0
- ert/gui/tools/workflows/workflow_dialog.py +2 -0
- ert/gui/tools/workflows/workflows_tool.py +2 -0
- ert/libres_facade.py +5 -7
- ert/logging/__init__.py +4 -1
- ert/mode_definitions.py +2 -0
- ert/plugins/__init__.py +4 -6
- ert/plugins/hook_implementations/workflows/csv_export.py +2 -3
- ert/plugins/hook_implementations/workflows/gen_data_rft_export.py +10 -2
- ert/plugins/hook_specifications/__init__.py +0 -10
- ert/plugins/hook_specifications/jobs.py +0 -9
- ert/plugins/plugin_manager.py +53 -124
- ert/resources/forward_models/run_reservoirsimulator.py +8 -4
- ert/resources/forward_models/template_render.py +10 -10
- ert/resources/shell_scripts/delete_directory.py +2 -2
- ert/run_models/__init__.py +24 -6
- ert/run_models/_create_run_path.py +133 -38
- ert/run_models/ensemble_experiment.py +10 -4
- ert/run_models/ensemble_information_filter.py +8 -1
- ert/run_models/ensemble_smoother.py +9 -3
- ert/run_models/evaluate_ensemble.py +8 -6
- ert/run_models/event.py +7 -3
- ert/run_models/everest_run_model.py +337 -113
- ert/run_models/initial_ensemble_run_model.py +25 -24
- ert/run_models/manual_update.py +6 -3
- ert/run_models/manual_update_enif.py +37 -0
- ert/run_models/model_factory.py +78 -18
- ert/run_models/multiple_data_assimilation.py +22 -11
- ert/run_models/run_model.py +72 -73
- ert/run_models/single_test_run.py +7 -4
- ert/run_models/update_run_model.py +4 -2
- ert/runpaths.py +5 -6
- ert/sample_prior.py +9 -4
- ert/scheduler/__init__.py +10 -5
- ert/scheduler/driver.py +40 -0
- ert/scheduler/event.py +3 -1
- ert/scheduler/job.py +23 -13
- ert/scheduler/lsf_driver.py +15 -5
- ert/scheduler/openpbs_driver.py +10 -4
- ert/scheduler/scheduler.py +5 -0
- ert/scheduler/slurm_driver.py +20 -5
- ert/services/__init__.py +2 -2
- ert/services/_base_service.py +37 -20
- ert/services/_storage_main.py +20 -18
- ert/services/ert_server.py +317 -0
- ert/shared/_doc_utils/__init__.py +4 -2
- ert/shared/_doc_utils/ert_jobs.py +1 -4
- ert/shared/net_utils.py +43 -18
- ert/shared/storage/connection.py +3 -3
- ert/shared/version.py +3 -3
- ert/storage/__init__.py +14 -1
- ert/storage/local_ensemble.py +44 -13
- ert/storage/local_experiment.py +54 -34
- ert/storage/local_storage.py +90 -58
- ert/storage/migration/to10.py +3 -2
- ert/storage/migration/to11.py +9 -10
- ert/storage/migration/to12.py +19 -20
- ert/storage/migration/to13.py +28 -27
- ert/storage/migration/to14.py +3 -3
- ert/storage/migration/to15.py +25 -0
- ert/storage/migration/to16.py +38 -0
- ert/storage/migration/to17.py +42 -0
- ert/storage/migration/to18.py +11 -0
- ert/storage/migration/to19.py +34 -0
- ert/storage/migration/to20.py +23 -0
- ert/storage/migration/to21.py +25 -0
- ert/storage/migration/to6.py +3 -2
- ert/storage/migration/to7.py +12 -13
- ert/storage/migration/to8.py +9 -11
- ert/storage/migration/to9.py +5 -4
- ert/storage/realization_storage_state.py +7 -7
- ert/substitutions.py +12 -28
- ert/validation/active_range.py +7 -7
- ert/validation/ensemble_realizations_argument.py +4 -2
- ert/validation/rangestring.py +16 -16
- ert/workflow_runner.py +6 -3
- {ert-16.0.9.dist-info → ert-19.0.0rc2.dist-info}/METADATA +21 -15
- ert-19.0.0rc2.dist-info/RECORD +524 -0
- {ert-16.0.9.dist-info → ert-19.0.0rc2.dist-info}/WHEEL +1 -1
- everest/api/everest_data_api.py +14 -1
- everest/assets/everest_logo.svg +406 -0
- everest/bin/config_branch_script.py +30 -14
- everest/bin/everconfigdump_script.py +2 -10
- everest/bin/everest_script.py +53 -33
- everest/bin/everlint_script.py +3 -5
- everest/bin/kill_script.py +7 -5
- everest/bin/main.py +11 -24
- everest/bin/monitor_script.py +64 -35
- everest/bin/utils.py +58 -43
- everest/bin/visualization_script.py +23 -13
- everest/config/__init__.py +4 -1
- everest/config/control_config.py +81 -6
- everest/config/control_variable_config.py +4 -3
- everest/config/everest_config.py +102 -79
- everest/config/forward_model_config.py +5 -3
- everest/config/install_data_config.py +7 -5
- everest/config/install_job_config.py +45 -3
- everest/config/install_template_config.py +3 -3
- everest/config/optimization_config.py +19 -6
- everest/config/output_constraint_config.py +8 -2
- everest/config/server_config.py +6 -55
- everest/config/simulator_config.py +62 -17
- everest/config/utils.py +25 -105
- everest/config/validation_utils.py +34 -15
- everest/config_file_loader.py +30 -21
- everest/detached/__init__.py +0 -6
- everest/detached/client.py +7 -52
- everest/detached/everserver.py +19 -45
- everest/everest_storage.py +24 -40
- everest/gui/everest_client.py +2 -3
- everest/gui/main_window.py +2 -2
- everest/optimizer/everest2ropt.py +68 -42
- everest/optimizer/opt_model_transforms.py +15 -20
- everest/optimizer/utils.py +0 -29
- everest/plugins/hook_specs.py +0 -24
- everest/strings.py +1 -6
- everest/util/__init__.py +3 -1
- ert/config/everest_objective_config.py +0 -95
- ert/config/ext_param_config.py +0 -107
- ert/gui/tools/export/__init__.py +0 -3
- ert/gui/tools/export/export_panel.py +0 -83
- ert/gui/tools/export/export_tool.py +0 -67
- ert/gui/tools/export/exporter.py +0 -36
- ert/plugins/hook_specifications/ecl_config.py +0 -29
- ert/services/storage_service.py +0 -127
- ert/summary_key_type.py +0 -234
- ert-16.0.9.dist-info/RECORD +0 -521
- everest/bin/everexport_script.py +0 -53
- everest/config/sampler_config.py +0 -103
- everest/simulator/__init__.py +0 -88
- everest/simulator/everest_to_ert.py +0 -252
- /ert/gui/{suggestor → ertwidgets/suggestor}/__init__.py +0 -0
- /ert/gui/{suggestor → ertwidgets/suggestor}/_colors.py +0 -0
- {ert-16.0.9.dist-info → ert-19.0.0rc2.dist-info}/entry_points.txt +0 -0
- {ert-16.0.9.dist-info → ert-19.0.0rc2.dist-info}/licenses/COPYING +0 -0
- {ert-16.0.9.dist-info → ert-19.0.0rc2.dist-info}/top_level.txt +0 -0
ert/config/forward_model_step.py
CHANGED
|
@@ -1,13 +1,33 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
import logging
|
|
4
|
-
from typing import
|
|
5
|
-
|
|
6
|
-
|
|
4
|
+
from typing import (
|
|
5
|
+
TYPE_CHECKING,
|
|
6
|
+
Annotated,
|
|
7
|
+
Any,
|
|
8
|
+
ClassVar,
|
|
9
|
+
Literal,
|
|
10
|
+
NotRequired,
|
|
11
|
+
Self,
|
|
12
|
+
cast,
|
|
13
|
+
)
|
|
14
|
+
|
|
15
|
+
from pydantic import (
|
|
16
|
+
BaseModel,
|
|
17
|
+
Field,
|
|
18
|
+
field_validator,
|
|
19
|
+
model_serializer,
|
|
20
|
+
model_validator,
|
|
21
|
+
)
|
|
22
|
+
from pydantic_core.core_schema import ValidationInfo
|
|
7
23
|
from typing_extensions import TypedDict, Unpack
|
|
8
24
|
|
|
25
|
+
from ..base_model_context import BaseModelWithContextSupport
|
|
9
26
|
from .parsing import ConfigValidationError, ConfigWarning, SchemaItemType
|
|
10
27
|
|
|
28
|
+
if TYPE_CHECKING:
|
|
29
|
+
from ert.plugins import ErtRuntimePlugins
|
|
30
|
+
|
|
11
31
|
logger = logging.getLogger(__name__)
|
|
12
32
|
|
|
13
33
|
|
|
@@ -72,6 +92,7 @@ class ForwardModelStepOptions(TypedDict, total=False):
|
|
|
72
92
|
max_running_minutes: NotRequired[int]
|
|
73
93
|
environment: NotRequired[dict[str, str]]
|
|
74
94
|
default_mapping: NotRequired[dict[str, str]]
|
|
95
|
+
required_keywords: NotRequired[list[str]]
|
|
75
96
|
|
|
76
97
|
|
|
77
98
|
class ForwardModelStepDocumentation(BaseModel):
|
|
@@ -80,18 +101,21 @@ class ForwardModelStepDocumentation(BaseModel):
|
|
|
80
101
|
source_function_name: str = Field(default="ert")
|
|
81
102
|
description: str = Field(default="No description")
|
|
82
103
|
examples: str = Field(default="No examples")
|
|
83
|
-
category:
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
"
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
104
|
+
category: Annotated[
|
|
105
|
+
str,
|
|
106
|
+
Field(
|
|
107
|
+
default="Uncategorized",
|
|
108
|
+
examples=[
|
|
109
|
+
"utility.file_system",
|
|
110
|
+
"simulators.reservoir",
|
|
111
|
+
"modelling.reservoir",
|
|
112
|
+
"utility.templating",
|
|
113
|
+
],
|
|
114
|
+
),
|
|
115
|
+
]
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
class ForwardModelStep(BaseModelWithContextSupport):
|
|
95
119
|
"""
|
|
96
120
|
Holds information to execute one step of a forward model
|
|
97
121
|
|
|
@@ -202,7 +226,87 @@ class ForwardModelStep(BaseModel):
|
|
|
202
226
|
return None if v == "null" else v
|
|
203
227
|
|
|
204
228
|
|
|
205
|
-
class
|
|
229
|
+
class UserInstalledForwardModelStep(ForwardModelStep):
|
|
230
|
+
"""
|
|
231
|
+
Represents a forward model step installed by a user via the ERT Config
|
|
232
|
+
forward model step format provided via the INSTALL_JOB keyword.
|
|
233
|
+
User-installed forward model steps serialize with their full configuration,
|
|
234
|
+
unlike site-installed steps which only serialize as references.
|
|
235
|
+
"""
|
|
236
|
+
|
|
237
|
+
type: Literal["user_installed"] = "user_installed"
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
class _SerializedSiteInstalledForwardModelStep(TypedDict):
|
|
241
|
+
type: Literal["site_installed"]
|
|
242
|
+
name: str
|
|
243
|
+
private_args: dict[str, str]
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
class SiteInstalledForwardModelStep(ForwardModelStep):
|
|
247
|
+
"""
|
|
248
|
+
Represents a forward model step installed via external plugins.
|
|
249
|
+
Instances of this class serialize only as references to the plugin by name, and
|
|
250
|
+
the user-provided private_args, allowing them to dynamically update when plugins
|
|
251
|
+
change, rather than being locked to a specific executable at serialization time.
|
|
252
|
+
"""
|
|
253
|
+
|
|
254
|
+
type: Literal["site_installed"] = "site_installed"
|
|
255
|
+
|
|
256
|
+
@model_serializer(mode="plain")
|
|
257
|
+
def serialize_model(self) -> _SerializedSiteInstalledForwardModelStep:
|
|
258
|
+
return {
|
|
259
|
+
"type": "site_installed",
|
|
260
|
+
"name": self.name,
|
|
261
|
+
"private_args": self.private_args,
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
@model_validator(mode="before")
|
|
265
|
+
@classmethod
|
|
266
|
+
def deserialize_model(
|
|
267
|
+
cls, values: dict[str, Any], info: ValidationInfo
|
|
268
|
+
) -> dict[str, Any]:
|
|
269
|
+
runtime_plugins = cast("ErtRuntimePlugins", info.context)
|
|
270
|
+
name = values["name"]
|
|
271
|
+
|
|
272
|
+
if runtime_plugins is None:
|
|
273
|
+
if values.get("type") == "site_installed":
|
|
274
|
+
msg = (
|
|
275
|
+
f"Trying to find site-installed forward model step {name} "
|
|
276
|
+
f"without site plugins. This forward model must be loaded "
|
|
277
|
+
f"with ERT site plugins available."
|
|
278
|
+
)
|
|
279
|
+
raise KeyError(msg)
|
|
280
|
+
return values
|
|
281
|
+
|
|
282
|
+
if name not in runtime_plugins.installed_forward_model_steps:
|
|
283
|
+
msg = (
|
|
284
|
+
f"Expected forward model step {name} to be installed "
|
|
285
|
+
f"via plugins, but it was not found. Please check that "
|
|
286
|
+
f"your python environment has it installed."
|
|
287
|
+
)
|
|
288
|
+
raise KeyError(msg)
|
|
289
|
+
site_installed_fm = runtime_plugins.installed_forward_model_steps[name]
|
|
290
|
+
|
|
291
|
+
# Intent: copy the site installed forward model to this instance.
|
|
292
|
+
# bypassing the model_serializer
|
|
293
|
+
site_fm_instance = {
|
|
294
|
+
k: getattr(site_installed_fm, k)
|
|
295
|
+
for k in SiteInstalledForwardModelStep.model_fields
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
return site_fm_instance | (
|
|
299
|
+
{"private_args": values["private_args"]} if "private_args" in values else {}
|
|
300
|
+
)
|
|
301
|
+
|
|
302
|
+
|
|
303
|
+
SiteOrUserForwardModelStep = Annotated[
|
|
304
|
+
(UserInstalledForwardModelStep | SiteInstalledForwardModelStep),
|
|
305
|
+
Field(discriminator="type"),
|
|
306
|
+
]
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
class ForwardModelStepPlugin(SiteInstalledForwardModelStep):
|
|
206
310
|
def __init__(
|
|
207
311
|
self, name: str, command: list[str], **kwargs: Unpack[ForwardModelStepOptions]
|
|
208
312
|
) -> None:
|
|
@@ -221,6 +325,7 @@ class ForwardModelStepPlugin(ForwardModelStep):
|
|
|
221
325
|
max_running_minutes = kwargs.get("max_running_minutes")
|
|
222
326
|
environment = kwargs.get("environment", {}) or {}
|
|
223
327
|
default_mapping = kwargs.get("default_mapping", {}) or {}
|
|
328
|
+
required_keywords = kwargs.get("required_keywords", []) or []
|
|
224
329
|
|
|
225
330
|
super().__init__(
|
|
226
331
|
name=name,
|
|
@@ -235,7 +340,7 @@ class ForwardModelStepPlugin(ForwardModelStep):
|
|
|
235
340
|
max_running_minutes=max_running_minutes,
|
|
236
341
|
min_arg=0,
|
|
237
342
|
max_arg=0,
|
|
238
|
-
required_keywords=
|
|
343
|
+
required_keywords=required_keywords,
|
|
239
344
|
arg_types=[],
|
|
240
345
|
environment=environment,
|
|
241
346
|
default_mapping=default_mapping,
|
ert/config/gen_data_config.py
CHANGED
|
@@ -21,7 +21,6 @@ from .responses_index import responses_index
|
|
|
21
21
|
|
|
22
22
|
class GenDataConfig(ResponseConfig):
|
|
23
23
|
type: Literal["gen_data"] = "gen_data"
|
|
24
|
-
name: str = "gen_data"
|
|
25
24
|
report_steps_list: list[list[int] | None] = Field(default_factory=list)
|
|
26
25
|
has_finalized_keys: bool = True
|
|
27
26
|
|
|
@@ -29,7 +28,7 @@ class GenDataConfig(ResponseConfig):
|
|
|
29
28
|
def metadata(self) -> list[ResponseMetadata]:
|
|
30
29
|
return [
|
|
31
30
|
ResponseMetadata(
|
|
32
|
-
response_type=self.
|
|
31
|
+
response_type=self.type,
|
|
33
32
|
response_key=response_key,
|
|
34
33
|
finalized=self.has_finalized_keys,
|
|
35
34
|
filter_on={"report_step": report_steps}
|
|
@@ -198,28 +197,24 @@ class GenDataConfig(ResponseConfig):
|
|
|
198
197
|
if all(isinstance(err, FileNotFoundError) for err in errors):
|
|
199
198
|
raise FileNotFoundError(
|
|
200
199
|
"Could not find one or more files/directories while reading "
|
|
201
|
-
f"GEN_DATA
|
|
200
|
+
f"GEN_DATA: {','.join([str(err) for err in errors])}"
|
|
202
201
|
)
|
|
203
202
|
else:
|
|
204
203
|
raise InvalidResponseFile(
|
|
205
204
|
"Error reading GEN_DATA "
|
|
206
|
-
f"{self.
|
|
205
|
+
f"{self.type}, errors: {','.join([str(err) for err in errors])}"
|
|
207
206
|
)
|
|
208
207
|
|
|
209
208
|
combined = pl.concat(datasets_per_name)
|
|
210
209
|
return combined
|
|
211
210
|
|
|
212
211
|
def get_args_for_key(self, key: str) -> tuple[str | None, list[int] | None]:
|
|
213
|
-
for i,
|
|
214
|
-
if key ==
|
|
212
|
+
for i, key_ in enumerate(self.keys):
|
|
213
|
+
if key == key_:
|
|
215
214
|
return self.input_files[i], self.report_steps_list[i]
|
|
216
215
|
|
|
217
216
|
return None, None
|
|
218
217
|
|
|
219
|
-
@property
|
|
220
|
-
def response_type(self) -> str:
|
|
221
|
-
return "gen_data"
|
|
222
|
-
|
|
223
218
|
@property
|
|
224
219
|
def primary_key(self) -> list[str]:
|
|
225
220
|
return ["report_step", "index"]
|
ert/config/gen_kw_config.py
CHANGED
|
@@ -15,7 +15,7 @@ from typing_extensions import TypedDict
|
|
|
15
15
|
|
|
16
16
|
from ._str_to_bool import str_to_bool
|
|
17
17
|
from .distribution import DISTRIBUTION_CLASSES, DistributionSettings, get_distribution
|
|
18
|
-
from .parameter_config import ParameterCardinality, ParameterConfig
|
|
18
|
+
from .parameter_config import ParameterCardinality, ParameterConfig
|
|
19
19
|
from .parsing import ConfigValidationError, ConfigWarning
|
|
20
20
|
|
|
21
21
|
if TYPE_CHECKING:
|
|
@@ -53,6 +53,7 @@ class DataSource(StrEnum):
|
|
|
53
53
|
|
|
54
54
|
class GenKwConfig(ParameterConfig):
|
|
55
55
|
type: Literal["gen_kw"] = "gen_kw"
|
|
56
|
+
dimensionality: Literal[1] = 1
|
|
56
57
|
distribution: DistributionSettings
|
|
57
58
|
forward_init: bool = False
|
|
58
59
|
update: bool = True
|
|
@@ -73,17 +74,6 @@ class GenKwConfig(ParameterConfig):
|
|
|
73
74
|
def cardinality(self) -> ParameterCardinality:
|
|
74
75
|
return ParameterCardinality.multiple_configs_per_ensemble_dataset
|
|
75
76
|
|
|
76
|
-
@property
|
|
77
|
-
def metadata(self) -> list[ParameterMetadata]:
|
|
78
|
-
return [
|
|
79
|
-
ParameterMetadata(
|
|
80
|
-
key=f"{self.group}:{self.name}",
|
|
81
|
-
transformation=self.distribution.name.upper(),
|
|
82
|
-
dimensionality=1,
|
|
83
|
-
userdata={"data_origin": "GEN_KW"},
|
|
84
|
-
)
|
|
85
|
-
]
|
|
86
|
-
|
|
87
77
|
@classmethod
|
|
88
78
|
def templates_from_config(
|
|
89
79
|
cls, gen_kw: list[str | dict[str, str]]
|
|
@@ -117,11 +107,11 @@ class GenKwConfig(ParameterConfig):
|
|
|
117
107
|
return None
|
|
118
108
|
|
|
119
109
|
@classmethod
|
|
120
|
-
def from_config_list(cls,
|
|
121
|
-
gen_kw_key = cast(str,
|
|
110
|
+
def from_config_list(cls, config_list: list[str | dict[str, str]]) -> list[Self]:
|
|
111
|
+
gen_kw_key = cast(str, config_list[0])
|
|
122
112
|
|
|
123
|
-
options = cast(dict[str, str],
|
|
124
|
-
positional_args = cast(list[str],
|
|
113
|
+
options = cast(dict[str, str], config_list[-1])
|
|
114
|
+
positional_args = cast(list[str | list[str]], config_list[:-1])
|
|
125
115
|
errors = []
|
|
126
116
|
update_parameter = str_to_bool(options.get("UPDATE", "TRUE"))
|
|
127
117
|
if _get_abs_path(options.get("INIT_FILES")):
|
|
@@ -129,7 +119,7 @@ class GenKwConfig(ParameterConfig):
|
|
|
129
119
|
"INIT_FILES with GEN_KW has been removed. "
|
|
130
120
|
f"Please remove INIT_FILES from the GEN_KW {gen_kw_key} config. "
|
|
131
121
|
"Alternatively, use DESIGN_MATRIX to load parameters from files.",
|
|
132
|
-
|
|
122
|
+
config_list,
|
|
133
123
|
)
|
|
134
124
|
|
|
135
125
|
if len(positional_args) == 2:
|
|
@@ -153,7 +143,7 @@ class GenKwConfig(ParameterConfig):
|
|
|
153
143
|
ConfigValidationError.with_context(
|
|
154
144
|
f"Too few values on line {line_number} in parameter "
|
|
155
145
|
f"file {parameter_file_context}",
|
|
156
|
-
|
|
146
|
+
config_list,
|
|
157
147
|
)
|
|
158
148
|
)
|
|
159
149
|
else:
|
|
@@ -186,7 +176,7 @@ class GenKwConfig(ParameterConfig):
|
|
|
186
176
|
params[0], params[1], params[2:]
|
|
187
177
|
),
|
|
188
178
|
forward_init=False,
|
|
189
|
-
update=update_parameter,
|
|
179
|
+
update="CONST" not in params and update_parameter,
|
|
190
180
|
)
|
|
191
181
|
for params in distributions_spec
|
|
192
182
|
]
|
|
@@ -195,7 +185,7 @@ class GenKwConfig(ParameterConfig):
|
|
|
195
185
|
[err.set_context(gen_kw_key) for err in e.errors]
|
|
196
186
|
) from e
|
|
197
187
|
except ValidationError as e:
|
|
198
|
-
raise ConfigValidationError.from_pydantic(e,
|
|
188
|
+
raise ConfigValidationError.from_pydantic(e, config_list) from e
|
|
199
189
|
|
|
200
190
|
def load_parameter_graph(self) -> nx.Graph[int]:
|
|
201
191
|
# Create a graph with no edges
|
|
@@ -217,18 +207,7 @@ class GenKwConfig(ParameterConfig):
|
|
|
217
207
|
real_nr: int,
|
|
218
208
|
ensemble: Ensemble,
|
|
219
209
|
) -> dict[str, dict[str, float | str]]:
|
|
220
|
-
|
|
221
|
-
"realization"
|
|
222
|
-
)
|
|
223
|
-
|
|
224
|
-
assert isinstance(df, pl.DataFrame)
|
|
225
|
-
if not df.width == 1:
|
|
226
|
-
raise ValueError(
|
|
227
|
-
f"GEN_KW {self.group_name}:{self.name} should be a single parameter!"
|
|
228
|
-
)
|
|
229
|
-
|
|
230
|
-
data = df.to_dicts()[0]
|
|
231
|
-
return {self.group_name: data}
|
|
210
|
+
raise NotImplementedError
|
|
232
211
|
|
|
233
212
|
def load_parameters(
|
|
234
213
|
self, ensemble: Ensemble, realizations: npt.NDArray[np.int_]
|
|
@@ -259,15 +238,6 @@ class GenKwConfig(ParameterConfig):
|
|
|
259
238
|
def group_name(self) -> str:
|
|
260
239
|
return self.group
|
|
261
240
|
|
|
262
|
-
def copy_parameters(
|
|
263
|
-
self,
|
|
264
|
-
source_ensemble: Ensemble,
|
|
265
|
-
target_ensemble: Ensemble,
|
|
266
|
-
realizations: npt.NDArray[np.int_],
|
|
267
|
-
) -> None:
|
|
268
|
-
df = source_ensemble.load_parameters(self.name, realizations)
|
|
269
|
-
target_ensemble.save_parameters(dataset=df)
|
|
270
|
-
|
|
271
241
|
def get_priors(self) -> list[PriorDict]:
|
|
272
242
|
dist_json = self.distribution.model_dump(exclude={"name"})
|
|
273
243
|
return [
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
from .everest_response import EverestConstraintsConfig, EverestObjectivesConfig
|
|
2
|
+
from .gen_data_config import GenDataConfig
|
|
3
|
+
from .rft_config import RFTConfig
|
|
4
|
+
from .summary_config import SummaryConfig
|
|
5
|
+
|
|
6
|
+
KnownErtResponseTypes = SummaryConfig | GenDataConfig | RFTConfig
|
|
7
|
+
KNOWN_ERT_RESPONSE_TYPES = (
|
|
8
|
+
SummaryConfig,
|
|
9
|
+
GenDataConfig,
|
|
10
|
+
RFTConfig,
|
|
11
|
+
)
|
|
12
|
+
KnownResponseTypes = (
|
|
13
|
+
KnownErtResponseTypes | EverestConstraintsConfig | EverestObjectivesConfig
|
|
14
|
+
)
|
ert/config/parameter_config.py
CHANGED
|
@@ -5,7 +5,7 @@ from collections.abc import Callable, Iterator
|
|
|
5
5
|
from enum import StrEnum, auto
|
|
6
6
|
from hashlib import sha256
|
|
7
7
|
from pathlib import Path
|
|
8
|
-
from typing import TYPE_CHECKING
|
|
8
|
+
from typing import TYPE_CHECKING
|
|
9
9
|
|
|
10
10
|
import networkx as nx
|
|
11
11
|
import numpy as np
|
|
@@ -39,13 +39,6 @@ class ParameterCardinality(StrEnum):
|
|
|
39
39
|
one_config_per_realization_dataset = auto()
|
|
40
40
|
|
|
41
41
|
|
|
42
|
-
class ParameterMetadata(BaseModel):
|
|
43
|
-
key: str
|
|
44
|
-
transformation: str | None
|
|
45
|
-
dimensionality: Literal[1, 2, 3] = 1
|
|
46
|
-
userdata: dict[str, Any]
|
|
47
|
-
|
|
48
|
-
|
|
49
42
|
class ParameterConfig(BaseModel):
|
|
50
43
|
type: str
|
|
51
44
|
name: str
|
|
@@ -59,14 +52,6 @@ class ParameterConfig(BaseModel):
|
|
|
59
52
|
Returns a list of parameter keys within this parameter group
|
|
60
53
|
"""
|
|
61
54
|
|
|
62
|
-
@property
|
|
63
|
-
@abstractmethod
|
|
64
|
-
def metadata(self) -> list[ParameterMetadata]:
|
|
65
|
-
"""
|
|
66
|
-
Returns metadata describing this parameter
|
|
67
|
-
|
|
68
|
-
"""
|
|
69
|
-
|
|
70
55
|
@abstractmethod
|
|
71
56
|
def __len__(self) -> int:
|
|
72
57
|
"""Number of parameters"""
|
|
@@ -108,23 +93,6 @@ class ParameterConfig(BaseModel):
|
|
|
108
93
|
or polars DataFrame from the numpy data
|
|
109
94
|
"""
|
|
110
95
|
|
|
111
|
-
def copy_parameters(
|
|
112
|
-
self,
|
|
113
|
-
source_ensemble: Ensemble,
|
|
114
|
-
target_ensemble: Ensemble,
|
|
115
|
-
realizations: npt.NDArray[np.int_],
|
|
116
|
-
) -> None:
|
|
117
|
-
"""
|
|
118
|
-
Copy parameters from one ensemble to another.
|
|
119
|
-
If realizations is None, copy all realizations.
|
|
120
|
-
If realizations is given, copy only those realizations.
|
|
121
|
-
"""
|
|
122
|
-
for realization in realizations:
|
|
123
|
-
# Converts to standard python scalar due to mypy
|
|
124
|
-
realization_int = int(realization)
|
|
125
|
-
ds = source_ensemble.load_parameters(self.name, realization_int)
|
|
126
|
-
target_ensemble.save_parameters(ds, self.name, realization_int)
|
|
127
|
-
|
|
128
96
|
@abstractmethod
|
|
129
97
|
def load_parameters(
|
|
130
98
|
self, ensemble: Ensemble, realizations: npt.NDArray[np.int_]
|
|
@@ -48,13 +48,21 @@ def option_dict(option_list: Sequence[str], offset: int) -> dict[str, str]:
|
|
|
48
48
|
if len(option_pair.split(":")) == 2:
|
|
49
49
|
key, val = option_pair.split(":")
|
|
50
50
|
if val and key:
|
|
51
|
+
if key in result:
|
|
52
|
+
raise ConfigValidationError.with_context(
|
|
53
|
+
f"Option {key} occured multiple times.", option_pair
|
|
54
|
+
)
|
|
51
55
|
result[key] = val
|
|
52
56
|
else:
|
|
53
57
|
raise ConfigValidationError.with_context(
|
|
54
|
-
|
|
58
|
+
"Option argument should be of the form 'key':'value', "
|
|
59
|
+
f"got {option_pair!r}",
|
|
60
|
+
option_pair,
|
|
55
61
|
)
|
|
56
62
|
else:
|
|
57
63
|
raise ConfigValidationError.with_context(
|
|
58
|
-
|
|
64
|
+
"Option argument should be of the form 'key':'value', "
|
|
65
|
+
f"got {option_pair!r}",
|
|
66
|
+
option_pair,
|
|
59
67
|
)
|
|
60
68
|
return result
|
|
@@ -76,7 +76,7 @@ class ConfigValidationError(ValueError):
|
|
|
76
76
|
def from_pydantic(cls, error: ValidationError, context: Any = None) -> Self:
|
|
77
77
|
parsed_errors = []
|
|
78
78
|
for pydantic_error_info in error.errors():
|
|
79
|
-
actual_error = pydantic_error_info
|
|
79
|
+
actual_error = pydantic_error_info.get("ctx", {}).get("error")
|
|
80
80
|
|
|
81
81
|
if isinstance(actual_error, ConfigValidationError):
|
|
82
82
|
parsed_errors += [
|
|
@@ -25,7 +25,6 @@ class ConfigKeys(StrEnum):
|
|
|
25
25
|
HISTORY_SOURCE = "HISTORY_SOURCE"
|
|
26
26
|
INSTALL_JOB = "INSTALL_JOB"
|
|
27
27
|
INSTALL_JOB_DIRECTORY = "INSTALL_JOB_DIRECTORY"
|
|
28
|
-
JOB_SCRIPT = "JOB_SCRIPT"
|
|
29
28
|
JOBNAME = "JOBNAME"
|
|
30
29
|
MAX_SUBMIT = "MAX_SUBMIT"
|
|
31
30
|
DESIGN_MATRIX = "DESIGN_MATRIX"
|
|
@@ -43,6 +42,7 @@ class ConfigKeys(StrEnum):
|
|
|
43
42
|
SETENV = "SETENV"
|
|
44
43
|
STD_CUTOFF = "STD_CUTOFF"
|
|
45
44
|
SUMMARY = "SUMMARY"
|
|
45
|
+
RFT = "RFT"
|
|
46
46
|
SURFACE = "SURFACE"
|
|
47
47
|
UPDATE_LOG_PATH = "UPDATE_LOG_PATH"
|
|
48
48
|
RANDOM_SEED = "RANDOM_SEED"
|
|
@@ -56,6 +56,7 @@ class ConfigKeys(StrEnum):
|
|
|
56
56
|
REALIZATION_MEMORY = "REALIZATION_MEMORY"
|
|
57
57
|
SUBMIT_SLEEP = "SUBMIT_SLEEP"
|
|
58
58
|
MAX_RUNNING = "MAX_RUNNING"
|
|
59
|
+
PRIORITIZE_PRIVATE_IP_ADDRESS = "PRIORITIZE_PRIVATE_IP_ADDRESS"
|
|
59
60
|
|
|
60
61
|
def __repr__(self) -> str:
|
|
61
62
|
return f"{self.value!r}"
|
|
@@ -4,8 +4,8 @@ from .config_schema_deprecations import deprecated_keywords_list
|
|
|
4
4
|
from .config_schema_item import (
|
|
5
5
|
SchemaItem,
|
|
6
6
|
Varies,
|
|
7
|
+
existing_file_keyword,
|
|
7
8
|
existing_path_inline_keyword,
|
|
8
|
-
existing_path_keyword,
|
|
9
9
|
float_keyword,
|
|
10
10
|
int_keyword,
|
|
11
11
|
path_keyword,
|
|
@@ -63,6 +63,17 @@ def data_kw_keyword() -> SchemaItem:
|
|
|
63
63
|
)
|
|
64
64
|
|
|
65
65
|
|
|
66
|
+
def rft_keyword() -> SchemaItem:
|
|
67
|
+
return SchemaItem(
|
|
68
|
+
kw=ConfigKeys.RFT,
|
|
69
|
+
required_set=False,
|
|
70
|
+
multi_occurrence=True,
|
|
71
|
+
options_after=0,
|
|
72
|
+
argc_min=1,
|
|
73
|
+
argc_max=1,
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
|
|
66
77
|
def define_keyword() -> SchemaItem:
|
|
67
78
|
return SchemaItem(
|
|
68
79
|
kw=ConfigKeys.DEFINE,
|
|
@@ -115,6 +126,13 @@ def hook_workflow_keyword() -> SchemaItem:
|
|
|
115
126
|
)
|
|
116
127
|
|
|
117
128
|
|
|
129
|
+
def prioritize_private_ip_address_keyword() -> SchemaItem:
|
|
130
|
+
return SchemaItem(
|
|
131
|
+
kw=ConfigKeys.PRIORITIZE_PRIVATE_IP_ADDRESS,
|
|
132
|
+
type_map=[SchemaItemType.BOOL],
|
|
133
|
+
)
|
|
134
|
+
|
|
135
|
+
|
|
118
136
|
def set_env_keyword() -> SchemaItem:
|
|
119
137
|
# You can set environment variables which will be applied to the run-time
|
|
120
138
|
# environment.
|
|
@@ -176,13 +194,6 @@ def queue_option_keyword() -> SchemaItem:
|
|
|
176
194
|
)
|
|
177
195
|
|
|
178
196
|
|
|
179
|
-
def job_script_keyword() -> SchemaItem:
|
|
180
|
-
return SchemaItem(
|
|
181
|
-
kw=ConfigKeys.JOB_SCRIPT,
|
|
182
|
-
type_map=[SchemaItemType.EXECUTABLE],
|
|
183
|
-
)
|
|
184
|
-
|
|
185
|
-
|
|
186
197
|
def gen_kw_keyword() -> SchemaItem:
|
|
187
198
|
return SchemaItem(
|
|
188
199
|
kw=ConfigKeys.GEN_KW,
|
|
@@ -310,11 +321,12 @@ def init_user_config_schema() -> ConfigSchemaDict:
|
|
|
310
321
|
gen_kw_keyword(),
|
|
311
322
|
gen_data_keyword(),
|
|
312
323
|
summary_keyword(),
|
|
324
|
+
rft_keyword(),
|
|
313
325
|
surface_keyword(),
|
|
314
326
|
field_keyword(),
|
|
315
327
|
single_arg_keyword(ConfigKeys.ECLBASE),
|
|
316
|
-
|
|
317
|
-
|
|
328
|
+
existing_file_keyword(ConfigKeys.DATA_FILE),
|
|
329
|
+
existing_file_keyword(ConfigKeys.GRID),
|
|
318
330
|
path_keyword(ConfigKeys.REFCASE),
|
|
319
331
|
int_keyword(ConfigKeys.RANDOM_SEED),
|
|
320
332
|
num_realizations_keyword(),
|
|
@@ -339,12 +351,12 @@ def init_user_config_schema() -> ConfigSchemaDict:
|
|
|
339
351
|
design_matrix_keyword(),
|
|
340
352
|
queue_system_keyword(False),
|
|
341
353
|
queue_option_keyword(),
|
|
342
|
-
job_script_keyword(),
|
|
343
354
|
load_workflow_job_keyword(),
|
|
344
355
|
set_env_keyword(),
|
|
345
356
|
install_job_keyword(),
|
|
346
357
|
install_job_directory_keyword(),
|
|
347
358
|
hook_workflow_keyword(),
|
|
359
|
+
prioritize_private_ip_address_keyword(),
|
|
348
360
|
]:
|
|
349
361
|
schema[item.kw] = item
|
|
350
362
|
if item.kw in ConfigAliases:
|
|
@@ -39,9 +39,9 @@ deprecated_keywords_list = [
|
|
|
39
39
|
keyword=kw,
|
|
40
40
|
message=partial(
|
|
41
41
|
lambda line, kw: f"Using {kw} with substitution strings "
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
42
|
+
"that are not of the form '<KEY>' is deprecated. "
|
|
43
|
+
f"Please change {line[0]} to "
|
|
44
|
+
f"<{line[0].replace('<', '').replace('>', '')}>",
|
|
45
45
|
kw=kw,
|
|
46
46
|
),
|
|
47
47
|
check=lambda line: not DeprecationInfo.is_angle_bracketed(str(line[0])),
|
|
@@ -37,25 +37,34 @@ class SchemaItem:
|
|
|
37
37
|
|
|
38
38
|
# The minimum number of arguments
|
|
39
39
|
argc_min: NonNegativeInt = 1
|
|
40
|
+
|
|
40
41
|
# The maximum number of arguments: None means no upper limit
|
|
41
42
|
argc_max: NonNegativeInt | None = 1
|
|
43
|
+
|
|
42
44
|
# A list of types for the items. Set along with argc_minmax()
|
|
43
45
|
type_map: list[SchemaItemType | EnumType | None] = Field(default_factory=list)
|
|
46
|
+
|
|
44
47
|
# A list of item's which must also be set (if this item is set). (can be NULL)
|
|
45
48
|
required_children: list[str] = Field(default_factory=list)
|
|
49
|
+
|
|
46
50
|
# Information about the deprecation if deprecated
|
|
47
51
|
deprecation_info: list[DeprecationInfo] = Field(default_factory=list)
|
|
48
|
-
|
|
52
|
+
|
|
53
|
+
# If positive, arguments after this count will be concatenated with a " " between
|
|
49
54
|
join_after: PositiveInt | None = None
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
55
|
+
|
|
56
|
+
# If positive, arguments after this count will be interpreted as options
|
|
57
|
+
options_after: NonNegativeInt | Varies | None = None
|
|
58
|
+
|
|
59
|
+
# If true, will accumulate many values set for key, otherwise each entry will
|
|
53
60
|
# overwrite any previous value set
|
|
54
61
|
multi_occurrence: bool = False
|
|
62
|
+
|
|
55
63
|
# Only applies to SchemaItemType.EXISTING_PATH_INLINE where
|
|
56
64
|
# the contents is then parsed
|
|
57
65
|
parser: Callable[[str, str], Any] = lambda x, y: y
|
|
58
66
|
expand_envvar: bool = True
|
|
67
|
+
|
|
59
68
|
# Index of tokens to do substitution from until end
|
|
60
69
|
substitute_from: NonNegativeInt = 1
|
|
61
70
|
required_set: bool = False
|
|
@@ -155,9 +164,9 @@ class SchemaItem:
|
|
|
155
164
|
f"value as argument {index + 1!r}",
|
|
156
165
|
token,
|
|
157
166
|
)
|
|
158
|
-
|
|
159
167
|
case (
|
|
160
168
|
SchemaItemType.PATH
|
|
169
|
+
| SchemaItemType.EXISTING_FILE
|
|
161
170
|
| SchemaItemType.EXISTING_PATH
|
|
162
171
|
| SchemaItemType.EXISTING_PATH_INLINE
|
|
163
172
|
):
|
|
@@ -166,10 +175,14 @@ class SchemaItem:
|
|
|
166
175
|
path = os.path.normpath(
|
|
167
176
|
os.path.join(os.path.dirname(token.filename), token)
|
|
168
177
|
)
|
|
169
|
-
if val_type
|
|
170
|
-
SchemaItemType.
|
|
171
|
-
|
|
172
|
-
|
|
178
|
+
if val_type != SchemaItemType.PATH:
|
|
179
|
+
if val_type == SchemaItemType.EXISTING_FILE and not os.path.isfile(
|
|
180
|
+
str(path)
|
|
181
|
+
):
|
|
182
|
+
raise ConfigValidationError.with_context(
|
|
183
|
+
f"{self.kw} {token} is not a file.",
|
|
184
|
+
token,
|
|
185
|
+
)
|
|
173
186
|
if not os.path.exists(str(path)):
|
|
174
187
|
err = f'Cannot find file or directory "{token.value}". '
|
|
175
188
|
if path != token:
|
|
@@ -311,6 +324,8 @@ class SchemaItem:
|
|
|
311
324
|
self, line: Sequence[FileContextToken]
|
|
312
325
|
) -> Sequence[FileContextToken | dict[FileContextToken, FileContextToken]]:
|
|
313
326
|
n = self.options_after
|
|
327
|
+
if not line:
|
|
328
|
+
return []
|
|
314
329
|
if isinstance(n, Varies):
|
|
315
330
|
args, kwargs = parse_variable_options(list(line), n.max_positionals)
|
|
316
331
|
return [*args, kwargs] # type: ignore
|
|
@@ -343,8 +358,8 @@ def path_keyword(keyword: str) -> SchemaItem:
|
|
|
343
358
|
return SchemaItem(kw=keyword, type_map=[SchemaItemType.PATH])
|
|
344
359
|
|
|
345
360
|
|
|
346
|
-
def
|
|
347
|
-
return SchemaItem(kw=keyword, type_map=[SchemaItemType.
|
|
361
|
+
def existing_file_keyword(keyword: str) -> SchemaItem:
|
|
362
|
+
return SchemaItem(kw=keyword, type_map=[SchemaItemType.EXISTING_FILE])
|
|
348
363
|
|
|
349
364
|
|
|
350
365
|
def existing_path_inline_keyword(
|