ert 18.0.9__py3-none-any.whl → 19.0.0rc0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- _ert/forward_model_runner/client.py +6 -2
- ert/__main__.py +20 -6
- ert/analysis/_es_update.py +6 -19
- ert/cli/main.py +7 -3
- ert/config/__init__.py +3 -4
- ert/config/_create_observation_dataframes.py +57 -8
- ert/config/_get_num_cpu.py +1 -1
- ert/config/_observations.py +77 -1
- ert/config/distribution.py +1 -1
- ert/config/ensemble_config.py +3 -3
- ert/config/ert_config.py +50 -8
- ert/config/{ext_param_config.py → everest_control.py} +8 -12
- ert/config/everest_response.py +3 -5
- ert/config/field.py +76 -14
- ert/config/forward_model_step.py +12 -9
- ert/config/gen_data_config.py +3 -4
- ert/config/gen_kw_config.py +2 -12
- ert/config/parameter_config.py +1 -16
- ert/config/parsing/_option_dict.py +10 -2
- ert/config/parsing/config_keywords.py +1 -0
- ert/config/parsing/config_schema.py +8 -0
- ert/config/parsing/config_schema_deprecations.py +14 -3
- ert/config/parsing/config_schema_item.py +12 -3
- ert/config/parsing/context_values.py +3 -3
- ert/config/parsing/file_context_token.py +1 -1
- ert/config/parsing/observations_parser.py +6 -2
- ert/config/parsing/queue_system.py +9 -0
- ert/config/queue_config.py +0 -1
- ert/config/response_config.py +0 -1
- ert/config/rft_config.py +78 -33
- ert/config/summary_config.py +1 -2
- ert/config/surface_config.py +59 -16
- ert/dark_storage/common.py +1 -1
- ert/dark_storage/compute/misfits.py +4 -1
- ert/dark_storage/endpoints/compute/misfits.py +4 -2
- ert/dark_storage/endpoints/experiment_server.py +12 -9
- ert/dark_storage/endpoints/experiments.py +2 -2
- ert/dark_storage/endpoints/observations.py +4 -2
- ert/dark_storage/endpoints/parameters.py +2 -18
- ert/dark_storage/endpoints/responses.py +10 -5
- ert/dark_storage/json_schema/experiment.py +1 -1
- ert/data/_measured_data.py +6 -5
- ert/ensemble_evaluator/config.py +2 -1
- ert/field_utils/field_utils.py +1 -1
- ert/field_utils/grdecl_io.py +9 -26
- ert/field_utils/roff_io.py +1 -1
- ert/gui/__init__.py +5 -2
- ert/gui/ertnotifier.py +1 -1
- ert/gui/ertwidgets/pathchooser.py +0 -3
- ert/gui/ertwidgets/suggestor/suggestor.py +63 -30
- ert/gui/main.py +27 -5
- ert/gui/main_window.py +0 -5
- ert/gui/simulation/experiment_panel.py +12 -7
- ert/gui/simulation/run_dialog.py +2 -16
- ert/gui/summarypanel.py +0 -19
- ert/gui/tools/manage_experiments/export_dialog.py +136 -0
- ert/gui/tools/manage_experiments/storage_info_widget.py +110 -9
- ert/gui/tools/plot/plot_api.py +24 -15
- ert/gui/tools/plot/plot_widget.py +10 -2
- ert/gui/tools/plot/plot_window.py +26 -18
- ert/gui/tools/plot/plottery/plots/__init__.py +2 -0
- ert/gui/tools/plot/plottery/plots/cesp.py +3 -1
- ert/gui/tools/plot/plottery/plots/distribution.py +6 -1
- ert/gui/tools/plot/plottery/plots/ensemble.py +3 -1
- ert/gui/tools/plot/plottery/plots/gaussian_kde.py +12 -2
- ert/gui/tools/plot/plottery/plots/histogram.py +3 -1
- ert/gui/tools/plot/plottery/plots/misfits.py +436 -0
- ert/gui/tools/plot/plottery/plots/observations.py +18 -4
- ert/gui/tools/plot/plottery/plots/statistics.py +3 -1
- ert/gui/tools/plot/plottery/plots/std_dev.py +3 -1
- ert/plugins/hook_implementations/workflows/csv_export.py +2 -3
- ert/plugins/plugin_manager.py +4 -0
- ert/resources/forward_models/run_reservoirsimulator.py +8 -3
- ert/run_models/_create_run_path.py +3 -3
- ert/run_models/everest_run_model.py +13 -11
- ert/run_models/initial_ensemble_run_model.py +2 -2
- ert/run_models/run_model.py +30 -1
- ert/services/_base_service.py +6 -5
- ert/services/ert_server.py +4 -4
- ert/shared/_doc_utils/__init__.py +4 -2
- ert/shared/net_utils.py +43 -18
- ert/shared/version.py +3 -3
- ert/storage/__init__.py +2 -0
- ert/storage/local_ensemble.py +13 -7
- ert/storage/local_experiment.py +2 -2
- ert/storage/local_storage.py +41 -25
- ert/storage/migration/to11.py +1 -1
- ert/storage/migration/to18.py +0 -1
- ert/storage/migration/to19.py +34 -0
- ert/storage/migration/to20.py +23 -0
- ert/storage/migration/to21.py +25 -0
- ert/workflow_runner.py +2 -1
- {ert-18.0.9.dist-info → ert-19.0.0rc0.dist-info}/METADATA +1 -1
- {ert-18.0.9.dist-info → ert-19.0.0rc0.dist-info}/RECORD +112 -112
- {ert-18.0.9.dist-info → ert-19.0.0rc0.dist-info}/WHEEL +1 -1
- everest/bin/everlint_script.py +0 -2
- everest/bin/utils.py +2 -1
- everest/bin/visualization_script.py +4 -11
- everest/config/control_config.py +4 -4
- everest/config/control_variable_config.py +2 -2
- everest/config/everest_config.py +9 -0
- everest/config/utils.py +2 -2
- everest/config/validation_utils.py +7 -1
- everest/config_file_loader.py +0 -2
- everest/detached/client.py +3 -3
- everest/everest_storage.py +0 -2
- everest/gui/everest_client.py +2 -2
- everest/optimizer/everest2ropt.py +4 -4
- everest/optimizer/opt_model_transforms.py +2 -2
- ert/config/violations.py +0 -0
- ert/gui/tools/export/__init__.py +0 -3
- ert/gui/tools/export/export_panel.py +0 -83
- ert/gui/tools/export/export_tool.py +0 -69
- ert/gui/tools/export/exporter.py +0 -36
- {ert-18.0.9.dist-info → ert-19.0.0rc0.dist-info}/entry_points.txt +0 -0
- {ert-18.0.9.dist-info → ert-19.0.0rc0.dist-info}/licenses/COPYING +0 -0
- {ert-18.0.9.dist-info → ert-19.0.0rc0.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,436 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import TYPE_CHECKING, Literal, cast
|
|
4
|
+
|
|
5
|
+
import matplotlib.dates as mdates
|
|
6
|
+
import numpy as np
|
|
7
|
+
import pandas as pd
|
|
8
|
+
import polars as pl
|
|
9
|
+
import seaborn as sns
|
|
10
|
+
from matplotlib import pyplot as plt
|
|
11
|
+
from matplotlib.lines import Line2D
|
|
12
|
+
|
|
13
|
+
if TYPE_CHECKING:
|
|
14
|
+
import numpy.typing as npt
|
|
15
|
+
from matplotlib.figure import Figure
|
|
16
|
+
|
|
17
|
+
from ert.gui.tools.plot.plot_api import EnsembleObject, PlotApiKeyDefinition
|
|
18
|
+
from ert.gui.tools.plot.plottery import PlotContext
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class MisfitsPlot:
|
|
22
|
+
"""
|
|
23
|
+
Visualize signed chi-squared misfits between simulated responses and observations.
|
|
24
|
+
|
|
25
|
+
Layout:
|
|
26
|
+
- X-axis: index for gen data, time for summary
|
|
27
|
+
- Y-axis: signed chi-squared misfits
|
|
28
|
+
- Glyphs: One boxplot per time step / gendata index
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
def __init__(self) -> None:
|
|
32
|
+
self.dimensionality = 2
|
|
33
|
+
self.requires_observations = True
|
|
34
|
+
|
|
35
|
+
@staticmethod
|
|
36
|
+
def _fade_axis_ticklabels(ax: plt.Axes) -> None:
|
|
37
|
+
"""Apply the same alpha to all tick labels on an axis."""
|
|
38
|
+
for label in (*ax.get_xticklabels(), *ax.get_yticklabels()):
|
|
39
|
+
label.set_alpha(0.4)
|
|
40
|
+
|
|
41
|
+
ax.xaxis.label.set_alpha(0.4)
|
|
42
|
+
ax.yaxis.label.set_alpha(0.4)
|
|
43
|
+
|
|
44
|
+
@staticmethod
|
|
45
|
+
def _draw_legend(
|
|
46
|
+
figure: Figure,
|
|
47
|
+
ensemble_colors: dict[tuple[str, str], str],
|
|
48
|
+
sorted_ensemble_keys: list[tuple[str, str]],
|
|
49
|
+
) -> None:
|
|
50
|
+
legend_handles = [
|
|
51
|
+
Line2D(
|
|
52
|
+
[],
|
|
53
|
+
[],
|
|
54
|
+
marker="s",
|
|
55
|
+
linestyle="None",
|
|
56
|
+
color=ensemble_colors[key],
|
|
57
|
+
label=key[0],
|
|
58
|
+
)
|
|
59
|
+
for key in sorted_ensemble_keys
|
|
60
|
+
]
|
|
61
|
+
figure.legend(
|
|
62
|
+
handles=legend_handles,
|
|
63
|
+
loc="upper center",
|
|
64
|
+
bbox_to_anchor=(0.5, 0.93),
|
|
65
|
+
ncol=min(len(legend_handles), 4),
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
@staticmethod
|
|
69
|
+
def _make_ensemble_colors(
|
|
70
|
+
sorted_ensemble_keys: list[tuple[str, str]],
|
|
71
|
+
) -> dict[tuple[str, str], str]:
|
|
72
|
+
"""
|
|
73
|
+
Build a consistent color map and figure-level legend
|
|
74
|
+
used by all misfit plots
|
|
75
|
+
"""
|
|
76
|
+
colors = plt.rcParams["axes.prop_cycle"].by_key()["color"]
|
|
77
|
+
return {
|
|
78
|
+
key: colors[i % len(colors)] for i, key in enumerate(sorted_ensemble_keys)
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
@staticmethod
|
|
82
|
+
def _compute_misfits_padded_minmax(
|
|
83
|
+
misfits_df: pl.DataFrame, relative_pad_y_axis: float
|
|
84
|
+
) -> tuple[float, float]:
|
|
85
|
+
y_min, y_max = (
|
|
86
|
+
cast(float, misfits_df["misfit"].min()),
|
|
87
|
+
cast(float, misfits_df["misfit"].max()),
|
|
88
|
+
)
|
|
89
|
+
abs_pad_y = (y_max - y_min) * relative_pad_y_axis
|
|
90
|
+
y_min -= abs_pad_y
|
|
91
|
+
y_max += abs_pad_y
|
|
92
|
+
|
|
93
|
+
return y_min, y_max
|
|
94
|
+
|
|
95
|
+
@staticmethod
|
|
96
|
+
def _wide_pandas_to_long_polars_with_misfits(
|
|
97
|
+
ensemble_to_data_map: dict[tuple[str, str], pd.DataFrame],
|
|
98
|
+
observation_data: pd.DataFrame,
|
|
99
|
+
response_type: Literal["summary", "gen_data"],
|
|
100
|
+
) -> dict[tuple[str, str], pl.DataFrame]:
|
|
101
|
+
if response_type == "summary":
|
|
102
|
+
key_index_with_correct_dtype = pl.col("key_index").str.to_datetime(
|
|
103
|
+
strict=False
|
|
104
|
+
)
|
|
105
|
+
elif response_type == "gen_data":
|
|
106
|
+
key_index_with_correct_dtype = (
|
|
107
|
+
pl.col("key_index").cast(pl.Float32).cast(pl.UInt16)
|
|
108
|
+
)
|
|
109
|
+
else:
|
|
110
|
+
raise ValueError(f"Unsupported response_type: {response_type}")
|
|
111
|
+
|
|
112
|
+
obs_df = (
|
|
113
|
+
pl.from_pandas(observation_data.T)
|
|
114
|
+
.rename({"OBS": "observation", "STD": "error"})
|
|
115
|
+
.with_columns(pl.col("key_index").cast(pl.String))
|
|
116
|
+
.with_columns(key_index_with_correct_dtype)
|
|
117
|
+
)
|
|
118
|
+
|
|
119
|
+
return {
|
|
120
|
+
ens_key: (
|
|
121
|
+
pl.from_pandas(df, include_index=True)
|
|
122
|
+
.unpivot(
|
|
123
|
+
index=df.index.name,
|
|
124
|
+
variable_name="key_index",
|
|
125
|
+
value_name="response",
|
|
126
|
+
)
|
|
127
|
+
.with_columns(key_index_with_correct_dtype)
|
|
128
|
+
.join(obs_df, on="key_index", how="inner")
|
|
129
|
+
.with_columns(
|
|
130
|
+
(pl.col("response") - pl.col("observation")).alias("residual")
|
|
131
|
+
)
|
|
132
|
+
.with_columns(
|
|
133
|
+
(
|
|
134
|
+
pl.col("residual").sign()
|
|
135
|
+
* (pl.col("residual") / pl.col("error")).pow(2)
|
|
136
|
+
).alias("misfit")
|
|
137
|
+
)
|
|
138
|
+
.drop("residual")
|
|
139
|
+
)
|
|
140
|
+
for ens_key, df in ensemble_to_data_map.items()
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
def plot(
|
|
144
|
+
self,
|
|
145
|
+
figure: Figure,
|
|
146
|
+
plot_context: PlotContext,
|
|
147
|
+
ensemble_to_data_map: dict[EnsembleObject, pd.DataFrame],
|
|
148
|
+
observation_data: pd.DataFrame,
|
|
149
|
+
std_dev_images: dict[str, npt.NDArray[np.float32]],
|
|
150
|
+
key_def: PlotApiKeyDefinition | None = None,
|
|
151
|
+
) -> None:
|
|
152
|
+
assert key_def is not None
|
|
153
|
+
response_type = key_def.metadata["data_origin"]
|
|
154
|
+
data_with_misfits = self._wide_pandas_to_long_polars_with_misfits(
|
|
155
|
+
{(eo.name, eo.id): df for eo, df in ensemble_to_data_map.items()},
|
|
156
|
+
observation_data,
|
|
157
|
+
response_type,
|
|
158
|
+
)
|
|
159
|
+
|
|
160
|
+
if response_type == "summary":
|
|
161
|
+
self._plot_summary_misfits_boxplots(
|
|
162
|
+
figure,
|
|
163
|
+
data_with_misfits,
|
|
164
|
+
plot_context,
|
|
165
|
+
)
|
|
166
|
+
|
|
167
|
+
elif response_type == "gen_data":
|
|
168
|
+
self._plot_gendata_misfits(
|
|
169
|
+
figure,
|
|
170
|
+
data_with_misfits,
|
|
171
|
+
plot_context,
|
|
172
|
+
)
|
|
173
|
+
|
|
174
|
+
def _plot_gendata_misfits(
|
|
175
|
+
self,
|
|
176
|
+
figure: Figure,
|
|
177
|
+
data_with_misfits: dict[tuple[str, str], pl.DataFrame],
|
|
178
|
+
plot_context: PlotContext,
|
|
179
|
+
) -> None:
|
|
180
|
+
# Only plot ensembles with data (i.e., they pertain to an experiment
|
|
181
|
+
# with observations, and there are responses towards those in the ens)
|
|
182
|
+
ensemble_to_misfit_df = {
|
|
183
|
+
k: v for k, v in data_with_misfits.items() if not v.is_empty()
|
|
184
|
+
}
|
|
185
|
+
sorted_ensemble_keys = sorted(ensemble_to_misfit_df.keys())
|
|
186
|
+
|
|
187
|
+
all_misfits = pl.concat(
|
|
188
|
+
[
|
|
189
|
+
df.with_columns(pl.lit(key).alias("ensemble_key"))
|
|
190
|
+
for key, df in ensemble_to_misfit_df.items()
|
|
191
|
+
]
|
|
192
|
+
).select(["realization", "key_index", "misfit", "ensemble_key"])
|
|
193
|
+
|
|
194
|
+
distinct_gendata_index = all_misfits["key_index"].unique().sort().to_list()
|
|
195
|
+
num_gendata_index = len(distinct_gendata_index)
|
|
196
|
+
|
|
197
|
+
color_map = self._make_ensemble_colors(sorted_ensemble_keys)
|
|
198
|
+
self._draw_legend(
|
|
199
|
+
figure=figure,
|
|
200
|
+
ensemble_colors=color_map,
|
|
201
|
+
sorted_ensemble_keys=sorted_ensemble_keys,
|
|
202
|
+
)
|
|
203
|
+
|
|
204
|
+
y_min, y_max = self._compute_misfits_padded_minmax(all_misfits, 0.05)
|
|
205
|
+
|
|
206
|
+
# Create subplot grid (2 rows, N columns)
|
|
207
|
+
axes = figure.subplots(
|
|
208
|
+
nrows=2, ncols=num_gendata_index, sharex="col", sharey=True
|
|
209
|
+
)
|
|
210
|
+
axes = (
|
|
211
|
+
axes.reshape(2, num_gendata_index)
|
|
212
|
+
if num_gendata_index > 1
|
|
213
|
+
else np.array([[axes[0]], [axes[1]]])
|
|
214
|
+
)
|
|
215
|
+
axes_top, axes_bottom = axes[0, :], axes[1, :]
|
|
216
|
+
|
|
217
|
+
x_positions = np.arange(len(sorted_ensemble_keys))
|
|
218
|
+
box_width_relative = 0.6
|
|
219
|
+
|
|
220
|
+
for col_idx, key_index in enumerate(distinct_gendata_index):
|
|
221
|
+
ax_top, ax_bottom = axes_top[col_idx], axes_bottom[col_idx]
|
|
222
|
+
|
|
223
|
+
for ens_idx, ens_key in enumerate(sorted_ensemble_keys):
|
|
224
|
+
color = color_map.get(ens_key, "C0")
|
|
225
|
+
|
|
226
|
+
# Filter for the specific key and ensemble
|
|
227
|
+
mis_vals = all_misfits.filter(
|
|
228
|
+
(pl.col("key_index") == key_index)
|
|
229
|
+
& (pl.col("ensemble_key") == ens_key)
|
|
230
|
+
)["misfit"].to_numpy()
|
|
231
|
+
|
|
232
|
+
if mis_vals.size == 0:
|
|
233
|
+
continue
|
|
234
|
+
|
|
235
|
+
x_center = x_positions[ens_idx]
|
|
236
|
+
|
|
237
|
+
# Top: Boxplot
|
|
238
|
+
ax_top.boxplot(
|
|
239
|
+
mis_vals,
|
|
240
|
+
positions=[x_center],
|
|
241
|
+
widths=box_width_relative,
|
|
242
|
+
patch_artist=True,
|
|
243
|
+
showfliers=False,
|
|
244
|
+
boxprops={"facecolor": color, "alpha": 0.35},
|
|
245
|
+
whiskerprops={"color": color, "alpha": 0.8},
|
|
246
|
+
capprops={"color": color, "alpha": 0.8},
|
|
247
|
+
medianprops={"color": color, "alpha": 0.8},
|
|
248
|
+
)
|
|
249
|
+
ax_top.plot(x_center, np.mean(mis_vals), "o", markersize=4, color=color)
|
|
250
|
+
|
|
251
|
+
# Bottom: Strip plot with dynamic marker size
|
|
252
|
+
num_points = len(mis_vals)
|
|
253
|
+
|
|
254
|
+
if num_points >= 200:
|
|
255
|
+
marker_size = 2
|
|
256
|
+
elif num_points >= 100:
|
|
257
|
+
marker_size = 3
|
|
258
|
+
else:
|
|
259
|
+
marker_size = 4
|
|
260
|
+
|
|
261
|
+
# Use stripplot as a robust alternative for dense data
|
|
262
|
+
sns.stripplot(
|
|
263
|
+
x=[x_center] * num_points, # Plot all points at the same x-center
|
|
264
|
+
y=mis_vals,
|
|
265
|
+
ax=ax_bottom,
|
|
266
|
+
color=color,
|
|
267
|
+
size=marker_size,
|
|
268
|
+
alpha=0.35,
|
|
269
|
+
jitter=True, # Explicitly add jitter
|
|
270
|
+
)
|
|
271
|
+
|
|
272
|
+
# Axis/spine styling
|
|
273
|
+
(n_rows, n_cols) = axes.shape
|
|
274
|
+
for r_idx in range(n_rows):
|
|
275
|
+
for c_idx in range(n_cols):
|
|
276
|
+
ax = axes[r_idx, c_idx]
|
|
277
|
+
is_first_col = c_idx == 0
|
|
278
|
+
is_bottom_row = r_idx == (n_rows - 1)
|
|
279
|
+
|
|
280
|
+
# Common styling
|
|
281
|
+
ax.set(ylim=(y_min, y_max))
|
|
282
|
+
ax.axhline(0.0, color="black", linewidth=0.5, alpha=0.5)
|
|
283
|
+
ax.grid(True, axis="y", linestyle=":", alpha=0.4)
|
|
284
|
+
self._fade_axis_ticklabels(ax)
|
|
285
|
+
|
|
286
|
+
# Spines
|
|
287
|
+
ax.spines["top"].set_visible(False)
|
|
288
|
+
ax.spines["right"].set_visible(False)
|
|
289
|
+
ax.spines["left"].set_visible(is_first_col)
|
|
290
|
+
ax.spines["bottom"].set_visible(is_bottom_row)
|
|
291
|
+
|
|
292
|
+
# Ticks
|
|
293
|
+
ax.set_xticks(x_positions, labels=[])
|
|
294
|
+
if not is_bottom_row:
|
|
295
|
+
ax.tick_params(axis="x", which="both", bottom=False)
|
|
296
|
+
if not is_first_col:
|
|
297
|
+
ax.tick_params(axis="y", which="both", left=False)
|
|
298
|
+
|
|
299
|
+
for ax, key_val in zip(axes_bottom, distinct_gendata_index, strict=True):
|
|
300
|
+
ax.set_xlabel(f"index={int(key_val)}", rotation=25, ha="right")
|
|
301
|
+
|
|
302
|
+
figure.suptitle(
|
|
303
|
+
f"{plot_context.key()} (Signed Chi-squared misfits per index)",
|
|
304
|
+
fontsize=14,
|
|
305
|
+
y=0.98,
|
|
306
|
+
)
|
|
307
|
+
figure.tight_layout(rect=(0.02, 0.02, 0.98, 0.88))
|
|
308
|
+
|
|
309
|
+
def _plot_summary_misfits_boxplots(
|
|
310
|
+
self,
|
|
311
|
+
figure: Figure,
|
|
312
|
+
data_with_misfits: dict[tuple[str, str], pl.DataFrame],
|
|
313
|
+
plot_context: PlotContext,
|
|
314
|
+
) -> None:
|
|
315
|
+
# Calculate shared y-axis limits from all misfits
|
|
316
|
+
all_misfits = pl.concat(
|
|
317
|
+
[df.select("misfit") for df in data_with_misfits.values()]
|
|
318
|
+
)
|
|
319
|
+
y_min, y_max = self._compute_misfits_padded_minmax(all_misfits, 0.05)
|
|
320
|
+
|
|
321
|
+
# Prepare ensemble colors and draw the legend
|
|
322
|
+
sorted_ensemble_keys = sorted(data_with_misfits.keys())
|
|
323
|
+
color_map = self._make_ensemble_colors(sorted_ensemble_keys)
|
|
324
|
+
self._draw_legend(
|
|
325
|
+
figure=figure,
|
|
326
|
+
ensemble_colors=color_map,
|
|
327
|
+
sorted_ensemble_keys=sorted_ensemble_keys,
|
|
328
|
+
)
|
|
329
|
+
|
|
330
|
+
# Create all subplots at once with shared axes
|
|
331
|
+
n_ens = len(sorted_ensemble_keys)
|
|
332
|
+
axes = figure.subplots(nrows=n_ens, ncols=1, sharex=True, sharey=True)
|
|
333
|
+
axes = [axes] if n_ens == 1 else axes.tolist()
|
|
334
|
+
axes[0].set_ylim(y_min, y_max)
|
|
335
|
+
|
|
336
|
+
for ax, ensemble_key in zip(axes, sorted_ensemble_keys, strict=True):
|
|
337
|
+
df = data_with_misfits[ensemble_key]
|
|
338
|
+
if df.is_empty():
|
|
339
|
+
continue
|
|
340
|
+
|
|
341
|
+
df = df.select(["key_index", "misfit"]).sort("key_index")
|
|
342
|
+
times_py = df["key_index"].unique(maintain_order=True).to_list()
|
|
343
|
+
positions = mdates.date2num(times_py) # type: ignore[no-untyped-call]
|
|
344
|
+
|
|
345
|
+
# Calculate dynamic box width based on time spacing
|
|
346
|
+
min_dt = np.min(np.diff(positions)) if len(positions) > 1 else 1.0
|
|
347
|
+
|
|
348
|
+
# multiplier to downsize outlier sizes etc
|
|
349
|
+
# (without this, outliers, whiskers etc are sized way
|
|
350
|
+
# out of proportion when there are many tiny boxplots)
|
|
351
|
+
many_boxes_factor = min(1, len(times_py) / 50)
|
|
352
|
+
box_width = min_dt * (0.7 - 0.3 * (1 - many_boxes_factor))
|
|
353
|
+
|
|
354
|
+
# One boxplot per time step
|
|
355
|
+
grouped_misfits = df.group_by("key_index", maintain_order=True).agg(
|
|
356
|
+
pl.col("misfit")
|
|
357
|
+
)
|
|
358
|
+
data_for_boxes = [
|
|
359
|
+
s.to_numpy() if s.len() > 0 else np.array([np.nan])
|
|
360
|
+
for s in grouped_misfits["misfit"]
|
|
361
|
+
]
|
|
362
|
+
|
|
363
|
+
color = color_map.get(ensemble_key)
|
|
364
|
+
|
|
365
|
+
# Draw the boxplots with inlined styles
|
|
366
|
+
bp = ax.boxplot(
|
|
367
|
+
data_for_boxes,
|
|
368
|
+
positions=positions,
|
|
369
|
+
widths=box_width,
|
|
370
|
+
whis=(5, 95),
|
|
371
|
+
showfliers=True,
|
|
372
|
+
manage_ticks=False,
|
|
373
|
+
patch_artist=True,
|
|
374
|
+
boxprops={
|
|
375
|
+
"facecolor": color,
|
|
376
|
+
"alpha": 0.18,
|
|
377
|
+
"edgecolor": color,
|
|
378
|
+
"linewidth": 0.7,
|
|
379
|
+
},
|
|
380
|
+
whiskerprops={"color": color, "alpha": 0.6, "linewidth": 0.7},
|
|
381
|
+
capprops={"color": color, "alpha": 0.6, "linewidth": 0.7},
|
|
382
|
+
medianprops={"color": color, "linewidth": 0.6, "alpha": 0.9},
|
|
383
|
+
flierprops={
|
|
384
|
+
"marker": "o",
|
|
385
|
+
"markersize": min(6, box_width * (0.4 - (0.2 * many_boxes_factor))),
|
|
386
|
+
"alpha": 0.7,
|
|
387
|
+
"markeredgewidth": 0.3 + (0.4 * (1 - many_boxes_factor)),
|
|
388
|
+
"markeredgecolor": color,
|
|
389
|
+
"markerfacecolor": "none",
|
|
390
|
+
},
|
|
391
|
+
)
|
|
392
|
+
plt.setp(bp["fliers"], zorder=1.5) # Put fliers behind other elements
|
|
393
|
+
|
|
394
|
+
# Add ensemble name text and a horizontal line at y=0
|
|
395
|
+
ax.text(
|
|
396
|
+
0.01,
|
|
397
|
+
0.85,
|
|
398
|
+
f"{ensemble_key[0]}",
|
|
399
|
+
transform=ax.transAxes,
|
|
400
|
+
ha="left",
|
|
401
|
+
va="center",
|
|
402
|
+
fontsize=9,
|
|
403
|
+
alpha=0.75,
|
|
404
|
+
)
|
|
405
|
+
ax.axhline(0.0, color="black", linewidth=0.8, alpha=0.4, zorder=0)
|
|
406
|
+
|
|
407
|
+
# Apply common styling to all axes
|
|
408
|
+
for ax in axes:
|
|
409
|
+
ax.spines["top"].set_visible(False)
|
|
410
|
+
ax.spines["right"].set_visible(False)
|
|
411
|
+
ax.spines["left"].set_visible(True)
|
|
412
|
+
ax.tick_params(axis="y", labelsize=10, width=0.5, length=3)
|
|
413
|
+
ax.grid(True, axis="y", linestyle=":", linewidth=0.5, alpha=0.75)
|
|
414
|
+
self._fade_axis_ticklabels(ax)
|
|
415
|
+
|
|
416
|
+
# Hide the x-axis on all but the last plot
|
|
417
|
+
for ax in axes[:-1]:
|
|
418
|
+
ax.spines["bottom"].set_visible(False)
|
|
419
|
+
ax.tick_params(axis="x", which="both", bottom=False, labelbottom=False)
|
|
420
|
+
|
|
421
|
+
# Style the x-axis only on the last plot
|
|
422
|
+
bottom_ax = axes[-1]
|
|
423
|
+
bottom_ax.spines["bottom"].set_visible(True)
|
|
424
|
+
bottom_ax.xaxis.set_major_locator(mdates.AutoDateLocator()) # type: ignore[no-untyped-call]
|
|
425
|
+
bottom_ax.xaxis.set_major_formatter(mdates.DateFormatter("%Y-%m-%d")) # type: ignore[no-untyped-call]
|
|
426
|
+
|
|
427
|
+
bottom_ax.tick_params(axis="x", labelsize=8, width=0.5, length=3)
|
|
428
|
+
plt.setp(bottom_ax.get_xticklabels(), rotation=25, ha="right")
|
|
429
|
+
|
|
430
|
+
figure.suptitle(
|
|
431
|
+
f"{plot_context.key()} (Signed Chi-squared misfits over time)",
|
|
432
|
+
fontsize=14,
|
|
433
|
+
y=0.97,
|
|
434
|
+
alpha=0.9,
|
|
435
|
+
)
|
|
436
|
+
figure.tight_layout(rect=(0.02, 0.02, 0.98, 0.92))
|
|
@@ -27,7 +27,10 @@ def plotObservations(
|
|
|
27
27
|
|
|
28
28
|
|
|
29
29
|
def _plotObservations(
|
|
30
|
-
axes: Axes,
|
|
30
|
+
axes: Axes,
|
|
31
|
+
plot_config: PlotConfig,
|
|
32
|
+
data: DataFrame,
|
|
33
|
+
value_column: str,
|
|
31
34
|
) -> None:
|
|
32
35
|
"""
|
|
33
36
|
Observations are always plotted on top. z-order set to 1000
|
|
@@ -46,11 +49,22 @@ def _plotObservations(
|
|
|
46
49
|
# line style set to 'off' toggles errorbar visibility
|
|
47
50
|
if not style.line_style:
|
|
48
51
|
style.width = 0
|
|
52
|
+
if plot_config.depth_y_axis:
|
|
53
|
+
errorbar_data = {
|
|
54
|
+
"x": data.loc["OBS"].to_numpy(),
|
|
55
|
+
"y": data.loc["key_index"].to_numpy(),
|
|
56
|
+
"xerr": data.loc["STD"].to_numpy(),
|
|
57
|
+
}
|
|
58
|
+
axes.yaxis.set_inverted(True)
|
|
59
|
+
else:
|
|
60
|
+
errorbar_data = {
|
|
61
|
+
"x": data.loc["key_index"].to_numpy(),
|
|
62
|
+
"y": data.loc["OBS"].to_numpy(),
|
|
63
|
+
"yerr": data.loc["STD"].to_numpy(),
|
|
64
|
+
}
|
|
49
65
|
|
|
50
66
|
axes.errorbar(
|
|
51
|
-
|
|
52
|
-
y=data.loc["OBS"].values,
|
|
53
|
-
yerr=data.loc["STD"].values,
|
|
67
|
+
**errorbar_data,
|
|
54
68
|
fmt=style.line_style,
|
|
55
69
|
ecolor=style.color,
|
|
56
70
|
color=style.color,
|
|
@@ -18,12 +18,13 @@ if TYPE_CHECKING:
|
|
|
18
18
|
from matplotlib.axes import Axes
|
|
19
19
|
from matplotlib.figure import Figure
|
|
20
20
|
|
|
21
|
-
from ert.gui.tools.plot.plot_api import EnsembleObject
|
|
21
|
+
from ert.gui.tools.plot.plot_api import EnsembleObject, PlotApiKeyDefinition
|
|
22
22
|
|
|
23
23
|
|
|
24
24
|
class StatisticsPlot:
|
|
25
25
|
def __init__(self) -> None:
|
|
26
26
|
self.dimensionality = 2
|
|
27
|
+
self.requires_observations = False
|
|
27
28
|
|
|
28
29
|
@staticmethod
|
|
29
30
|
def plot(
|
|
@@ -32,6 +33,7 @@ class StatisticsPlot:
|
|
|
32
33
|
ensemble_to_data_map: dict[EnsembleObject, DataFrame],
|
|
33
34
|
observation_data: DataFrame,
|
|
34
35
|
std_dev_images: dict[str, npt.NDArray[np.float32]],
|
|
36
|
+
key_def: PlotApiKeyDefinition | None = None,
|
|
35
37
|
) -> None:
|
|
36
38
|
config = plot_context.plotConfig()
|
|
37
39
|
axes = figure.add_subplot(111)
|
|
@@ -10,13 +10,14 @@ from matplotlib.figure import Figure
|
|
|
10
10
|
from mpl_toolkits.axes_grid1 import make_axes_locatable
|
|
11
11
|
|
|
12
12
|
if TYPE_CHECKING:
|
|
13
|
-
from ert.gui.tools.plot.plot_api import EnsembleObject
|
|
13
|
+
from ert.gui.tools.plot.plot_api import EnsembleObject, PlotApiKeyDefinition
|
|
14
14
|
from ert.gui.tools.plot.plottery import PlotContext
|
|
15
15
|
|
|
16
16
|
|
|
17
17
|
class StdDevPlot:
|
|
18
18
|
def __init__(self) -> None:
|
|
19
19
|
self.dimensionality = 3
|
|
20
|
+
self.requires_observations = False
|
|
20
21
|
|
|
21
22
|
def plot(
|
|
22
23
|
self,
|
|
@@ -25,6 +26,7 @@ class StdDevPlot:
|
|
|
25
26
|
ensemble_to_data_map: dict[EnsembleObject, pd.DataFrame],
|
|
26
27
|
observation_data: pd.DataFrame,
|
|
27
28
|
std_dev_data: dict[str, npt.NDArray[np.float32]],
|
|
29
|
+
key_def: PlotApiKeyDefinition | None = None,
|
|
28
30
|
) -> None:
|
|
29
31
|
ensemble_count = len(plot_context.ensembles())
|
|
30
32
|
layer = plot_context.layer
|
|
@@ -111,7 +111,7 @@ class CSVExportJob(ErtScript):
|
|
|
111
111
|
summary_data = pl.DataFrame({})
|
|
112
112
|
|
|
113
113
|
if not summary_data.is_empty():
|
|
114
|
-
pivoted_summary = summary_data.pivot(
|
|
114
|
+
pivoted_summary = summary_data.pivot( # noqa: PD010
|
|
115
115
|
index=["realization", "time"], on="response_key", values="values"
|
|
116
116
|
).to_pandas()
|
|
117
117
|
|
|
@@ -123,8 +123,7 @@ class CSVExportJob(ErtScript):
|
|
|
123
123
|
# Reset index to make 'Realization' a regular column
|
|
124
124
|
ensemble_data = ensemble_data.reset_index()
|
|
125
125
|
|
|
126
|
-
ensemble_data =
|
|
127
|
-
ensemble_data,
|
|
126
|
+
ensemble_data = ensemble_data.merge(
|
|
128
127
|
pivoted_summary,
|
|
129
128
|
on="Realization",
|
|
130
129
|
how="left",
|
ert/plugins/plugin_manager.py
CHANGED
|
@@ -338,6 +338,7 @@ class ErtRuntimePlugins(BaseModel):
|
|
|
338
338
|
environment_variables: Mapping[str, str] = Field(default_factory=dict)
|
|
339
339
|
env_pr_fm_step: Mapping[str, Mapping[str, Any]] = Field(default_factory=dict)
|
|
340
340
|
help_links: dict[str, str] = Field(default_factory=dict)
|
|
341
|
+
prioritize_private_ip_address: bool = False
|
|
341
342
|
|
|
342
343
|
|
|
343
344
|
def get_site_plugins(
|
|
@@ -386,6 +387,9 @@ def get_site_plugins(
|
|
|
386
387
|
),
|
|
387
388
|
env_pr_fm_step=plugin_manager.get_forward_model_configuration(),
|
|
388
389
|
help_links=plugin_manager.get_help_links(),
|
|
390
|
+
prioritize_private_ip_address=site_configurations.prioritize_private_ip_address
|
|
391
|
+
if site_configurations
|
|
392
|
+
else False,
|
|
389
393
|
)
|
|
390
394
|
|
|
391
395
|
return runtime_plugins
|
|
@@ -8,10 +8,9 @@ import subprocess
|
|
|
8
8
|
import sys
|
|
9
9
|
import time
|
|
10
10
|
from argparse import ArgumentParser
|
|
11
|
-
from collections import namedtuple
|
|
12
11
|
from pathlib import Path
|
|
13
12
|
from random import random
|
|
14
|
-
from typing import Literal, get_args
|
|
13
|
+
from typing import Literal, NamedTuple, get_args
|
|
15
14
|
|
|
16
15
|
import resfo
|
|
17
16
|
|
|
@@ -43,7 +42,13 @@ class EclError(RuntimeError):
|
|
|
43
42
|
|
|
44
43
|
|
|
45
44
|
Simulators = Literal["flow", "eclipse", "e300"]
|
|
46
|
-
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class EclipseResult(NamedTuple):
|
|
48
|
+
errors: int
|
|
49
|
+
bugs: int
|
|
50
|
+
|
|
51
|
+
|
|
47
52
|
body_sub_pattern = r"(\s^\s@.+$)*"
|
|
48
53
|
date_sub_pattern = r"\s+AT TIME\s+(?P<Days>\d+\.\d+)\s+DAYS\s+\((?P<Date>(.+)):\s*$"
|
|
49
54
|
error_pattern_e100 = (
|
|
@@ -16,7 +16,7 @@ import orjson
|
|
|
16
16
|
|
|
17
17
|
from _ert.utils import file_safe_timestamp
|
|
18
18
|
from ert.config import (
|
|
19
|
-
|
|
19
|
+
EverestControl,
|
|
20
20
|
Field,
|
|
21
21
|
ForwardModelStep,
|
|
22
22
|
GenKwConfig,
|
|
@@ -134,7 +134,7 @@ def _generate_parameter_files(
|
|
|
134
134
|
scalar_data: dict[str, float | str] = {}
|
|
135
135
|
if keys:
|
|
136
136
|
start_time = time.perf_counter()
|
|
137
|
-
df = fs.
|
|
137
|
+
df = fs.load_scalar_keys(keys=keys, realizations=iens, transformed=True)
|
|
138
138
|
scalar_data = df.to_dicts()[0]
|
|
139
139
|
export_timings["load_scalar_keys"] = time.perf_counter() - start_time
|
|
140
140
|
exports: dict[str, dict[str, float | str]] = {}
|
|
@@ -190,7 +190,7 @@ def _manifest_to_json(ensemble: Ensemble, iens: int, iter_: int) -> dict[str, An
|
|
|
190
190
|
for param_config in ensemble.experiment.parameter_configuration.values():
|
|
191
191
|
assert isinstance(
|
|
192
192
|
param_config,
|
|
193
|
-
|
|
193
|
+
EverestControl | GenKwConfig | Field | SurfaceConfig,
|
|
194
194
|
)
|
|
195
195
|
if param_config.forward_init and ensemble.iteration == 0:
|
|
196
196
|
assert not isinstance(param_config, GenKwConfig)
|
|
@@ -31,8 +31,8 @@ from typing_extensions import TypedDict
|
|
|
31
31
|
|
|
32
32
|
from ert.config import (
|
|
33
33
|
EverestConstraintsConfig,
|
|
34
|
+
EverestControl,
|
|
34
35
|
EverestObjectivesConfig,
|
|
35
|
-
ExtParamConfig,
|
|
36
36
|
GenDataConfig,
|
|
37
37
|
HookRuntime,
|
|
38
38
|
KnownQueueOptionsAdapter,
|
|
@@ -535,19 +535,19 @@ class EverestRunModel(RunModel, EverestRunModelConfig):
|
|
|
535
535
|
)
|
|
536
536
|
|
|
537
537
|
@property
|
|
538
|
-
def
|
|
539
|
-
|
|
538
|
+
def _everest_control_configs(self) -> list[EverestControl]:
|
|
539
|
+
controls = [
|
|
540
540
|
c for c in self.parameter_configuration if c.type == "everest_parameters"
|
|
541
541
|
]
|
|
542
542
|
|
|
543
|
-
# There will and must always be one
|
|
543
|
+
# There will and must always be one EverestControl config for an
|
|
544
544
|
# Everest optimization.
|
|
545
|
-
return cast(list[
|
|
545
|
+
return cast(list[EverestControl], controls)
|
|
546
546
|
|
|
547
547
|
@cached_property
|
|
548
548
|
def _transforms(self) -> EverestOptModelTransforms:
|
|
549
549
|
return get_optimization_domain_transforms(
|
|
550
|
-
self.
|
|
550
|
+
self._everest_control_configs,
|
|
551
551
|
self.objectives_config,
|
|
552
552
|
self.input_constraints,
|
|
553
553
|
self.output_constraints_config,
|
|
@@ -693,7 +693,9 @@ class EverestRunModel(RunModel, EverestRunModelConfig):
|
|
|
693
693
|
)
|
|
694
694
|
|
|
695
695
|
formatted_control_names = [
|
|
696
|
-
name
|
|
696
|
+
name
|
|
697
|
+
for config in self._everest_control_configs
|
|
698
|
+
for name in config.input_keys
|
|
697
699
|
]
|
|
698
700
|
self._ever_storage.init(
|
|
699
701
|
formatted_control_names=formatted_control_names,
|
|
@@ -762,7 +764,7 @@ class EverestRunModel(RunModel, EverestRunModelConfig):
|
|
|
762
764
|
|
|
763
765
|
def _create_optimizer(self) -> tuple[BasicOptimizer, list[float]]:
|
|
764
766
|
enopt_config, initial_guesses = everest2ropt(
|
|
765
|
-
cast(list[
|
|
767
|
+
cast(list[EverestControl], self.parameter_configuration),
|
|
766
768
|
self.objectives_config,
|
|
767
769
|
self.input_constraints,
|
|
768
770
|
self.output_constraints_config,
|
|
@@ -832,13 +834,13 @@ class EverestRunModel(RunModel, EverestRunModelConfig):
|
|
|
832
834
|
for sim_id in range(sim_to_control_vector.shape[0]):
|
|
833
835
|
sim_controls = sim_to_control_vector[sim_id]
|
|
834
836
|
offset = 0
|
|
835
|
-
for
|
|
836
|
-
n_param_keys = len(
|
|
837
|
+
for control_config in self._everest_control_configs:
|
|
838
|
+
n_param_keys = len(control_config.parameter_keys)
|
|
837
839
|
|
|
838
840
|
# Save controls to ensemble
|
|
839
841
|
ensemble.save_parameters_numpy(
|
|
840
842
|
sim_controls[offset : (offset + n_param_keys)].reshape(-1, 1),
|
|
841
|
-
|
|
843
|
+
control_config.name,
|
|
842
844
|
np.array([sim_id]),
|
|
843
845
|
)
|
|
844
846
|
offset += n_param_keys
|