wrfrun 0.2.0__py3-none-any.whl → 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- wrfrun/__init__.py +8 -3
- wrfrun/cli.py +69 -29
- wrfrun/core/__init__.py +27 -10
- wrfrun/core/_config.py +308 -0
- wrfrun/core/_constant.py +236 -0
- wrfrun/core/_exec_db.py +105 -0
- wrfrun/core/_namelist.py +287 -0
- wrfrun/core/_record.py +178 -0
- wrfrun/core/_resource.py +172 -0
- wrfrun/core/base.py +132 -406
- wrfrun/core/core.py +196 -0
- wrfrun/core/error.py +28 -2
- wrfrun/core/replay.py +10 -96
- wrfrun/core/server.py +52 -27
- wrfrun/core/type.py +171 -0
- wrfrun/data.py +304 -139
- wrfrun/extension/goos_sst/__init__.py +2 -2
- wrfrun/extension/goos_sst/core.py +9 -14
- wrfrun/extension/goos_sst/res/__init__.py +0 -1
- wrfrun/extension/goos_sst/utils.py +50 -44
- wrfrun/extension/littler/core.py +105 -88
- wrfrun/extension/utils.py +4 -3
- wrfrun/log.py +117 -0
- wrfrun/model/__init__.py +11 -7
- wrfrun/model/constants.py +52 -0
- wrfrun/model/palm/__init__.py +30 -0
- wrfrun/model/palm/core.py +145 -0
- wrfrun/model/palm/namelist.py +33 -0
- wrfrun/model/plot.py +99 -119
- wrfrun/model/type.py +116 -0
- wrfrun/model/utils.py +9 -20
- wrfrun/model/wrf/__init__.py +4 -9
- wrfrun/model/wrf/core.py +246 -161
- wrfrun/model/wrf/exec_wrap.py +13 -12
- wrfrun/model/wrf/geodata.py +116 -100
- wrfrun/model/wrf/log.py +103 -0
- wrfrun/model/wrf/namelist.py +90 -73
- wrfrun/model/wrf/plot.py +102 -0
- wrfrun/model/wrf/scheme.py +108 -52
- wrfrun/model/wrf/utils.py +39 -25
- wrfrun/model/wrf/vtable.py +35 -3
- wrfrun/plot/__init__.py +20 -0
- wrfrun/plot/wps.py +90 -73
- wrfrun/res/__init__.py +103 -5
- wrfrun/res/config/config.template.toml +8 -0
- wrfrun/res/config/palm.template.toml +23 -0
- wrfrun/run.py +105 -77
- wrfrun/scheduler/__init__.py +1 -0
- wrfrun/scheduler/lsf.py +3 -2
- wrfrun/scheduler/pbs.py +3 -2
- wrfrun/scheduler/script.py +17 -5
- wrfrun/scheduler/slurm.py +3 -2
- wrfrun/scheduler/utils.py +14 -2
- wrfrun/utils.py +88 -199
- wrfrun/workspace/__init__.py +8 -5
- wrfrun/workspace/core.py +20 -12
- wrfrun/workspace/palm.py +137 -0
- wrfrun/workspace/wrf.py +16 -15
- wrfrun-0.3.0.dist-info/METADATA +240 -0
- wrfrun-0.3.0.dist-info/RECORD +78 -0
- wrfrun/core/config.py +0 -923
- wrfrun/model/base.py +0 -14
- wrfrun-0.2.0.dist-info/METADATA +0 -68
- wrfrun-0.2.0.dist-info/RECORD +0 -62
- {wrfrun-0.2.0.dist-info → wrfrun-0.3.0.dist-info}/WHEEL +0 -0
- {wrfrun-0.2.0.dist-info → wrfrun-0.3.0.dist-info}/entry_points.txt +0 -0
wrfrun/core/base.py
CHANGED
|
@@ -2,23 +2,27 @@
|
|
|
2
2
|
wrfrun.core.base
|
|
3
3
|
################
|
|
4
4
|
|
|
5
|
-
Defines what :class:`ExecutableBase <Executable>` is, how it works and how ``wrfrun`` records simulations.
|
|
6
|
-
|
|
7
5
|
.. autosummary::
|
|
8
6
|
:toctree: generated/
|
|
9
7
|
|
|
10
8
|
check_subprocess_status
|
|
11
9
|
call_subprocess
|
|
12
|
-
InputFileType
|
|
13
|
-
FileConfigDict
|
|
14
|
-
ExecutableClassConfig
|
|
15
|
-
ExecutableConfig
|
|
16
|
-
ExecutableConfigRecord
|
|
17
|
-
create_recorder
|
|
18
10
|
ExecutableBase
|
|
19
11
|
|
|
20
|
-
|
|
21
|
-
|
|
12
|
+
ExecutableBase
|
|
13
|
+
**************
|
|
14
|
+
|
|
15
|
+
What is the ``Executable`` in wrfrun?
|
|
16
|
+
=====================================
|
|
17
|
+
|
|
18
|
+
``Executable`` is the part to interact with external resources (for example, external programs, datasets, etc.).
|
|
19
|
+
They are classes that implement some essential methods,
|
|
20
|
+
controlled by wrfrun to achieve the goal: managing numerical simulations.
|
|
21
|
+
|
|
22
|
+
In wrfrun, all the ``Executable`` is the subclass of :class:`ExecutableBase`.
|
|
23
|
+
|
|
24
|
+
Why wrfrun defines ``ExecutableBase``?
|
|
25
|
+
======================================
|
|
22
26
|
|
|
23
27
|
While ``wrfrun`` aims to provide Python interfaces to various Numerical Weather Prediction model,
|
|
24
28
|
it is important to provide a clear standard about how should an external executable file be implemented in ``wrfrun``.
|
|
@@ -28,41 +32,35 @@ but also:
|
|
|
28
32
|
|
|
29
33
|
* Store all the information about the program (e.g., its inputs and outputs, its configuration).
|
|
30
34
|
* Provide the interface to import and export ``Executable``'s config.
|
|
31
|
-
* Support the ``replay`` functionality of ``wrfrun``.
|
|
32
35
|
|
|
33
|
-
|
|
36
|
+
With all the ``Executable`` have the same interface, wrfrun could provide some really cool features, like:
|
|
34
37
|
|
|
35
|
-
|
|
36
|
-
|
|
38
|
+
* Record the whole simulation, generate a record file (I call it the replay file).
|
|
39
|
+
* Replay the whole simulation based on the replay file.
|
|
37
40
|
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
``ExecConfigRecorder`` can store the config of any ``Executable`` in the correct order and export them to a JSON file,
|
|
41
|
-
enabling the precise replay of the simulation. Depending on the internal implementation of the ``Executable``,
|
|
42
|
-
the recorded config may even include the complete namelist values, Vtable config, and more.
|
|
43
|
-
Please check :meth:`ExecutableBase.generate_custom_config` for more details.
|
|
41
|
+
If you want to improve wrfrun's function to interact with external resources,
|
|
42
|
+
I strongly recommend you to implement your code by inheriting :class:`ExecutableBase`.
|
|
44
43
|
"""
|
|
45
44
|
|
|
46
45
|
import subprocess
|
|
47
46
|
from copy import deepcopy
|
|
48
|
-
from enum import Enum
|
|
49
|
-
from json import dumps
|
|
50
47
|
from os import chdir, getcwd, listdir, makedirs, remove, symlink
|
|
51
|
-
from os.path import abspath, basename,
|
|
52
|
-
from shutil import
|
|
53
|
-
from typing import Optional,
|
|
48
|
+
from os.path import abspath, basename, exists
|
|
49
|
+
from shutil import move
|
|
50
|
+
from typing import Optional, Union
|
|
54
51
|
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
from .
|
|
58
|
-
from .
|
|
59
|
-
from ..utils import check_path, logger
|
|
52
|
+
from ..log import logger
|
|
53
|
+
from .core import WRFRUN
|
|
54
|
+
from .error import CommandError, ConfigError, OutputFileError
|
|
55
|
+
from .type import ExecutableClassConfig, ExecutableConfig, FileConfigDict
|
|
60
56
|
|
|
61
57
|
|
|
62
58
|
def check_subprocess_status(status: subprocess.CompletedProcess):
|
|
63
59
|
"""
|
|
64
60
|
Check subprocess return code.
|
|
65
|
-
|
|
61
|
+
|
|
62
|
+
An ``RuntimeError`` exception will be raised if ``return_code != 0``,
|
|
63
|
+
and the ``stdout`` and ``stderr`` of the subprocess will be logged.
|
|
66
64
|
|
|
67
65
|
:param status: Status from subprocess.
|
|
68
66
|
:type status: CompletedProcess
|
|
@@ -102,7 +100,7 @@ def call_subprocess(command: list[str], work_path: Optional[str] = None, print_o
|
|
|
102
100
|
else:
|
|
103
101
|
origin_path = None
|
|
104
102
|
|
|
105
|
-
status = subprocess.run(
|
|
103
|
+
status = subprocess.run(" ".join(command), shell=True, capture_output=True)
|
|
106
104
|
|
|
107
105
|
if origin_path is not None:
|
|
108
106
|
chdir(origin_path)
|
|
@@ -114,339 +112,10 @@ def call_subprocess(command: list[str], work_path: Optional[str] = None, print_o
|
|
|
114
112
|
logger.warning(status.stderr.decode())
|
|
115
113
|
|
|
116
114
|
|
|
117
|
-
def _json_default(obj):
|
|
118
|
-
"""
|
|
119
|
-
Used for json.dumps.
|
|
120
|
-
|
|
121
|
-
:param obj:
|
|
122
|
-
:type obj:
|
|
123
|
-
:return:
|
|
124
|
-
:rtype:
|
|
125
|
-
"""
|
|
126
|
-
if isinstance(obj, np.integer):
|
|
127
|
-
return int(obj)
|
|
128
|
-
elif isinstance(obj, np.floating):
|
|
129
|
-
return float(obj)
|
|
130
|
-
else:
|
|
131
|
-
raise TypeError(f"Object of type {type(obj)} is not JSON serializable.")
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
class InputFileType(Enum):
|
|
135
|
-
"""
|
|
136
|
-
This class is an ``Enum`` class, providing the following values:
|
|
137
|
-
|
|
138
|
-
.. py:attribute:: WRFRUN_RES
|
|
139
|
-
:type: int
|
|
140
|
-
:value: 1
|
|
141
|
-
|
|
142
|
-
Indicating resource files are from the model or ``wrfrun``.
|
|
143
|
-
``wrfrun`` won't save these files when recording the simulation.
|
|
144
|
-
|
|
145
|
-
.. py:attribute:: CUSTOM_RES
|
|
146
|
-
:type: int
|
|
147
|
-
:value: 2
|
|
148
|
-
|
|
149
|
-
Indicating resource files are provided by the user.
|
|
150
|
-
``wrfrun`` will also save these files to ``.replay`` file when recording the simulation to ensure the simulation is replayable.
|
|
151
|
-
"""
|
|
152
|
-
WRFRUN_RES = 1
|
|
153
|
-
CUSTOM_RES = 2
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
class FileConfigDict(TypedDict):
|
|
157
|
-
"""
|
|
158
|
-
This dict is used to store information about the file, including its path,
|
|
159
|
-
the path it will be copied or moved to, its new name, etc.
|
|
160
|
-
This dict contains following keys:
|
|
161
|
-
|
|
162
|
-
.. py:attribute:: file_path
|
|
163
|
-
:type: str
|
|
164
|
-
|
|
165
|
-
A real file path or a valid URI which can be converted to a file path.
|
|
166
|
-
|
|
167
|
-
.. py:attribute:: save_path
|
|
168
|
-
:type: str
|
|
169
|
-
|
|
170
|
-
Save path of the file.
|
|
171
|
-
|
|
172
|
-
.. py:attribute:: save_name
|
|
173
|
-
:type: str
|
|
174
|
-
|
|
175
|
-
Save name of the file.
|
|
176
|
-
|
|
177
|
-
.. py:attribute:: is_data
|
|
178
|
-
:type: bool
|
|
179
|
-
|
|
180
|
-
If the file is data. If not, ``wrfrun`` will treat it as a config file,
|
|
181
|
-
and always save it to ``.replay`` file when recording the simulation.
|
|
182
|
-
|
|
183
|
-
.. py:attribute:: is_output
|
|
184
|
-
:type: bool
|
|
185
|
-
|
|
186
|
-
If the file is model's output. Output file will never be saved to ``.replay`` file.
|
|
187
|
-
"""
|
|
188
|
-
file_path: str
|
|
189
|
-
save_path: str
|
|
190
|
-
save_name: str
|
|
191
|
-
is_data: bool
|
|
192
|
-
is_output: bool
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
class ExecutableClassConfig(TypedDict):
|
|
196
|
-
"""
|
|
197
|
-
This dict is used to store arguments of ``Executable``'s ``__init__`` function.
|
|
198
|
-
|
|
199
|
-
.. py:attribute:: class_args
|
|
200
|
-
:type: tuple
|
|
201
|
-
|
|
202
|
-
Positional arguments of the class.
|
|
203
|
-
|
|
204
|
-
.. py:attribute:: class_kwargs
|
|
205
|
-
:type: dict
|
|
206
|
-
|
|
207
|
-
Keyword arguments of the class.
|
|
208
|
-
"""
|
|
209
|
-
# only list essential config
|
|
210
|
-
class_args: tuple
|
|
211
|
-
class_kwargs: dict
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
class ExecutableConfig(TypedDict):
|
|
215
|
-
"""
|
|
216
|
-
This dict is used to store all configs of a :class:`ExecutableBase`.
|
|
217
|
-
|
|
218
|
-
.. py:attribute:: name
|
|
219
|
-
:type: str
|
|
220
|
-
|
|
221
|
-
Name of the executable. Each type of executable has a unique name.
|
|
222
|
-
|
|
223
|
-
.. py:attribute:: cmd
|
|
224
|
-
:type: str | list[str]
|
|
225
|
-
|
|
226
|
-
Command of the executable.
|
|
227
|
-
|
|
228
|
-
.. py:attribute:: work_path
|
|
229
|
-
:type: str | None
|
|
230
|
-
|
|
231
|
-
Work path of the executable.
|
|
232
|
-
|
|
233
|
-
.. py:attribute:: mpi_use
|
|
234
|
-
:type: bool
|
|
235
|
-
|
|
236
|
-
If the executable will use MPI.
|
|
237
|
-
|
|
238
|
-
.. py:attribute:: mpi_cmd
|
|
239
|
-
:type: str | None
|
|
240
|
-
|
|
241
|
-
Command name of the MPI.
|
|
242
|
-
|
|
243
|
-
.. py:attribute:: mpi_core_num
|
|
244
|
-
:type: int | None
|
|
245
|
-
|
|
246
|
-
Number of the CPU core to use with MPI.
|
|
247
|
-
|
|
248
|
-
.. py:attribute:: class_config
|
|
249
|
-
:type: ExecutableClassConfig | None
|
|
250
|
-
|
|
251
|
-
A dict stores arguments of ``Executable``'s ``__init__`` function.
|
|
252
|
-
|
|
253
|
-
.. py:attribute:: input_file_config
|
|
254
|
-
:type: list[FileConfigDict] | None
|
|
255
|
-
|
|
256
|
-
A list stores information about input files of the executable.
|
|
257
|
-
|
|
258
|
-
.. py:attribute:: output_file_config
|
|
259
|
-
:type: list[FileConfigDict] | None
|
|
260
|
-
|
|
261
|
-
A list stores information about output files of the executable.
|
|
262
|
-
|
|
263
|
-
.. py:attribute:: custom_config
|
|
264
|
-
:type: dict | None
|
|
265
|
-
|
|
266
|
-
A dict that can be used by subclass to store other configs.
|
|
267
|
-
"""
|
|
268
|
-
name: str
|
|
269
|
-
cmd: Union[str, list[str]]
|
|
270
|
-
work_path: Optional[str]
|
|
271
|
-
mpi_use: bool
|
|
272
|
-
mpi_cmd: Optional[str]
|
|
273
|
-
mpi_core_num: Optional[int]
|
|
274
|
-
class_config: Optional[ExecutableClassConfig]
|
|
275
|
-
input_file_config: Optional[list[FileConfigDict]]
|
|
276
|
-
output_file_config: Optional[list[FileConfigDict]]
|
|
277
|
-
custom_config: Optional[dict]
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
class ExecutableConfigRecord:
|
|
281
|
-
"""
|
|
282
|
-
A class to helps store configs of various executables and exports them to a file.
|
|
283
|
-
"""
|
|
284
|
-
_instance = None
|
|
285
|
-
_initialized = False
|
|
286
|
-
|
|
287
|
-
def __init__(self, save_path: str, include_data=False):
|
|
288
|
-
"""
|
|
289
|
-
|
|
290
|
-
:param save_path: Save path of the exported config file.
|
|
291
|
-
:type save_path: str | None
|
|
292
|
-
:param include_data: If includes input data.
|
|
293
|
-
:type include_data: bool
|
|
294
|
-
"""
|
|
295
|
-
|
|
296
|
-
if self._initialized:
|
|
297
|
-
return
|
|
298
|
-
|
|
299
|
-
if save_path is None:
|
|
300
|
-
WRFRUNConfig.IS_RECORDING = False
|
|
301
|
-
else:
|
|
302
|
-
WRFRUNConfig.IS_RECORDING = True
|
|
303
|
-
|
|
304
|
-
self.save_path = save_path
|
|
305
|
-
self.include_data = include_data
|
|
306
|
-
|
|
307
|
-
self.work_path = WRFRUNConfig.parse_resource_uri(WRFRUNConfig.WRFRUN_WORKSPACE_REPLAY)
|
|
308
|
-
self.content_path = f"{self.work_path}/config_and_data"
|
|
309
|
-
|
|
310
|
-
self._recorded_config = []
|
|
311
|
-
self._name_count = {}
|
|
312
|
-
|
|
313
|
-
self._initialized = True
|
|
314
|
-
|
|
315
|
-
def __new__(cls, *args, **kwargs):
|
|
316
|
-
if cls._instance is None:
|
|
317
|
-
cls._instance = super().__new__(cls)
|
|
318
|
-
|
|
319
|
-
return cls._instance
|
|
320
|
-
|
|
321
|
-
@classmethod
|
|
322
|
-
def reinit(cls, save_path: str, include_data=False):
|
|
323
|
-
"""
|
|
324
|
-
Reinitialize this instance.
|
|
325
|
-
|
|
326
|
-
:param save_path: Save path of the exported config file.
|
|
327
|
-
:type save_path: str
|
|
328
|
-
:param include_data: If includes input data.
|
|
329
|
-
:type include_data: bool
|
|
330
|
-
:return: New instance.
|
|
331
|
-
:rtype: ExecutableConfigRecord
|
|
332
|
-
"""
|
|
333
|
-
cls._initialized = False
|
|
334
|
-
return cls(save_path, include_data)
|
|
335
|
-
|
|
336
|
-
def record(self, exported_config: ExecutableConfig):
|
|
337
|
-
"""
|
|
338
|
-
Record exported config for replay.
|
|
339
|
-
|
|
340
|
-
:param exported_config: Executable config.
|
|
341
|
-
:type exported_config: ExecutableConfig
|
|
342
|
-
"""
|
|
343
|
-
if not self.include_data:
|
|
344
|
-
self._recorded_config.append(exported_config)
|
|
345
|
-
return
|
|
346
|
-
|
|
347
|
-
check_path(self.content_path)
|
|
348
|
-
|
|
349
|
-
# process exported config so we can also include data.
|
|
350
|
-
# create directory to place data
|
|
351
|
-
name = exported_config["name"]
|
|
352
|
-
if name in self._name_count:
|
|
353
|
-
self._name_count[name] += 1
|
|
354
|
-
index = self._name_count[name]
|
|
355
|
-
else:
|
|
356
|
-
self._name_count[name] = 1
|
|
357
|
-
index = 1
|
|
358
|
-
|
|
359
|
-
data_save_uri = f"{WRFRUNConfig.WRFRUN_WORKSPACE_REPLAY}/{name}/{index}"
|
|
360
|
-
data_save_path = f"{self.content_path}/{name}/{index}"
|
|
361
|
-
makedirs(data_save_path)
|
|
362
|
-
|
|
363
|
-
input_file_config = exported_config["input_file_config"]
|
|
364
|
-
|
|
365
|
-
for _config_index, _config in enumerate(input_file_config):
|
|
366
|
-
if not _config["is_data"]:
|
|
367
|
-
continue
|
|
368
|
-
|
|
369
|
-
if _config["is_output"]:
|
|
370
|
-
continue
|
|
371
|
-
|
|
372
|
-
file_path = _config["file_path"]
|
|
373
|
-
file_path = WRFRUNConfig.parse_resource_uri(file_path)
|
|
374
|
-
filename = basename(file_path)
|
|
375
|
-
copyfile(file_path, f"{data_save_path}/{filename}")
|
|
376
|
-
|
|
377
|
-
_config["file_path"] = f"{data_save_uri}/{filename}"
|
|
378
|
-
input_file_config[_config_index] = _config
|
|
379
|
-
|
|
380
|
-
exported_config["input_file_config"] = input_file_config
|
|
381
|
-
self._recorded_config.append(exported_config)
|
|
382
|
-
|
|
383
|
-
def clear_records(self):
|
|
384
|
-
"""
|
|
385
|
-
Clean recorded configs.
|
|
386
|
-
"""
|
|
387
|
-
self._recorded_config = []
|
|
388
|
-
|
|
389
|
-
def export_replay_file(self):
|
|
390
|
-
"""
|
|
391
|
-
Save replay file to the save path.
|
|
392
|
-
"""
|
|
393
|
-
if len(self._recorded_config) == 0:
|
|
394
|
-
logger.warning("No replay config has been recorded.")
|
|
395
|
-
return
|
|
396
|
-
|
|
397
|
-
logger.info("Exporting replay config... It may take a few minutes if you include data.")
|
|
398
|
-
|
|
399
|
-
check_path(self.content_path)
|
|
400
|
-
|
|
401
|
-
with open(f"{self.content_path}/config.json", "w") as f:
|
|
402
|
-
f.write(dumps(self._recorded_config, indent=4, default=_json_default))
|
|
403
|
-
|
|
404
|
-
if exists(self.save_path):
|
|
405
|
-
if isdir(self.save_path):
|
|
406
|
-
self.save_path = f"{self.save_path}/1.replay"
|
|
407
|
-
else:
|
|
408
|
-
if not self.save_path.endswith(".replay"):
|
|
409
|
-
self.save_path = f"{self.save_path}.replay"
|
|
410
|
-
|
|
411
|
-
if exists(self.save_path):
|
|
412
|
-
logger.warning(f"Found existed replay file with the same name '{basename(self.save_path)}', overwrite it")
|
|
413
|
-
remove(self.save_path)
|
|
414
|
-
|
|
415
|
-
if not exists(dirname(self.save_path)):
|
|
416
|
-
makedirs(dirname(self.save_path))
|
|
417
|
-
|
|
418
|
-
temp_file = f"{self.work_path}/config_and_data"
|
|
419
|
-
make_archive(temp_file, "zip", self.content_path)
|
|
420
|
-
move(f"{temp_file}.zip", self.save_path)
|
|
421
|
-
|
|
422
|
-
logger.info(f"Replay config exported to {self.save_path}")
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
ExecConfigRecorder: Optional[ExecutableConfigRecord] = None
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
def create_recorder(save_path: str, include_data=False) -> ExecutableConfigRecord:
|
|
429
|
-
"""
|
|
430
|
-
Create a recorder to record simulations.
|
|
431
|
-
|
|
432
|
-
:param save_path: Save path of the exported config file.
|
|
433
|
-
:type save_path: str
|
|
434
|
-
:param include_data: If includes input data.
|
|
435
|
-
:type include_data: bool
|
|
436
|
-
:return: Recorder instance.
|
|
437
|
-
:rtype: ExecutableConfigRecord
|
|
438
|
-
"""
|
|
439
|
-
global ExecConfigRecorder
|
|
440
|
-
|
|
441
|
-
ExecConfigRecorder = ExecutableConfigRecord.reinit(save_path, include_data)
|
|
442
|
-
|
|
443
|
-
return ExecConfigRecorder
|
|
444
|
-
|
|
445
|
-
|
|
446
115
|
class ExecutableBase:
|
|
447
116
|
"""
|
|
448
117
|
Base class for all executables.
|
|
449
|
-
|
|
118
|
+
|
|
450
119
|
.. py:attribute:: class_config
|
|
451
120
|
:type: ExecutableClassConfig
|
|
452
121
|
:value: {"class_args": (), "class_kwargs": {}}
|
|
@@ -472,10 +141,18 @@ class ExecutableBase:
|
|
|
472
141
|
A list stores information about output files of the executable.
|
|
473
142
|
|
|
474
143
|
"""
|
|
144
|
+
|
|
475
145
|
_instance = None
|
|
476
146
|
|
|
477
|
-
def __init__(
|
|
478
|
-
|
|
147
|
+
def __init__(
|
|
148
|
+
self,
|
|
149
|
+
name: str,
|
|
150
|
+
cmd: Union[str, list[str]],
|
|
151
|
+
work_path: str,
|
|
152
|
+
mpi_use=False,
|
|
153
|
+
mpi_cmd: Optional[str] = None,
|
|
154
|
+
mpi_core_num: Optional[int] = None,
|
|
155
|
+
):
|
|
479
156
|
"""
|
|
480
157
|
|
|
481
158
|
:param name: Unique name to identify different executables.
|
|
@@ -486,7 +163,7 @@ class ExecutableBase:
|
|
|
486
163
|
:type cmd: str
|
|
487
164
|
:param work_path: Working directory path.
|
|
488
165
|
:type work_path: str
|
|
489
|
-
:param mpi_use: If
|
|
166
|
+
:param mpi_use: If use mpi. You have to give ``mpi_cmd`` and ``mpi_core_num`` if you use mpi. Defaults to False.
|
|
490
167
|
:type mpi_use: bool
|
|
491
168
|
:param mpi_cmd: MPI command. For example, ``"mpirun"``. Defaults to None.
|
|
492
169
|
:type mpi_cmd: str
|
|
@@ -504,13 +181,17 @@ class ExecutableBase:
|
|
|
504
181
|
self.mpi_cmd = mpi_cmd
|
|
505
182
|
self.mpi_core_num = mpi_core_num
|
|
506
183
|
|
|
184
|
+
# don't use mpi if mpi_core_num = 1
|
|
185
|
+
if isinstance(self.mpi_core_num, int) and self.mpi_core_num < 2:
|
|
186
|
+
self.mpi_core_num = None
|
|
187
|
+
|
|
507
188
|
self.class_config: ExecutableClassConfig = {"class_args": (), "class_kwargs": {}}
|
|
508
189
|
self.custom_config: dict = {}
|
|
509
190
|
self.input_file_config: list[FileConfigDict] = []
|
|
510
191
|
self.output_file_config: list[FileConfigDict] = []
|
|
511
192
|
|
|
512
193
|
# directory to save outputs
|
|
513
|
-
self._output_save_path = f"{
|
|
194
|
+
self._output_save_path = f"{WRFRUN.config.WRFRUN_OUTPUT_PATH}/{self.name}"
|
|
514
195
|
self._log_save_path = f"{self._output_save_path}/logs"
|
|
515
196
|
|
|
516
197
|
def __new__(cls, *args, **kwargs):
|
|
@@ -521,7 +202,9 @@ class ExecutableBase:
|
|
|
521
202
|
|
|
522
203
|
def generate_custom_config(self):
|
|
523
204
|
"""
|
|
524
|
-
Generate custom configs.
|
|
205
|
+
Generate custom configs.
|
|
206
|
+
|
|
207
|
+
This method should be overwritten in the child class,
|
|
525
208
|
and **MUST STORE THE CUSTOM CONFIG IN THE ATTRIBUTE** :attr:`ExecutableBase.custom_config`,
|
|
526
209
|
or it will do nothing except print a debug log.
|
|
527
210
|
|
|
@@ -536,7 +219,9 @@ class ExecutableBase:
|
|
|
536
219
|
def load_custom_config(self):
|
|
537
220
|
"""
|
|
538
221
|
Load custom configs.
|
|
539
|
-
|
|
222
|
+
|
|
223
|
+
This method should be overwritten in the child class to
|
|
224
|
+
process the custom config stored in :attr:`ExecutableBase.custom_config`,
|
|
540
225
|
or it will do nothing except print a debug log.
|
|
541
226
|
"""
|
|
542
227
|
logger.debug(f"Method 'load_custom_config' not implemented in '{self.name}'")
|
|
@@ -560,7 +245,7 @@ class ExecutableBase:
|
|
|
560
245
|
"class_config": deepcopy(self.class_config),
|
|
561
246
|
"custom_config": deepcopy(self.custom_config),
|
|
562
247
|
"input_file_config": deepcopy(self.input_file_config),
|
|
563
|
-
"output_file_config": deepcopy(self.output_file_config)
|
|
248
|
+
"output_file_config": deepcopy(self.output_file_config),
|
|
564
249
|
}
|
|
565
250
|
|
|
566
251
|
def load_config(self, config: ExecutableConfig):
|
|
@@ -599,8 +284,9 @@ class ExecutableBase:
|
|
|
599
284
|
logger.debug(f"Method 'replay' not implemented in '{self.name}', fall back to default action.")
|
|
600
285
|
self()
|
|
601
286
|
|
|
602
|
-
def add_input_files(
|
|
603
|
-
|
|
287
|
+
def add_input_files(
|
|
288
|
+
self, input_files: Union[str, list[str], FileConfigDict, list[FileConfigDict]], is_data=True, is_output=True
|
|
289
|
+
):
|
|
604
290
|
"""
|
|
605
291
|
Add input files the executable will use.
|
|
606
292
|
|
|
@@ -644,14 +330,18 @@ class ExecutableBase:
|
|
|
644
330
|
:type input_files: str | list | dict
|
|
645
331
|
:param is_data: If it is a data file. This parameter will be overwritten by the value in ``input_files``.
|
|
646
332
|
:type is_data: bool
|
|
647
|
-
:param is_output: If it is an output from another executable.
|
|
333
|
+
:param is_output: If it is an output from another executable.
|
|
334
|
+
This parameter will be overwritten by the value in ``input_files``.
|
|
648
335
|
:type is_output: bool
|
|
649
336
|
"""
|
|
650
337
|
if isinstance(input_files, str):
|
|
651
338
|
self.input_file_config.append(
|
|
652
339
|
{
|
|
653
|
-
"file_path": input_files,
|
|
654
|
-
"
|
|
340
|
+
"file_path": input_files,
|
|
341
|
+
"save_path": self.work_path,
|
|
342
|
+
"save_name": basename(input_files),
|
|
343
|
+
"is_data": is_data,
|
|
344
|
+
"is_output": is_output,
|
|
655
345
|
}
|
|
656
346
|
)
|
|
657
347
|
|
|
@@ -663,8 +353,11 @@ class ExecutableBase:
|
|
|
663
353
|
elif isinstance(_file, str):
|
|
664
354
|
self.input_file_config.append(
|
|
665
355
|
{
|
|
666
|
-
"file_path": _file,
|
|
667
|
-
"
|
|
356
|
+
"file_path": _file,
|
|
357
|
+
"save_path": self.work_path,
|
|
358
|
+
"save_name": basename(_file),
|
|
359
|
+
"is_data": is_data,
|
|
360
|
+
"is_output": is_output,
|
|
668
361
|
}
|
|
669
362
|
)
|
|
670
363
|
|
|
@@ -680,8 +373,13 @@ class ExecutableBase:
|
|
|
680
373
|
raise TypeError(f"Input file config should be string or `FileConfigDict`, but got '{type(input_files)}'")
|
|
681
374
|
|
|
682
375
|
def add_output_files(
|
|
683
|
-
self,
|
|
684
|
-
|
|
376
|
+
self,
|
|
377
|
+
output_dir: Optional[str] = None,
|
|
378
|
+
save_path: Optional[str] = None,
|
|
379
|
+
startswith: Union[None, str, tuple[str, ...]] = None,
|
|
380
|
+
endswith: Union[None, str, tuple[str, ...]] = None,
|
|
381
|
+
outputs: Union[None, str, list[str]] = None,
|
|
382
|
+
no_file_error=True,
|
|
685
383
|
):
|
|
686
384
|
"""
|
|
687
385
|
Find and save model's outputs to the output save path.
|
|
@@ -692,7 +390,8 @@ class ExecutableBase:
|
|
|
692
390
|
>>> self.add_output_files(outputs="wrfout.d01")
|
|
693
391
|
>>> self.add_output_files(outputs=["wrfout.d01", "wrfout.d02"])
|
|
694
392
|
|
|
695
|
-
If you have too many outputs, but they have the same prefix or postfix,
|
|
393
|
+
If you have too many outputs, but they have the same prefix or postfix,
|
|
394
|
+
you can use ``startswith`` or ``endswith``.
|
|
696
395
|
|
|
697
396
|
>>> self.add_output_files(startswith="rsl.out.")
|
|
698
397
|
>>> self.add_output_files(endswith="log")
|
|
@@ -707,7 +406,8 @@ class ExecutableBase:
|
|
|
707
406
|
|
|
708
407
|
:param output_dir: Search path of outputs.
|
|
709
408
|
:type output_dir: str
|
|
710
|
-
:param save_path: New save path of outputs.
|
|
409
|
+
:param save_path: New save path of outputs.
|
|
410
|
+
By default, it is ``f"{WRFRUNConfig.WRFRUN_OUTPUT_PATH}/{self.name}"``.
|
|
711
411
|
:type save_path: str
|
|
712
412
|
:param startswith: Prefix string or prefix list of output files.
|
|
713
413
|
:type startswith: str | list
|
|
@@ -715,19 +415,20 @@ class ExecutableBase:
|
|
|
715
415
|
:type endswith: str | list
|
|
716
416
|
:param outputs: Files name list. All files in the list will be saved.
|
|
717
417
|
:type outputs: str | list
|
|
718
|
-
:param no_file_error: If True, an OutputFileError will be raised if no output file can be found.
|
|
418
|
+
:param no_file_error: If True, an OutputFileError will be raised if no output file can be found.
|
|
419
|
+
Defaults to True.
|
|
719
420
|
:type no_file_error: bool
|
|
720
421
|
"""
|
|
721
|
-
if
|
|
422
|
+
if WRFRUN.config.FAKE_SIMULATION_MODE:
|
|
722
423
|
return
|
|
723
424
|
|
|
724
425
|
if output_dir is None:
|
|
725
426
|
output_dir = self.work_path
|
|
726
427
|
|
|
727
428
|
if save_path is None:
|
|
728
|
-
save_path = f"{
|
|
429
|
+
save_path = f"{WRFRUN.config.WRFRUN_OUTPUT_PATH}/{self.name}"
|
|
729
430
|
|
|
730
|
-
file_list = listdir(
|
|
431
|
+
file_list = listdir(WRFRUN.config.parse_resource_uri(output_dir))
|
|
731
432
|
save_file_list = []
|
|
732
433
|
|
|
733
434
|
if startswith is not None:
|
|
@@ -757,24 +458,47 @@ class ExecutableBase:
|
|
|
757
458
|
|
|
758
459
|
if len(save_file_list) < 1:
|
|
759
460
|
if no_file_error:
|
|
760
|
-
logger.error(
|
|
761
|
-
|
|
461
|
+
logger.error(
|
|
462
|
+
(
|
|
463
|
+
"Can't find any files match the giving rules: "
|
|
464
|
+
f"startswith='{startswith}', endswith='{endswith}', outputs='{outputs}'"
|
|
465
|
+
)
|
|
466
|
+
)
|
|
467
|
+
raise OutputFileError(
|
|
468
|
+
(
|
|
469
|
+
"Can't find any files match the giving rules: "
|
|
470
|
+
f"startswith='{startswith}', endswith='{endswith}', outputs='{outputs}'"
|
|
471
|
+
)
|
|
472
|
+
)
|
|
762
473
|
|
|
763
474
|
else:
|
|
764
|
-
logger.warning(
|
|
475
|
+
logger.warning(
|
|
476
|
+
(
|
|
477
|
+
"Can't find any files match the giving rules: "
|
|
478
|
+
f"startswith='{startswith}', endswith='{endswith}', outputs='{outputs}'. Skip it."
|
|
479
|
+
)
|
|
480
|
+
)
|
|
765
481
|
return
|
|
766
482
|
|
|
767
483
|
save_file_list = list(set(save_file_list))
|
|
768
484
|
logger.debug(f"Files to be processed: {save_file_list}")
|
|
769
485
|
|
|
770
486
|
for _file in save_file_list:
|
|
771
|
-
self.output_file_config.append(
|
|
487
|
+
self.output_file_config.append(
|
|
488
|
+
{
|
|
489
|
+
"file_path": f"{output_dir}/{_file}",
|
|
490
|
+
"save_path": save_path,
|
|
491
|
+
"save_name": _file,
|
|
492
|
+
"is_data": True,
|
|
493
|
+
"is_output": True,
|
|
494
|
+
}
|
|
495
|
+
)
|
|
772
496
|
|
|
773
497
|
def before_exec(self):
|
|
774
498
|
"""
|
|
775
499
|
Prepare input files before executing the external program.
|
|
776
500
|
"""
|
|
777
|
-
if
|
|
501
|
+
if WRFRUN.config.FAKE_SIMULATION_MODE:
|
|
778
502
|
logger.info(f"We are in fake simulation mode, skip preparing input files for '{self.name}'")
|
|
779
503
|
return
|
|
780
504
|
|
|
@@ -783,8 +507,8 @@ class ExecutableBase:
|
|
|
783
507
|
save_path = input_file["save_path"]
|
|
784
508
|
save_name = input_file["save_name"]
|
|
785
509
|
|
|
786
|
-
file_path =
|
|
787
|
-
save_path =
|
|
510
|
+
file_path = WRFRUN.config.parse_resource_uri(file_path)
|
|
511
|
+
save_path = WRFRUN.config.parse_resource_uri(save_path)
|
|
788
512
|
|
|
789
513
|
file_path = abspath(file_path)
|
|
790
514
|
save_path = abspath(save_path)
|
|
@@ -807,7 +531,7 @@ class ExecutableBase:
|
|
|
807
531
|
"""
|
|
808
532
|
Save outputs and logs after executing the external program.
|
|
809
533
|
"""
|
|
810
|
-
if
|
|
534
|
+
if WRFRUN.config.FAKE_SIMULATION_MODE:
|
|
811
535
|
logger.info(f"We are in fake simulation mode, skip saving outputs for '{self.name}'")
|
|
812
536
|
return
|
|
813
537
|
|
|
@@ -816,8 +540,8 @@ class ExecutableBase:
|
|
|
816
540
|
save_path = output_file["save_path"]
|
|
817
541
|
save_name = output_file["save_name"]
|
|
818
542
|
|
|
819
|
-
file_path =
|
|
820
|
-
save_path =
|
|
543
|
+
file_path = WRFRUN.config.parse_resource_uri(file_path)
|
|
544
|
+
save_path = WRFRUN.config.parse_resource_uri(save_path)
|
|
821
545
|
|
|
822
546
|
file_path = abspath(file_path)
|
|
823
547
|
save_path = abspath(save_path)
|
|
@@ -831,7 +555,12 @@ class ExecutableBase:
|
|
|
831
555
|
|
|
832
556
|
target_path = f"{save_path}/{save_name}"
|
|
833
557
|
if exists(target_path):
|
|
834
|
-
logger.warning(
|
|
558
|
+
logger.warning(
|
|
559
|
+
(
|
|
560
|
+
f"Found existed file, which means you already may have output files in '{save_path}'. "
|
|
561
|
+
"If you are saving logs, ignore this warning."
|
|
562
|
+
)
|
|
563
|
+
)
|
|
835
564
|
|
|
836
565
|
move(file_path, target_path)
|
|
837
566
|
|
|
@@ -839,11 +568,13 @@ class ExecutableBase:
|
|
|
839
568
|
"""
|
|
840
569
|
Execute the given command.
|
|
841
570
|
"""
|
|
842
|
-
work_path =
|
|
571
|
+
work_path = WRFRUN.config.parse_resource_uri(self.work_path)
|
|
843
572
|
|
|
844
573
|
if not self.mpi_use or None in [self.mpi_cmd, self.mpi_core_num]:
|
|
845
574
|
if isinstance(self.cmd, str):
|
|
846
|
-
self.cmd = [
|
|
575
|
+
self.cmd = [
|
|
576
|
+
self.cmd,
|
|
577
|
+
]
|
|
847
578
|
|
|
848
579
|
logger.info(f"Running `{' '.join(self.cmd)}` ...")
|
|
849
580
|
_cmd = self.cmd
|
|
@@ -852,7 +583,7 @@ class ExecutableBase:
|
|
|
852
583
|
logger.info(f"Running `{self.mpi_cmd} --oversubscribe -np {self.mpi_core_num} {self.cmd}` ...")
|
|
853
584
|
_cmd = [self.mpi_cmd, "--oversubscribe", "-np", str(self.mpi_core_num), self.cmd]
|
|
854
585
|
|
|
855
|
-
if
|
|
586
|
+
if WRFRUN.config.FAKE_SIMULATION_MODE:
|
|
856
587
|
logger.info(f"We are in fake simulation mode, skip calling numerical model for '{self.name}'")
|
|
857
588
|
return
|
|
858
589
|
|
|
@@ -869,13 +600,8 @@ class ExecutableBase:
|
|
|
869
600
|
self.exec()
|
|
870
601
|
self.after_exec()
|
|
871
602
|
|
|
872
|
-
if not
|
|
873
|
-
|
|
874
|
-
if ExecConfigRecorder is None:
|
|
875
|
-
logger.error(f"Trying to record simulation before create the recorder.")
|
|
876
|
-
raise RecordError(f"Trying to record simulation before create the recorder.")
|
|
877
|
-
|
|
878
|
-
ExecConfigRecorder.record(self.export_config())
|
|
603
|
+
if not WRFRUN.config.IS_IN_REPLAY and WRFRUN.config.IS_RECORDING:
|
|
604
|
+
WRFRUN.record.record(self.export_config())
|
|
879
605
|
|
|
880
606
|
|
|
881
|
-
__all__ = ["ExecutableBase", "
|
|
607
|
+
__all__ = ["ExecutableBase", "call_subprocess"]
|