wrfrun 0.1.9__py3-none-any.whl → 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- wrfrun/__init__.py +8 -3
- wrfrun/cli.py +74 -31
- wrfrun/core/__init__.py +27 -10
- wrfrun/core/_config.py +308 -0
- wrfrun/core/_constant.py +236 -0
- wrfrun/core/_exec_db.py +105 -0
- wrfrun/core/_namelist.py +287 -0
- wrfrun/core/_record.py +178 -0
- wrfrun/core/_resource.py +172 -0
- wrfrun/core/base.py +136 -380
- wrfrun/core/core.py +196 -0
- wrfrun/core/error.py +35 -2
- wrfrun/core/replay.py +10 -96
- wrfrun/core/server.py +74 -32
- wrfrun/core/type.py +171 -0
- wrfrun/data.py +312 -149
- wrfrun/extension/goos_sst/__init__.py +2 -2
- wrfrun/extension/goos_sst/core.py +9 -14
- wrfrun/extension/goos_sst/res/__init__.py +0 -1
- wrfrun/extension/goos_sst/utils.py +50 -44
- wrfrun/extension/littler/core.py +105 -88
- wrfrun/extension/utils.py +5 -3
- wrfrun/log.py +117 -0
- wrfrun/model/__init__.py +11 -7
- wrfrun/model/constants.py +52 -0
- wrfrun/model/palm/__init__.py +30 -0
- wrfrun/model/palm/core.py +145 -0
- wrfrun/model/palm/namelist.py +33 -0
- wrfrun/model/plot.py +99 -114
- wrfrun/model/type.py +116 -0
- wrfrun/model/utils.py +9 -19
- wrfrun/model/wrf/__init__.py +4 -9
- wrfrun/model/wrf/core.py +262 -165
- wrfrun/model/wrf/exec_wrap.py +13 -12
- wrfrun/model/wrf/geodata.py +116 -99
- wrfrun/model/wrf/log.py +103 -0
- wrfrun/model/wrf/namelist.py +92 -76
- wrfrun/model/wrf/plot.py +102 -0
- wrfrun/model/wrf/scheme.py +108 -52
- wrfrun/model/wrf/utils.py +39 -24
- wrfrun/model/wrf/vtable.py +42 -7
- wrfrun/plot/__init__.py +20 -0
- wrfrun/plot/wps.py +90 -73
- wrfrun/res/__init__.py +115 -5
- wrfrun/res/config/config.template.toml +15 -0
- wrfrun/res/config/palm.template.toml +23 -0
- wrfrun/run.py +121 -77
- wrfrun/scheduler/__init__.py +1 -0
- wrfrun/scheduler/lsf.py +4 -1
- wrfrun/scheduler/pbs.py +4 -1
- wrfrun/scheduler/script.py +19 -5
- wrfrun/scheduler/slurm.py +4 -1
- wrfrun/scheduler/utils.py +14 -2
- wrfrun/utils.py +88 -199
- wrfrun/workspace/__init__.py +8 -5
- wrfrun/workspace/core.py +21 -11
- wrfrun/workspace/palm.py +137 -0
- wrfrun/workspace/wrf.py +59 -14
- wrfrun-0.3.0.dist-info/METADATA +240 -0
- wrfrun-0.3.0.dist-info/RECORD +78 -0
- wrfrun/core/config.py +0 -767
- wrfrun/model/base.py +0 -14
- wrfrun-0.1.9.dist-info/METADATA +0 -68
- wrfrun-0.1.9.dist-info/RECORD +0 -62
- {wrfrun-0.1.9.dist-info → wrfrun-0.3.0.dist-info}/WHEEL +0 -0
- {wrfrun-0.1.9.dist-info → wrfrun-0.3.0.dist-info}/entry_points.txt +0 -0
wrfrun/core/base.py
CHANGED
|
@@ -2,22 +2,27 @@
|
|
|
2
2
|
wrfrun.core.base
|
|
3
3
|
################
|
|
4
4
|
|
|
5
|
-
Defines what :class:`ExecutableBase <Executable>` is, how it works and how ``wrfrun`` records simulations.
|
|
6
|
-
|
|
7
5
|
.. autosummary::
|
|
8
6
|
:toctree: generated/
|
|
9
7
|
|
|
10
8
|
check_subprocess_status
|
|
11
9
|
call_subprocess
|
|
12
|
-
InputFileType
|
|
13
|
-
FileConfigDict
|
|
14
|
-
ExecutableClassConfig
|
|
15
|
-
ExecutableConfig
|
|
16
|
-
_ExecutableConfigRecord
|
|
17
10
|
ExecutableBase
|
|
18
11
|
|
|
19
|
-
|
|
20
|
-
|
|
12
|
+
ExecutableBase
|
|
13
|
+
**************
|
|
14
|
+
|
|
15
|
+
What is the ``Executable`` in wrfrun?
|
|
16
|
+
=====================================
|
|
17
|
+
|
|
18
|
+
``Executable`` is the part to interact with external resources (for example, external programs, datasets, etc.).
|
|
19
|
+
They are classes that implement some essential methods,
|
|
20
|
+
controlled by wrfrun to achieve the goal: managing numerical simulations.
|
|
21
|
+
|
|
22
|
+
In wrfrun, all the ``Executable`` is the subclass of :class:`ExecutableBase`.
|
|
23
|
+
|
|
24
|
+
Why wrfrun defines ``ExecutableBase``?
|
|
25
|
+
======================================
|
|
21
26
|
|
|
22
27
|
While ``wrfrun`` aims to provide Python interfaces to various Numerical Weather Prediction model,
|
|
23
28
|
it is important to provide a clear standard about how should an external executable file be implemented in ``wrfrun``.
|
|
@@ -27,41 +32,35 @@ but also:
|
|
|
27
32
|
|
|
28
33
|
* Store all the information about the program (e.g., its inputs and outputs, its configuration).
|
|
29
34
|
* Provide the interface to import and export ``Executable``'s config.
|
|
30
|
-
* Support the ``replay`` functionality of ``wrfrun``.
|
|
31
35
|
|
|
32
|
-
|
|
36
|
+
With all the ``Executable`` have the same interface, wrfrun could provide some really cool features, like:
|
|
33
37
|
|
|
34
|
-
|
|
35
|
-
|
|
38
|
+
* Record the whole simulation, generate a record file (I call it the replay file).
|
|
39
|
+
* Replay the whole simulation based on the replay file.
|
|
36
40
|
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
``ExecConfigRecorder`` can store the config of any ``Executable`` in the correct order and export them to a JSON file,
|
|
40
|
-
enabling the precise replay of the simulation. Depending on the internal implementation of the ``Executable``,
|
|
41
|
-
the recorded config may even include the complete namelist values, Vtable config, and more.
|
|
42
|
-
Please check :meth:`ExecutableBase.generate_custom_config` for more details.
|
|
41
|
+
If you want to improve wrfrun's function to interact with external resources,
|
|
42
|
+
I strongly recommend you to implement your code by inheriting :class:`ExecutableBase`.
|
|
43
43
|
"""
|
|
44
44
|
|
|
45
45
|
import subprocess
|
|
46
46
|
from copy import deepcopy
|
|
47
|
-
from enum import Enum
|
|
48
|
-
from json import dumps
|
|
49
47
|
from os import chdir, getcwd, listdir, makedirs, remove, symlink
|
|
50
|
-
from os.path import abspath, basename,
|
|
51
|
-
from shutil import
|
|
52
|
-
from typing import Optional,
|
|
48
|
+
from os.path import abspath, basename, exists
|
|
49
|
+
from shutil import move
|
|
50
|
+
from typing import Optional, Union
|
|
53
51
|
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
from .config import WRFRUNConfig
|
|
52
|
+
from ..log import logger
|
|
53
|
+
from .core import WRFRUN
|
|
57
54
|
from .error import CommandError, ConfigError, OutputFileError
|
|
58
|
-
from
|
|
55
|
+
from .type import ExecutableClassConfig, ExecutableConfig, FileConfigDict
|
|
59
56
|
|
|
60
57
|
|
|
61
58
|
def check_subprocess_status(status: subprocess.CompletedProcess):
|
|
62
59
|
"""
|
|
63
60
|
Check subprocess return code.
|
|
64
|
-
|
|
61
|
+
|
|
62
|
+
An ``RuntimeError`` exception will be raised if ``return_code != 0``,
|
|
63
|
+
and the ``stdout`` and ``stderr`` of the subprocess will be logged.
|
|
65
64
|
|
|
66
65
|
:param status: Status from subprocess.
|
|
67
66
|
:type status: CompletedProcess
|
|
@@ -101,7 +100,7 @@ def call_subprocess(command: list[str], work_path: Optional[str] = None, print_o
|
|
|
101
100
|
else:
|
|
102
101
|
origin_path = None
|
|
103
102
|
|
|
104
|
-
status = subprocess.run(
|
|
103
|
+
status = subprocess.run(" ".join(command), shell=True, capture_output=True)
|
|
105
104
|
|
|
106
105
|
if origin_path is not None:
|
|
107
106
|
chdir(origin_path)
|
|
@@ -113,317 +112,10 @@ def call_subprocess(command: list[str], work_path: Optional[str] = None, print_o
|
|
|
113
112
|
logger.warning(status.stderr.decode())
|
|
114
113
|
|
|
115
114
|
|
|
116
|
-
def _json_default(obj):
|
|
117
|
-
"""
|
|
118
|
-
Used for json.dumps.
|
|
119
|
-
|
|
120
|
-
:param obj:
|
|
121
|
-
:type obj:
|
|
122
|
-
:return:
|
|
123
|
-
:rtype:
|
|
124
|
-
"""
|
|
125
|
-
if isinstance(obj, np.integer):
|
|
126
|
-
return int(obj)
|
|
127
|
-
elif isinstance(obj, np.floating):
|
|
128
|
-
return float(obj)
|
|
129
|
-
else:
|
|
130
|
-
raise TypeError(f"Object of type {type(obj)} is not JSON serializable.")
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
class InputFileType(Enum):
|
|
134
|
-
"""
|
|
135
|
-
This class is an ``Enum`` class, providing the following values:
|
|
136
|
-
|
|
137
|
-
.. py:attribute:: WRFRUN_RES
|
|
138
|
-
:type: int
|
|
139
|
-
:value: 1
|
|
140
|
-
|
|
141
|
-
Indicating resource files are from the model or ``wrfrun``.
|
|
142
|
-
``wrfrun`` won't save these files when recording the simulation.
|
|
143
|
-
|
|
144
|
-
.. py:attribute:: CUSTOM_RES
|
|
145
|
-
:type: int
|
|
146
|
-
:value: 2
|
|
147
|
-
|
|
148
|
-
Indicating resource files are provided by the user.
|
|
149
|
-
``wrfrun`` will also save these files to ``.replay`` file when recording the simulation to ensure the simulation is replayable.
|
|
150
|
-
"""
|
|
151
|
-
WRFRUN_RES = 1
|
|
152
|
-
CUSTOM_RES = 2
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
class FileConfigDict(TypedDict):
|
|
156
|
-
"""
|
|
157
|
-
This dict is used to store information about the file, including its path,
|
|
158
|
-
the path it will be copied or moved to, its new name, etc.
|
|
159
|
-
This dict contains following keys:
|
|
160
|
-
|
|
161
|
-
.. py:attribute:: file_path
|
|
162
|
-
:type: str
|
|
163
|
-
|
|
164
|
-
A real file path or a valid URI which can be converted to a file path.
|
|
165
|
-
|
|
166
|
-
.. py:attribute:: save_path
|
|
167
|
-
:type: str
|
|
168
|
-
|
|
169
|
-
Save path of the file.
|
|
170
|
-
|
|
171
|
-
.. py:attribute:: save_name
|
|
172
|
-
:type: str
|
|
173
|
-
|
|
174
|
-
Save name of the file.
|
|
175
|
-
|
|
176
|
-
.. py:attribute:: is_data
|
|
177
|
-
:type: bool
|
|
178
|
-
|
|
179
|
-
If the file is data. If not, ``wrfrun`` will treat it as a config file,
|
|
180
|
-
and always save it to ``.replay`` file when recording the simulation.
|
|
181
|
-
|
|
182
|
-
.. py:attribute:: is_output
|
|
183
|
-
:type: bool
|
|
184
|
-
|
|
185
|
-
If the file is model's output. Output file will never be saved to ``.replay`` file.
|
|
186
|
-
"""
|
|
187
|
-
file_path: str
|
|
188
|
-
save_path: str
|
|
189
|
-
save_name: str
|
|
190
|
-
is_data: bool
|
|
191
|
-
is_output: bool
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
class ExecutableClassConfig(TypedDict):
|
|
195
|
-
"""
|
|
196
|
-
This dict is used to store arguments of ``Executable``'s ``__init__`` function.
|
|
197
|
-
|
|
198
|
-
.. py:attribute:: class_args
|
|
199
|
-
:type: tuple
|
|
200
|
-
|
|
201
|
-
Positional arguments of the class.
|
|
202
|
-
|
|
203
|
-
.. py:attribute:: class_kwargs
|
|
204
|
-
:type: dict
|
|
205
|
-
|
|
206
|
-
Keyword arguments of the class.
|
|
207
|
-
"""
|
|
208
|
-
# only list essential config
|
|
209
|
-
class_args: tuple
|
|
210
|
-
class_kwargs: dict
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
class ExecutableConfig(TypedDict):
|
|
214
|
-
"""
|
|
215
|
-
This dict is used to store all configs of a :class:`ExecutableBase`.
|
|
216
|
-
|
|
217
|
-
.. py:attribute:: name
|
|
218
|
-
:type: str
|
|
219
|
-
|
|
220
|
-
Name of the executable. Each type of executable has a unique name.
|
|
221
|
-
|
|
222
|
-
.. py:attribute:: cmd
|
|
223
|
-
:type: str | list[str]
|
|
224
|
-
|
|
225
|
-
Command of the executable.
|
|
226
|
-
|
|
227
|
-
.. py:attribute:: work_path
|
|
228
|
-
:type: str | None
|
|
229
|
-
|
|
230
|
-
Work path of the executable.
|
|
231
|
-
|
|
232
|
-
.. py:attribute:: mpi_use
|
|
233
|
-
:type: bool
|
|
234
|
-
|
|
235
|
-
If the executable will use MPI.
|
|
236
|
-
|
|
237
|
-
.. py:attribute:: mpi_cmd
|
|
238
|
-
:type: str | None
|
|
239
|
-
|
|
240
|
-
Command name of the MPI.
|
|
241
|
-
|
|
242
|
-
.. py:attribute:: mpi_core_num
|
|
243
|
-
:type: int | None
|
|
244
|
-
|
|
245
|
-
Number of the CPU core to use with MPI.
|
|
246
|
-
|
|
247
|
-
.. py:attribute:: class_config
|
|
248
|
-
:type: ExecutableClassConfig | None
|
|
249
|
-
|
|
250
|
-
A dict stores arguments of ``Executable``'s ``__init__`` function.
|
|
251
|
-
|
|
252
|
-
.. py:attribute:: input_file_config
|
|
253
|
-
:type: list[FileConfigDict] | None
|
|
254
|
-
|
|
255
|
-
A list stores information about input files of the executable.
|
|
256
|
-
|
|
257
|
-
.. py:attribute:: output_file_config
|
|
258
|
-
:type: list[FileConfigDict] | None
|
|
259
|
-
|
|
260
|
-
A list stores information about output files of the executable.
|
|
261
|
-
|
|
262
|
-
.. py:attribute:: custom_config
|
|
263
|
-
:type: dict | None
|
|
264
|
-
|
|
265
|
-
A dict that can be used by subclass to store other configs.
|
|
266
|
-
"""
|
|
267
|
-
name: str
|
|
268
|
-
cmd: Union[str, list[str]]
|
|
269
|
-
work_path: Optional[str]
|
|
270
|
-
mpi_use: bool
|
|
271
|
-
mpi_cmd: Optional[str]
|
|
272
|
-
mpi_core_num: Optional[int]
|
|
273
|
-
class_config: Optional[ExecutableClassConfig]
|
|
274
|
-
input_file_config: Optional[list[FileConfigDict]]
|
|
275
|
-
output_file_config: Optional[list[FileConfigDict]]
|
|
276
|
-
custom_config: Optional[dict]
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
class _ExecutableConfigRecord:
|
|
280
|
-
"""
|
|
281
|
-
A class to helps store configs of various executables and exports them to a file.
|
|
282
|
-
"""
|
|
283
|
-
_instance = None
|
|
284
|
-
_initialized = False
|
|
285
|
-
|
|
286
|
-
def __init__(self, save_path: Optional[str] = None, include_data=False):
|
|
287
|
-
"""
|
|
288
|
-
|
|
289
|
-
:param save_path: Save path of the exported config file.
|
|
290
|
-
:type save_path: str | None
|
|
291
|
-
:param include_data: If includes input data.
|
|
292
|
-
:type include_data: bool
|
|
293
|
-
"""
|
|
294
|
-
|
|
295
|
-
if self._initialized:
|
|
296
|
-
return
|
|
297
|
-
|
|
298
|
-
if save_path is None:
|
|
299
|
-
WRFRUNConfig.IS_RECORDING = False
|
|
300
|
-
else:
|
|
301
|
-
WRFRUNConfig.IS_RECORDING = True
|
|
302
|
-
|
|
303
|
-
self.save_path = save_path
|
|
304
|
-
self.include_data = include_data
|
|
305
|
-
|
|
306
|
-
self.work_path = WRFRUNConfig.parse_resource_uri(WRFRUNConfig.WRFRUN_WORKSPACE_REPLAY)
|
|
307
|
-
self.content_path = f"{self.work_path}/config_and_data"
|
|
308
|
-
check_path(self.content_path)
|
|
309
|
-
|
|
310
|
-
self._recorded_config = []
|
|
311
|
-
self._name_count = {}
|
|
312
|
-
|
|
313
|
-
self._initialized = True
|
|
314
|
-
|
|
315
|
-
def __new__(cls, *args, **kwargs):
|
|
316
|
-
if cls._instance is None:
|
|
317
|
-
cls._instance = super().__new__(cls)
|
|
318
|
-
|
|
319
|
-
return cls._instance
|
|
320
|
-
|
|
321
|
-
def reinit(self, save_path: Optional[str] = None, include_data=False):
|
|
322
|
-
"""
|
|
323
|
-
Reinitialize this instance.
|
|
324
|
-
|
|
325
|
-
:return: New instance.
|
|
326
|
-
:rtype: _ExecutableConfigRecord
|
|
327
|
-
"""
|
|
328
|
-
self._initialized = False
|
|
329
|
-
return _ExecutableConfigRecord(save_path, include_data)
|
|
330
|
-
|
|
331
|
-
def record(self, exported_config: ExecutableConfig):
|
|
332
|
-
"""
|
|
333
|
-
Record exported config for replay.
|
|
334
|
-
|
|
335
|
-
:param exported_config: Executable config.
|
|
336
|
-
:type exported_config: ExecutableConfig
|
|
337
|
-
"""
|
|
338
|
-
if not self.include_data:
|
|
339
|
-
self._recorded_config.append(exported_config)
|
|
340
|
-
return
|
|
341
|
-
|
|
342
|
-
check_path(self.content_path)
|
|
343
|
-
|
|
344
|
-
# process exported config so we can also include data.
|
|
345
|
-
# create directory to place data
|
|
346
|
-
name = exported_config["name"]
|
|
347
|
-
if name in self._name_count:
|
|
348
|
-
self._name_count[name] += 1
|
|
349
|
-
index = self._name_count[name]
|
|
350
|
-
else:
|
|
351
|
-
self._name_count[name] = 1
|
|
352
|
-
index = 1
|
|
353
|
-
|
|
354
|
-
data_save_uri = f"{WRFRUNConfig.WRFRUN_WORKSPACE_REPLAY}/{name}/{index}"
|
|
355
|
-
data_save_path = f"{self.content_path}/{name}/{index}"
|
|
356
|
-
makedirs(data_save_path)
|
|
357
|
-
|
|
358
|
-
input_file_config = exported_config["input_file_config"]
|
|
359
|
-
|
|
360
|
-
for _config_index, _config in enumerate(input_file_config):
|
|
361
|
-
if not _config["is_data"]:
|
|
362
|
-
continue
|
|
363
|
-
|
|
364
|
-
if _config["is_output"]:
|
|
365
|
-
continue
|
|
366
|
-
|
|
367
|
-
file_path = _config["file_path"]
|
|
368
|
-
file_path = WRFRUNConfig.parse_resource_uri(file_path)
|
|
369
|
-
filename = basename(file_path)
|
|
370
|
-
copyfile(file_path, f"{data_save_path}/{filename}")
|
|
371
|
-
|
|
372
|
-
_config["file_path"] = f"{data_save_uri}/{filename}"
|
|
373
|
-
input_file_config[_config_index] = _config
|
|
374
|
-
|
|
375
|
-
exported_config["input_file_config"] = input_file_config
|
|
376
|
-
self._recorded_config.append(exported_config)
|
|
377
|
-
|
|
378
|
-
def clear_records(self):
|
|
379
|
-
"""
|
|
380
|
-
Clean recorded configs.
|
|
381
|
-
"""
|
|
382
|
-
self._recorded_config = []
|
|
383
|
-
|
|
384
|
-
def export_replay_file(self):
|
|
385
|
-
"""
|
|
386
|
-
Save replay file to the save path.
|
|
387
|
-
"""
|
|
388
|
-
if len(self._recorded_config) == 0:
|
|
389
|
-
logger.warning("No replay config has been recorded.")
|
|
390
|
-
return
|
|
391
|
-
|
|
392
|
-
logger.info("Exporting replay config... It may take a few minutes if you include data.")
|
|
393
|
-
|
|
394
|
-
check_path(self.content_path)
|
|
395
|
-
|
|
396
|
-
with open(f"{self.content_path}/config.json", "w") as f:
|
|
397
|
-
f.write(dumps(self._recorded_config, indent=4, default=_json_default))
|
|
398
|
-
|
|
399
|
-
if exists(self.save_path):
|
|
400
|
-
if isdir(self.save_path):
|
|
401
|
-
self.save_path = f"{self.save_path}/1.replay"
|
|
402
|
-
else:
|
|
403
|
-
if not self.save_path.endswith(".replay"):
|
|
404
|
-
self.save_path = f"{self.save_path}.replay"
|
|
405
|
-
|
|
406
|
-
if exists(self.save_path):
|
|
407
|
-
logger.warning(f"Found existed replay file with the same name '{basename(self.save_path)}', overwrite it")
|
|
408
|
-
remove(self.save_path)
|
|
409
|
-
|
|
410
|
-
if not exists(dirname(self.save_path)):
|
|
411
|
-
makedirs(dirname(self.save_path))
|
|
412
|
-
|
|
413
|
-
temp_file = f"{self.work_path}/config_and_data"
|
|
414
|
-
make_archive(temp_file, "zip", self.content_path)
|
|
415
|
-
move(f"{temp_file}.zip", self.save_path)
|
|
416
|
-
|
|
417
|
-
logger.info(f"Replay config exported to {self.save_path}")
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
ExecConfigRecorder = _ExecutableConfigRecord()
|
|
421
|
-
|
|
422
|
-
|
|
423
115
|
class ExecutableBase:
|
|
424
116
|
"""
|
|
425
117
|
Base class for all executables.
|
|
426
|
-
|
|
118
|
+
|
|
427
119
|
.. py:attribute:: class_config
|
|
428
120
|
:type: ExecutableClassConfig
|
|
429
121
|
:value: {"class_args": (), "class_kwargs": {}}
|
|
@@ -449,10 +141,18 @@ class ExecutableBase:
|
|
|
449
141
|
A list stores information about output files of the executable.
|
|
450
142
|
|
|
451
143
|
"""
|
|
144
|
+
|
|
452
145
|
_instance = None
|
|
453
146
|
|
|
454
|
-
def __init__(
|
|
455
|
-
|
|
147
|
+
def __init__(
|
|
148
|
+
self,
|
|
149
|
+
name: str,
|
|
150
|
+
cmd: Union[str, list[str]],
|
|
151
|
+
work_path: str,
|
|
152
|
+
mpi_use=False,
|
|
153
|
+
mpi_cmd: Optional[str] = None,
|
|
154
|
+
mpi_core_num: Optional[int] = None,
|
|
155
|
+
):
|
|
456
156
|
"""
|
|
457
157
|
|
|
458
158
|
:param name: Unique name to identify different executables.
|
|
@@ -463,7 +163,7 @@ class ExecutableBase:
|
|
|
463
163
|
:type cmd: str
|
|
464
164
|
:param work_path: Working directory path.
|
|
465
165
|
:type work_path: str
|
|
466
|
-
:param mpi_use: If
|
|
166
|
+
:param mpi_use: If use mpi. You have to give ``mpi_cmd`` and ``mpi_core_num`` if you use mpi. Defaults to False.
|
|
467
167
|
:type mpi_use: bool
|
|
468
168
|
:param mpi_cmd: MPI command. For example, ``"mpirun"``. Defaults to None.
|
|
469
169
|
:type mpi_cmd: str
|
|
@@ -481,13 +181,17 @@ class ExecutableBase:
|
|
|
481
181
|
self.mpi_cmd = mpi_cmd
|
|
482
182
|
self.mpi_core_num = mpi_core_num
|
|
483
183
|
|
|
184
|
+
# don't use mpi if mpi_core_num = 1
|
|
185
|
+
if isinstance(self.mpi_core_num, int) and self.mpi_core_num < 2:
|
|
186
|
+
self.mpi_core_num = None
|
|
187
|
+
|
|
484
188
|
self.class_config: ExecutableClassConfig = {"class_args": (), "class_kwargs": {}}
|
|
485
189
|
self.custom_config: dict = {}
|
|
486
190
|
self.input_file_config: list[FileConfigDict] = []
|
|
487
191
|
self.output_file_config: list[FileConfigDict] = []
|
|
488
192
|
|
|
489
193
|
# directory to save outputs
|
|
490
|
-
self._output_save_path = f"{
|
|
194
|
+
self._output_save_path = f"{WRFRUN.config.WRFRUN_OUTPUT_PATH}/{self.name}"
|
|
491
195
|
self._log_save_path = f"{self._output_save_path}/logs"
|
|
492
196
|
|
|
493
197
|
def __new__(cls, *args, **kwargs):
|
|
@@ -498,7 +202,9 @@ class ExecutableBase:
|
|
|
498
202
|
|
|
499
203
|
def generate_custom_config(self):
|
|
500
204
|
"""
|
|
501
|
-
Generate custom configs.
|
|
205
|
+
Generate custom configs.
|
|
206
|
+
|
|
207
|
+
This method should be overwritten in the child class,
|
|
502
208
|
and **MUST STORE THE CUSTOM CONFIG IN THE ATTRIBUTE** :attr:`ExecutableBase.custom_config`,
|
|
503
209
|
or it will do nothing except print a debug log.
|
|
504
210
|
|
|
@@ -513,7 +219,9 @@ class ExecutableBase:
|
|
|
513
219
|
def load_custom_config(self):
|
|
514
220
|
"""
|
|
515
221
|
Load custom configs.
|
|
516
|
-
|
|
222
|
+
|
|
223
|
+
This method should be overwritten in the child class to
|
|
224
|
+
process the custom config stored in :attr:`ExecutableBase.custom_config`,
|
|
517
225
|
or it will do nothing except print a debug log.
|
|
518
226
|
"""
|
|
519
227
|
logger.debug(f"Method 'load_custom_config' not implemented in '{self.name}'")
|
|
@@ -537,7 +245,7 @@ class ExecutableBase:
|
|
|
537
245
|
"class_config": deepcopy(self.class_config),
|
|
538
246
|
"custom_config": deepcopy(self.custom_config),
|
|
539
247
|
"input_file_config": deepcopy(self.input_file_config),
|
|
540
|
-
"output_file_config": deepcopy(self.output_file_config)
|
|
248
|
+
"output_file_config": deepcopy(self.output_file_config),
|
|
541
249
|
}
|
|
542
250
|
|
|
543
251
|
def load_config(self, config: ExecutableConfig):
|
|
@@ -576,8 +284,9 @@ class ExecutableBase:
|
|
|
576
284
|
logger.debug(f"Method 'replay' not implemented in '{self.name}', fall back to default action.")
|
|
577
285
|
self()
|
|
578
286
|
|
|
579
|
-
def add_input_files(
|
|
580
|
-
|
|
287
|
+
def add_input_files(
|
|
288
|
+
self, input_files: Union[str, list[str], FileConfigDict, list[FileConfigDict]], is_data=True, is_output=True
|
|
289
|
+
):
|
|
581
290
|
"""
|
|
582
291
|
Add input files the executable will use.
|
|
583
292
|
|
|
@@ -588,25 +297,27 @@ class ExecutableBase:
|
|
|
588
297
|
|
|
589
298
|
You can give more information with a ``FileConfigDict``.
|
|
590
299
|
|
|
300
|
+
>>> from wrfrun.workspace.wrf import get_wrf_workspace_path
|
|
591
301
|
>>> file_dict: FileConfigDict = {
|
|
592
302
|
... "file_path": "data/custom_file.nc",
|
|
593
|
-
... "save_path":
|
|
303
|
+
... "save_path": get_wrf_workspace_path("wps"),
|
|
594
304
|
... "save_name": "custom_file.nc",
|
|
595
305
|
... "is_data": True,
|
|
596
306
|
... "is_output": False
|
|
597
307
|
... }
|
|
598
308
|
>>> self.add_input_files(file_dict)
|
|
599
309
|
|
|
310
|
+
>>> from wrfrun.workspace.wrf import get_wrf_workspace_path
|
|
600
311
|
>>> file_dict_1: FileConfigDict = {
|
|
601
312
|
... "file_path": "data/custom_file",
|
|
602
|
-
... "save_path": f"{
|
|
313
|
+
... "save_path": f"{get_wrf_workspace_path('wps')}/geogrid",
|
|
603
314
|
... "save_name": "GEOGRID.TBL",
|
|
604
315
|
... "is_data": False,
|
|
605
316
|
... "is_output": False
|
|
606
317
|
... }
|
|
607
318
|
>>> file_dict_2: FileConfigDict = {
|
|
608
319
|
... "file_path": "data/custom_file",
|
|
609
|
-
... "save_path": f"{
|
|
320
|
+
... "save_path": f"{get_wrf_workspace_path('wps')}/outputs",
|
|
610
321
|
... "save_name": "test_file",
|
|
611
322
|
... "is_data": True,
|
|
612
323
|
... "is_output": True
|
|
@@ -619,14 +330,18 @@ class ExecutableBase:
|
|
|
619
330
|
:type input_files: str | list | dict
|
|
620
331
|
:param is_data: If it is a data file. This parameter will be overwritten by the value in ``input_files``.
|
|
621
332
|
:type is_data: bool
|
|
622
|
-
:param is_output: If it is an output from another executable.
|
|
333
|
+
:param is_output: If it is an output from another executable.
|
|
334
|
+
This parameter will be overwritten by the value in ``input_files``.
|
|
623
335
|
:type is_output: bool
|
|
624
336
|
"""
|
|
625
337
|
if isinstance(input_files, str):
|
|
626
338
|
self.input_file_config.append(
|
|
627
339
|
{
|
|
628
|
-
"file_path": input_files,
|
|
629
|
-
"
|
|
340
|
+
"file_path": input_files,
|
|
341
|
+
"save_path": self.work_path,
|
|
342
|
+
"save_name": basename(input_files),
|
|
343
|
+
"is_data": is_data,
|
|
344
|
+
"is_output": is_output,
|
|
630
345
|
}
|
|
631
346
|
)
|
|
632
347
|
|
|
@@ -638,8 +353,11 @@ class ExecutableBase:
|
|
|
638
353
|
elif isinstance(_file, str):
|
|
639
354
|
self.input_file_config.append(
|
|
640
355
|
{
|
|
641
|
-
"file_path": _file,
|
|
642
|
-
"
|
|
356
|
+
"file_path": _file,
|
|
357
|
+
"save_path": self.work_path,
|
|
358
|
+
"save_name": basename(_file),
|
|
359
|
+
"is_data": is_data,
|
|
360
|
+
"is_output": is_output,
|
|
643
361
|
}
|
|
644
362
|
)
|
|
645
363
|
|
|
@@ -655,8 +373,13 @@ class ExecutableBase:
|
|
|
655
373
|
raise TypeError(f"Input file config should be string or `FileConfigDict`, but got '{type(input_files)}'")
|
|
656
374
|
|
|
657
375
|
def add_output_files(
|
|
658
|
-
self,
|
|
659
|
-
|
|
376
|
+
self,
|
|
377
|
+
output_dir: Optional[str] = None,
|
|
378
|
+
save_path: Optional[str] = None,
|
|
379
|
+
startswith: Union[None, str, tuple[str, ...]] = None,
|
|
380
|
+
endswith: Union[None, str, tuple[str, ...]] = None,
|
|
381
|
+
outputs: Union[None, str, list[str]] = None,
|
|
382
|
+
no_file_error=True,
|
|
660
383
|
):
|
|
661
384
|
"""
|
|
662
385
|
Find and save model's outputs to the output save path.
|
|
@@ -667,7 +390,8 @@ class ExecutableBase:
|
|
|
667
390
|
>>> self.add_output_files(outputs="wrfout.d01")
|
|
668
391
|
>>> self.add_output_files(outputs=["wrfout.d01", "wrfout.d02"])
|
|
669
392
|
|
|
670
|
-
If you have too many outputs, but they have the same prefix or postfix,
|
|
393
|
+
If you have too many outputs, but they have the same prefix or postfix,
|
|
394
|
+
you can use ``startswith`` or ``endswith``.
|
|
671
395
|
|
|
672
396
|
>>> self.add_output_files(startswith="rsl.out.")
|
|
673
397
|
>>> self.add_output_files(endswith="log")
|
|
@@ -682,7 +406,8 @@ class ExecutableBase:
|
|
|
682
406
|
|
|
683
407
|
:param output_dir: Search path of outputs.
|
|
684
408
|
:type output_dir: str
|
|
685
|
-
:param save_path: New save path of outputs.
|
|
409
|
+
:param save_path: New save path of outputs.
|
|
410
|
+
By default, it is ``f"{WRFRUNConfig.WRFRUN_OUTPUT_PATH}/{self.name}"``.
|
|
686
411
|
:type save_path: str
|
|
687
412
|
:param startswith: Prefix string or prefix list of output files.
|
|
688
413
|
:type startswith: str | list
|
|
@@ -690,19 +415,20 @@ class ExecutableBase:
|
|
|
690
415
|
:type endswith: str | list
|
|
691
416
|
:param outputs: Files name list. All files in the list will be saved.
|
|
692
417
|
:type outputs: str | list
|
|
693
|
-
:param no_file_error: If True, an OutputFileError will be raised if no output file can be found.
|
|
418
|
+
:param no_file_error: If True, an OutputFileError will be raised if no output file can be found.
|
|
419
|
+
Defaults to True.
|
|
694
420
|
:type no_file_error: bool
|
|
695
421
|
"""
|
|
696
|
-
if
|
|
422
|
+
if WRFRUN.config.FAKE_SIMULATION_MODE:
|
|
697
423
|
return
|
|
698
424
|
|
|
699
425
|
if output_dir is None:
|
|
700
426
|
output_dir = self.work_path
|
|
701
427
|
|
|
702
428
|
if save_path is None:
|
|
703
|
-
save_path = f"{
|
|
429
|
+
save_path = f"{WRFRUN.config.WRFRUN_OUTPUT_PATH}/{self.name}"
|
|
704
430
|
|
|
705
|
-
file_list = listdir(
|
|
431
|
+
file_list = listdir(WRFRUN.config.parse_resource_uri(output_dir))
|
|
706
432
|
save_file_list = []
|
|
707
433
|
|
|
708
434
|
if startswith is not None:
|
|
@@ -732,24 +458,47 @@ class ExecutableBase:
|
|
|
732
458
|
|
|
733
459
|
if len(save_file_list) < 1:
|
|
734
460
|
if no_file_error:
|
|
735
|
-
logger.error(
|
|
736
|
-
|
|
461
|
+
logger.error(
|
|
462
|
+
(
|
|
463
|
+
"Can't find any files match the giving rules: "
|
|
464
|
+
f"startswith='{startswith}', endswith='{endswith}', outputs='{outputs}'"
|
|
465
|
+
)
|
|
466
|
+
)
|
|
467
|
+
raise OutputFileError(
|
|
468
|
+
(
|
|
469
|
+
"Can't find any files match the giving rules: "
|
|
470
|
+
f"startswith='{startswith}', endswith='{endswith}', outputs='{outputs}'"
|
|
471
|
+
)
|
|
472
|
+
)
|
|
737
473
|
|
|
738
474
|
else:
|
|
739
|
-
logger.warning(
|
|
475
|
+
logger.warning(
|
|
476
|
+
(
|
|
477
|
+
"Can't find any files match the giving rules: "
|
|
478
|
+
f"startswith='{startswith}', endswith='{endswith}', outputs='{outputs}'. Skip it."
|
|
479
|
+
)
|
|
480
|
+
)
|
|
740
481
|
return
|
|
741
482
|
|
|
742
483
|
save_file_list = list(set(save_file_list))
|
|
743
484
|
logger.debug(f"Files to be processed: {save_file_list}")
|
|
744
485
|
|
|
745
486
|
for _file in save_file_list:
|
|
746
|
-
self.output_file_config.append(
|
|
487
|
+
self.output_file_config.append(
|
|
488
|
+
{
|
|
489
|
+
"file_path": f"{output_dir}/{_file}",
|
|
490
|
+
"save_path": save_path,
|
|
491
|
+
"save_name": _file,
|
|
492
|
+
"is_data": True,
|
|
493
|
+
"is_output": True,
|
|
494
|
+
}
|
|
495
|
+
)
|
|
747
496
|
|
|
748
497
|
def before_exec(self):
|
|
749
498
|
"""
|
|
750
499
|
Prepare input files before executing the external program.
|
|
751
500
|
"""
|
|
752
|
-
if
|
|
501
|
+
if WRFRUN.config.FAKE_SIMULATION_MODE:
|
|
753
502
|
logger.info(f"We are in fake simulation mode, skip preparing input files for '{self.name}'")
|
|
754
503
|
return
|
|
755
504
|
|
|
@@ -758,8 +507,8 @@ class ExecutableBase:
|
|
|
758
507
|
save_path = input_file["save_path"]
|
|
759
508
|
save_name = input_file["save_name"]
|
|
760
509
|
|
|
761
|
-
file_path =
|
|
762
|
-
save_path =
|
|
510
|
+
file_path = WRFRUN.config.parse_resource_uri(file_path)
|
|
511
|
+
save_path = WRFRUN.config.parse_resource_uri(save_path)
|
|
763
512
|
|
|
764
513
|
file_path = abspath(file_path)
|
|
765
514
|
save_path = abspath(save_path)
|
|
@@ -782,7 +531,7 @@ class ExecutableBase:
|
|
|
782
531
|
"""
|
|
783
532
|
Save outputs and logs after executing the external program.
|
|
784
533
|
"""
|
|
785
|
-
if
|
|
534
|
+
if WRFRUN.config.FAKE_SIMULATION_MODE:
|
|
786
535
|
logger.info(f"We are in fake simulation mode, skip saving outputs for '{self.name}'")
|
|
787
536
|
return
|
|
788
537
|
|
|
@@ -791,8 +540,8 @@ class ExecutableBase:
|
|
|
791
540
|
save_path = output_file["save_path"]
|
|
792
541
|
save_name = output_file["save_name"]
|
|
793
542
|
|
|
794
|
-
file_path =
|
|
795
|
-
save_path =
|
|
543
|
+
file_path = WRFRUN.config.parse_resource_uri(file_path)
|
|
544
|
+
save_path = WRFRUN.config.parse_resource_uri(save_path)
|
|
796
545
|
|
|
797
546
|
file_path = abspath(file_path)
|
|
798
547
|
save_path = abspath(save_path)
|
|
@@ -806,7 +555,12 @@ class ExecutableBase:
|
|
|
806
555
|
|
|
807
556
|
target_path = f"{save_path}/{save_name}"
|
|
808
557
|
if exists(target_path):
|
|
809
|
-
logger.warning(
|
|
558
|
+
logger.warning(
|
|
559
|
+
(
|
|
560
|
+
f"Found existed file, which means you already may have output files in '{save_path}'. "
|
|
561
|
+
"If you are saving logs, ignore this warning."
|
|
562
|
+
)
|
|
563
|
+
)
|
|
810
564
|
|
|
811
565
|
move(file_path, target_path)
|
|
812
566
|
|
|
@@ -814,11 +568,13 @@ class ExecutableBase:
|
|
|
814
568
|
"""
|
|
815
569
|
Execute the given command.
|
|
816
570
|
"""
|
|
817
|
-
work_path =
|
|
571
|
+
work_path = WRFRUN.config.parse_resource_uri(self.work_path)
|
|
818
572
|
|
|
819
573
|
if not self.mpi_use or None in [self.mpi_cmd, self.mpi_core_num]:
|
|
820
574
|
if isinstance(self.cmd, str):
|
|
821
|
-
self.cmd = [
|
|
575
|
+
self.cmd = [
|
|
576
|
+
self.cmd,
|
|
577
|
+
]
|
|
822
578
|
|
|
823
579
|
logger.info(f"Running `{' '.join(self.cmd)}` ...")
|
|
824
580
|
_cmd = self.cmd
|
|
@@ -827,7 +583,7 @@ class ExecutableBase:
|
|
|
827
583
|
logger.info(f"Running `{self.mpi_cmd} --oversubscribe -np {self.mpi_core_num} {self.cmd}` ...")
|
|
828
584
|
_cmd = [self.mpi_cmd, "--oversubscribe", "-np", str(self.mpi_core_num), self.cmd]
|
|
829
585
|
|
|
830
|
-
if
|
|
586
|
+
if WRFRUN.config.FAKE_SIMULATION_MODE:
|
|
831
587
|
logger.info(f"We are in fake simulation mode, skip calling numerical model for '{self.name}'")
|
|
832
588
|
return
|
|
833
589
|
|
|
@@ -844,8 +600,8 @@ class ExecutableBase:
|
|
|
844
600
|
self.exec()
|
|
845
601
|
self.after_exec()
|
|
846
602
|
|
|
847
|
-
if not
|
|
848
|
-
|
|
603
|
+
if not WRFRUN.config.IS_IN_REPLAY and WRFRUN.config.IS_RECORDING:
|
|
604
|
+
WRFRUN.record.record(self.export_config())
|
|
849
605
|
|
|
850
606
|
|
|
851
|
-
__all__ = ["ExecutableBase", "
|
|
607
|
+
__all__ = ["ExecutableBase", "call_subprocess"]
|