vortex-nwp 2.0.0b1__py3-none-any.whl → 2.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- vortex/__init__.py +75 -47
- vortex/algo/__init__.py +3 -2
- vortex/algo/components.py +944 -618
- vortex/algo/mpitools.py +802 -497
- vortex/algo/mpitools_templates/__init__.py +1 -0
- vortex/algo/serversynctools.py +34 -33
- vortex/config.py +19 -22
- vortex/data/__init__.py +9 -3
- vortex/data/abstractstores.py +593 -655
- vortex/data/containers.py +217 -162
- vortex/data/contents.py +65 -39
- vortex/data/executables.py +93 -102
- vortex/data/flow.py +40 -34
- vortex/data/geometries.py +228 -132
- vortex/data/handlers.py +436 -227
- vortex/data/outflow.py +15 -15
- vortex/data/providers.py +185 -163
- vortex/data/resources.py +48 -42
- vortex/data/stores.py +540 -417
- vortex/data/sync_templates/__init__.py +0 -0
- vortex/gloves.py +114 -87
- vortex/layout/__init__.py +1 -8
- vortex/layout/contexts.py +150 -84
- vortex/layout/dataflow.py +353 -202
- vortex/layout/monitor.py +264 -128
- vortex/nwp/__init__.py +5 -2
- vortex/nwp/algo/__init__.py +14 -5
- vortex/nwp/algo/assim.py +205 -151
- vortex/nwp/algo/clim.py +683 -517
- vortex/nwp/algo/coupling.py +447 -225
- vortex/nwp/algo/eda.py +437 -229
- vortex/nwp/algo/eps.py +403 -231
- vortex/nwp/algo/forecasts.py +416 -275
- vortex/nwp/algo/fpserver.py +683 -307
- vortex/nwp/algo/ifsnaming.py +205 -145
- vortex/nwp/algo/ifsroot.py +215 -122
- vortex/nwp/algo/monitoring.py +137 -76
- vortex/nwp/algo/mpitools.py +330 -190
- vortex/nwp/algo/odbtools.py +637 -353
- vortex/nwp/algo/oopsroot.py +454 -273
- vortex/nwp/algo/oopstests.py +90 -56
- vortex/nwp/algo/request.py +287 -206
- vortex/nwp/algo/stdpost.py +878 -522
- vortex/nwp/data/__init__.py +22 -4
- vortex/nwp/data/assim.py +125 -137
- vortex/nwp/data/boundaries.py +121 -68
- vortex/nwp/data/climfiles.py +193 -211
- vortex/nwp/data/configfiles.py +73 -69
- vortex/nwp/data/consts.py +426 -401
- vortex/nwp/data/ctpini.py +59 -43
- vortex/nwp/data/diagnostics.py +94 -66
- vortex/nwp/data/eda.py +50 -51
- vortex/nwp/data/eps.py +195 -146
- vortex/nwp/data/executables.py +440 -434
- vortex/nwp/data/fields.py +63 -48
- vortex/nwp/data/gridfiles.py +183 -111
- vortex/nwp/data/logs.py +250 -217
- vortex/nwp/data/modelstates.py +180 -151
- vortex/nwp/data/monitoring.py +72 -99
- vortex/nwp/data/namelists.py +254 -202
- vortex/nwp/data/obs.py +400 -308
- vortex/nwp/data/oopsexec.py +22 -20
- vortex/nwp/data/providers.py +90 -65
- vortex/nwp/data/query.py +71 -82
- vortex/nwp/data/stores.py +49 -36
- vortex/nwp/data/surfex.py +136 -137
- vortex/nwp/syntax/__init__.py +1 -1
- vortex/nwp/syntax/stdattrs.py +173 -111
- vortex/nwp/tools/__init__.py +2 -2
- vortex/nwp/tools/addons.py +22 -17
- vortex/nwp/tools/agt.py +24 -12
- vortex/nwp/tools/bdap.py +16 -5
- vortex/nwp/tools/bdcp.py +4 -1
- vortex/nwp/tools/bdm.py +3 -0
- vortex/nwp/tools/bdmp.py +14 -9
- vortex/nwp/tools/conftools.py +728 -378
- vortex/nwp/tools/drhook.py +12 -8
- vortex/nwp/tools/grib.py +65 -39
- vortex/nwp/tools/gribdiff.py +22 -17
- vortex/nwp/tools/ifstools.py +82 -42
- vortex/nwp/tools/igastuff.py +167 -143
- vortex/nwp/tools/mars.py +14 -2
- vortex/nwp/tools/odb.py +234 -125
- vortex/nwp/tools/partitioning.py +61 -37
- vortex/nwp/tools/satrad.py +27 -12
- vortex/nwp/util/async.py +83 -55
- vortex/nwp/util/beacon.py +10 -10
- vortex/nwp/util/diffpygram.py +174 -86
- vortex/nwp/util/ens.py +144 -63
- vortex/nwp/util/hooks.py +30 -19
- vortex/nwp/util/taskdeco.py +28 -24
- vortex/nwp/util/usepygram.py +278 -172
- vortex/nwp/util/usetnt.py +31 -17
- vortex/sessions.py +72 -39
- vortex/syntax/__init__.py +1 -1
- vortex/syntax/stdattrs.py +410 -171
- vortex/syntax/stddeco.py +31 -22
- vortex/toolbox.py +327 -192
- vortex/tools/__init__.py +11 -2
- vortex/tools/actions.py +110 -121
- vortex/tools/addons.py +111 -92
- vortex/tools/arm.py +42 -22
- vortex/tools/compression.py +72 -69
- vortex/tools/date.py +11 -4
- vortex/tools/delayedactions.py +242 -132
- vortex/tools/env.py +75 -47
- vortex/tools/folder.py +342 -171
- vortex/tools/grib.py +341 -162
- vortex/tools/lfi.py +423 -216
- vortex/tools/listings.py +109 -40
- vortex/tools/names.py +218 -156
- vortex/tools/net.py +655 -299
- vortex/tools/parallelism.py +93 -61
- vortex/tools/prestaging.py +55 -31
- vortex/tools/schedulers.py +172 -105
- vortex/tools/services.py +403 -334
- vortex/tools/storage.py +293 -358
- vortex/tools/surfex.py +24 -24
- vortex/tools/systems.py +1234 -643
- vortex/tools/targets.py +156 -100
- vortex/util/__init__.py +1 -1
- vortex/util/config.py +378 -327
- vortex/util/empty.py +2 -2
- vortex/util/helpers.py +56 -24
- vortex/util/introspection.py +18 -12
- vortex/util/iosponge.py +8 -4
- vortex/util/roles.py +4 -6
- vortex/util/storefunctions.py +39 -13
- vortex/util/structs.py +3 -3
- vortex/util/worker.py +29 -17
- vortex_nwp-2.1.0.dist-info/METADATA +67 -0
- vortex_nwp-2.1.0.dist-info/RECORD +144 -0
- {vortex_nwp-2.0.0b1.dist-info → vortex_nwp-2.1.0.dist-info}/WHEEL +1 -1
- vortex/layout/appconf.py +0 -109
- vortex/layout/jobs.py +0 -1276
- vortex/layout/nodes.py +0 -1424
- vortex/layout/subjobs.py +0 -464
- vortex_nwp-2.0.0b1.dist-info/METADATA +0 -50
- vortex_nwp-2.0.0b1.dist-info/RECORD +0 -146
- {vortex_nwp-2.0.0b1.dist-info → vortex_nwp-2.1.0.dist-info/licenses}/LICENSE +0 -0
- {vortex_nwp-2.0.0b1.dist-info → vortex_nwp-2.1.0.dist-info}/top_level.txt +0 -0
vortex/nwp/util/usetnt.py
CHANGED
|
@@ -14,7 +14,7 @@ from vortex.util.roles import setrole
|
|
|
14
14
|
|
|
15
15
|
logger = loggers.getLogger(__name__)
|
|
16
16
|
|
|
17
|
-
tnt_checker = ExternalCodeImportChecker(
|
|
17
|
+
tnt_checker = ExternalCodeImportChecker("thenamelisttool")
|
|
18
18
|
with tnt_checker as tnt_register:
|
|
19
19
|
import thenamelisttool
|
|
20
20
|
|
|
@@ -48,40 +48,54 @@ def compose_nam(options):
|
|
|
48
48
|
|
|
49
49
|
:rtype: A file like object
|
|
50
50
|
"""
|
|
51
|
-
rhdict = options.get(
|
|
52
|
-
source = rhdict[
|
|
51
|
+
rhdict = options.get("rhandler", None)
|
|
52
|
+
source = rhdict["resource"].get("source", None)
|
|
53
53
|
if source is None:
|
|
54
54
|
logger.error("Inapropriate type of resource. Got:\n%s", rhdict)
|
|
55
|
-
raise FunctionStoreCallbackError(
|
|
55
|
+
raise FunctionStoreCallbackError("Inapropriate type of resources.")
|
|
56
56
|
|
|
57
57
|
t = sessions.current()
|
|
58
58
|
|
|
59
59
|
def _get_pack_adress(role):
|
|
60
60
|
role = setrole(role)
|
|
61
|
-
packlist = [
|
|
61
|
+
packlist = [
|
|
62
|
+
sec.rh for sec in t.context.sequence.filtered_inputs(role=role)
|
|
63
|
+
]
|
|
62
64
|
if len(packlist) != 1:
|
|
63
65
|
logger.error("The number of namelist packs with role=%s is not 1.")
|
|
64
|
-
raise FunctionStoreCallbackError(
|
|
66
|
+
raise FunctionStoreCallbackError(
|
|
67
|
+
"Incorrect number of namelist packs."
|
|
68
|
+
)
|
|
65
69
|
packrh = packlist[0]
|
|
66
|
-
if packrh.resource.realkind !=
|
|
67
|
-
logger.error(
|
|
68
|
-
|
|
69
|
-
|
|
70
|
+
if packrh.resource.realkind != "namelistpack":
|
|
71
|
+
logger.error(
|
|
72
|
+
"Incorrect resource type for role %s. Resource handler:\n%s",
|
|
73
|
+
role,
|
|
74
|
+
packrh.icdard(),
|
|
75
|
+
)
|
|
76
|
+
raise FunctionStoreCallbackError("Incorrect resource type.")
|
|
70
77
|
if not packrh.container.filled:
|
|
71
|
-
logger.error(
|
|
72
|
-
|
|
78
|
+
logger.error(
|
|
79
|
+
"The resource handler's container is not filled for role %s",
|
|
80
|
+
role,
|
|
81
|
+
)
|
|
82
|
+
raise FunctionStoreCallbackError("RH container is not filled.")
|
|
73
83
|
return packrh.container.abspath
|
|
74
84
|
|
|
75
|
-
nampack_path = _get_pack_adress(
|
|
76
|
-
|
|
77
|
-
|
|
85
|
+
nampack_path = _get_pack_adress(
|
|
86
|
+
options.get("nampack", "Main Namelist Pack")
|
|
87
|
+
)
|
|
88
|
+
dirpack_role = options.get("dirpack", None)
|
|
89
|
+
dirpack_path = (
|
|
90
|
+
_get_pack_adress(dirpack_role) if dirpack_role else nampack_path
|
|
91
|
+
)
|
|
78
92
|
|
|
79
93
|
out_io = io.BytesIO()
|
|
80
94
|
thenamelisttool.util.compose_namelist(
|
|
81
|
-
t.sh.path.join(dirpack_path, source +
|
|
95
|
+
t.sh.path.join(dirpack_path, source + ".yaml"),
|
|
82
96
|
sourcenam_directory=nampack_path,
|
|
83
97
|
sorting=thenamelisttool.namadapter.FIRST_ORDER_SORTING,
|
|
84
98
|
squeeze=False,
|
|
85
|
-
fhoutput=codecs.getwriter(
|
|
99
|
+
fhoutput=codecs.getwriter("utf-8")(out_io),
|
|
86
100
|
)
|
|
87
101
|
return out_io
|
vortex/sessions.py
CHANGED
|
@@ -17,7 +17,7 @@ import footprints
|
|
|
17
17
|
|
|
18
18
|
from vortex.tools.env import Environment
|
|
19
19
|
|
|
20
|
-
from vortex import gloves #
|
|
20
|
+
from vortex import gloves as gloves # footprints import
|
|
21
21
|
from vortex.layout import contexts
|
|
22
22
|
|
|
23
23
|
#: No automatic export
|
|
@@ -28,9 +28,13 @@ logger = loggers.getLogger(__name__)
|
|
|
28
28
|
|
|
29
29
|
# Module Interface
|
|
30
30
|
|
|
31
|
+
|
|
31
32
|
def get(**kw):
|
|
32
33
|
"""Return actual session ticket object matching description."""
|
|
33
|
-
if
|
|
34
|
+
if (
|
|
35
|
+
kw.get("tag", "current") == "current"
|
|
36
|
+
and Ticket.tag_focus() is not None
|
|
37
|
+
):
|
|
34
38
|
return current()
|
|
35
39
|
else:
|
|
36
40
|
return Ticket(**kw)
|
|
@@ -73,7 +77,7 @@ def getglove(**kw):
|
|
|
73
77
|
|
|
74
78
|
def system(**kw):
|
|
75
79
|
"""Returns the system associated to the current ticket."""
|
|
76
|
-
return get(tag=kw.pop(
|
|
80
|
+
return get(tag=kw.pop("tag", Ticket.tag_focus())).system(**kw)
|
|
77
81
|
|
|
78
82
|
|
|
79
83
|
# noinspection PyShadowingBuiltins
|
|
@@ -94,17 +98,19 @@ class Ticket(getbytag.GetByTag):
|
|
|
94
98
|
Default session ticket class, defined by tag.
|
|
95
99
|
"""
|
|
96
100
|
|
|
97
|
-
_tag_default =
|
|
98
|
-
|
|
99
|
-
def __init__(
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
101
|
+
_tag_default = "root"
|
|
102
|
+
|
|
103
|
+
def __init__(
|
|
104
|
+
self,
|
|
105
|
+
active=False,
|
|
106
|
+
topenv=None,
|
|
107
|
+
glove=None,
|
|
108
|
+
context=None,
|
|
109
|
+
datastore=None,
|
|
110
|
+
prompt="Vortex:",
|
|
111
|
+
):
|
|
106
112
|
self.prompt = prompt
|
|
107
|
-
self.line = "\n" +
|
|
113
|
+
self.line = "\n" + "-" * 100 + "\n"
|
|
108
114
|
|
|
109
115
|
self._started = date.now()
|
|
110
116
|
self._closed = 0
|
|
@@ -120,18 +126,24 @@ class Ticket(getbytag.GetByTag):
|
|
|
120
126
|
else:
|
|
121
127
|
self._glove = getglove()
|
|
122
128
|
|
|
123
|
-
logger.debug(
|
|
129
|
+
logger.debug("New session system is %s", self.system())
|
|
124
130
|
|
|
125
131
|
self._rundir = self.sh.getcwd()
|
|
126
132
|
|
|
127
|
-
logger.debug(
|
|
133
|
+
logger.debug("Open session %s %s", self.tag, self)
|
|
128
134
|
|
|
129
135
|
if datastore is None:
|
|
130
|
-
datastore = DataStore(
|
|
136
|
+
datastore = DataStore(
|
|
137
|
+
default_picklefile="{:s}_session_datastore.pickled".format(
|
|
138
|
+
self.tag
|
|
139
|
+
)
|
|
140
|
+
)
|
|
131
141
|
self._dstore = datastore
|
|
132
142
|
|
|
133
143
|
if context is None:
|
|
134
|
-
context = contexts.Context(
|
|
144
|
+
context = contexts.Context(
|
|
145
|
+
tag=self.tag, topenv=self._topenv, path=self.path
|
|
146
|
+
)
|
|
135
147
|
self._last_context = context
|
|
136
148
|
|
|
137
149
|
if active:
|
|
@@ -144,12 +156,20 @@ class Ticket(getbytag.GetByTag):
|
|
|
144
156
|
def _set_rundir(self, path):
|
|
145
157
|
"""Set a new default rundir for this session."""
|
|
146
158
|
if self._rundir:
|
|
147
|
-
logger.warning(
|
|
159
|
+
logger.warning(
|
|
160
|
+
"Session <%s> is changing its working directory <%s>",
|
|
161
|
+
self.tag,
|
|
162
|
+
self._rundir,
|
|
163
|
+
)
|
|
148
164
|
if self.sh.path.isdir(path):
|
|
149
165
|
self._rundir = path
|
|
150
|
-
logger.info(
|
|
166
|
+
logger.info("Session <%s> set rundir <%s>", self.tag, self._rundir)
|
|
151
167
|
else:
|
|
152
|
-
logger.error(
|
|
168
|
+
logger.error(
|
|
169
|
+
"Try to change session <%s> to invalid path <%s>",
|
|
170
|
+
self.tag,
|
|
171
|
+
path,
|
|
172
|
+
)
|
|
153
173
|
|
|
154
174
|
rundir = property(_get_rundir, _set_rundir)
|
|
155
175
|
|
|
@@ -210,11 +230,14 @@ class Ticket(getbytag.GetByTag):
|
|
|
210
230
|
Returns the current OS handler used or set a new one according
|
|
211
231
|
to ``kw`` dictionary-like arguments.
|
|
212
232
|
"""
|
|
213
|
-
refill = kw.pop(
|
|
233
|
+
refill = kw.pop("refill", False)
|
|
214
234
|
if not self._system or kw or refill:
|
|
215
235
|
self._system = footprints.proxy.system(glove=self.glove, **kw)
|
|
216
236
|
if not self._system:
|
|
217
|
-
logger.critical(
|
|
237
|
+
logger.critical(
|
|
238
|
+
"Could not load a system object with description %s",
|
|
239
|
+
str(kw),
|
|
240
|
+
)
|
|
218
241
|
return self._system
|
|
219
242
|
|
|
220
243
|
def duration(self):
|
|
@@ -237,27 +260,31 @@ class Ticket(getbytag.GetByTag):
|
|
|
237
260
|
def close(self):
|
|
238
261
|
"""Closes the current session."""
|
|
239
262
|
if self.closed:
|
|
240
|
-
logger.warning(
|
|
263
|
+
logger.warning(
|
|
264
|
+
"Session %s already closed at %s", self.tag, self.closed
|
|
265
|
+
)
|
|
241
266
|
else:
|
|
242
267
|
self._closed = date.now()
|
|
243
|
-
logger.debug(
|
|
268
|
+
logger.debug(
|
|
269
|
+
"Close session %s ( time = %s )", self.tag, self.duration()
|
|
270
|
+
)
|
|
244
271
|
|
|
245
272
|
@property
|
|
246
273
|
def path(self):
|
|
247
|
-
return
|
|
274
|
+
return "/" + self.tag
|
|
248
275
|
|
|
249
276
|
@property
|
|
250
277
|
def subcontexts(self):
|
|
251
278
|
"""The current contexts binded to this session."""
|
|
252
|
-
rootpath = self.path +
|
|
279
|
+
rootpath = self.path + "/"
|
|
253
280
|
return [x for x in contexts.values() if x.path.startswith(rootpath)]
|
|
254
281
|
|
|
255
282
|
def exit(self):
|
|
256
283
|
"""Exit from the current session."""
|
|
257
284
|
ok = True
|
|
258
|
-
logger.debug(
|
|
285
|
+
logger.debug("Exit session %s %s", self.tag, self)
|
|
259
286
|
for kid in self.subcontexts:
|
|
260
|
-
logger.debug(
|
|
287
|
+
logger.debug("Exit from context %s", kid)
|
|
261
288
|
ok = ok and kid.exit()
|
|
262
289
|
if self.opened:
|
|
263
290
|
self.close()
|
|
@@ -295,20 +322,26 @@ class Ticket(getbytag.GetByTag):
|
|
|
295
322
|
"""
|
|
296
323
|
Returns the logging level.
|
|
297
324
|
"""
|
|
298
|
-
v_logger = loggers.getLogger(
|
|
325
|
+
v_logger = loggers.getLogger("vortex")
|
|
299
326
|
return logging.getLevelName(v_logger.getEffectiveLevel())
|
|
300
327
|
|
|
301
|
-
def idcard(self, indent=
|
|
328
|
+
def idcard(self, indent="+ "):
|
|
302
329
|
"""Returns a printable description of the current session."""
|
|
303
|
-
card = "\n".join(
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
330
|
+
card = "\n".join(
|
|
331
|
+
(
|
|
332
|
+
"{0}Name = {1:s}",
|
|
333
|
+
"{0}Started = {2!s}",
|
|
334
|
+
"{0}Opened = {3!s}",
|
|
335
|
+
"{0}Duration = {4!s}",
|
|
336
|
+
"{0}Loglevel = {5:s}",
|
|
337
|
+
)
|
|
338
|
+
).format(
|
|
310
339
|
indent,
|
|
311
|
-
self.tag,
|
|
340
|
+
self.tag,
|
|
341
|
+
self.started,
|
|
342
|
+
self.opened,
|
|
343
|
+
self.duration(),
|
|
344
|
+
self.loglevel,
|
|
312
345
|
)
|
|
313
346
|
return card
|
|
314
347
|
|
|
@@ -333,7 +366,7 @@ class Ticket(getbytag.GetByTag):
|
|
|
333
366
|
obj.catch_focus()
|
|
334
367
|
return obj
|
|
335
368
|
else:
|
|
336
|
-
logger.error(
|
|
369
|
+
logger.error("Try to switch to an undefined session: %s", tag)
|
|
337
370
|
return None
|
|
338
371
|
|
|
339
372
|
def __del__(self):
|
vortex/syntax/__init__.py
CHANGED