vortex-nwp 2.0.0b1__py3-none-any.whl → 2.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- vortex/__init__.py +75 -47
- vortex/algo/__init__.py +3 -2
- vortex/algo/components.py +944 -618
- vortex/algo/mpitools.py +802 -497
- vortex/algo/mpitools_templates/__init__.py +1 -0
- vortex/algo/serversynctools.py +34 -33
- vortex/config.py +19 -22
- vortex/data/__init__.py +9 -3
- vortex/data/abstractstores.py +593 -655
- vortex/data/containers.py +217 -162
- vortex/data/contents.py +65 -39
- vortex/data/executables.py +93 -102
- vortex/data/flow.py +40 -34
- vortex/data/geometries.py +228 -132
- vortex/data/handlers.py +436 -227
- vortex/data/outflow.py +15 -15
- vortex/data/providers.py +185 -163
- vortex/data/resources.py +48 -42
- vortex/data/stores.py +540 -417
- vortex/data/sync_templates/__init__.py +0 -0
- vortex/gloves.py +114 -87
- vortex/layout/__init__.py +1 -8
- vortex/layout/contexts.py +150 -84
- vortex/layout/dataflow.py +353 -202
- vortex/layout/monitor.py +264 -128
- vortex/nwp/__init__.py +5 -2
- vortex/nwp/algo/__init__.py +14 -5
- vortex/nwp/algo/assim.py +205 -151
- vortex/nwp/algo/clim.py +683 -517
- vortex/nwp/algo/coupling.py +447 -225
- vortex/nwp/algo/eda.py +437 -229
- vortex/nwp/algo/eps.py +403 -231
- vortex/nwp/algo/forecasts.py +416 -275
- vortex/nwp/algo/fpserver.py +683 -307
- vortex/nwp/algo/ifsnaming.py +205 -145
- vortex/nwp/algo/ifsroot.py +215 -122
- vortex/nwp/algo/monitoring.py +137 -76
- vortex/nwp/algo/mpitools.py +330 -190
- vortex/nwp/algo/odbtools.py +637 -353
- vortex/nwp/algo/oopsroot.py +454 -273
- vortex/nwp/algo/oopstests.py +90 -56
- vortex/nwp/algo/request.py +287 -206
- vortex/nwp/algo/stdpost.py +878 -522
- vortex/nwp/data/__init__.py +22 -4
- vortex/nwp/data/assim.py +125 -137
- vortex/nwp/data/boundaries.py +121 -68
- vortex/nwp/data/climfiles.py +193 -211
- vortex/nwp/data/configfiles.py +73 -69
- vortex/nwp/data/consts.py +426 -401
- vortex/nwp/data/ctpini.py +59 -43
- vortex/nwp/data/diagnostics.py +94 -66
- vortex/nwp/data/eda.py +50 -51
- vortex/nwp/data/eps.py +195 -146
- vortex/nwp/data/executables.py +440 -434
- vortex/nwp/data/fields.py +63 -48
- vortex/nwp/data/gridfiles.py +183 -111
- vortex/nwp/data/logs.py +250 -217
- vortex/nwp/data/modelstates.py +180 -151
- vortex/nwp/data/monitoring.py +72 -99
- vortex/nwp/data/namelists.py +254 -202
- vortex/nwp/data/obs.py +400 -308
- vortex/nwp/data/oopsexec.py +22 -20
- vortex/nwp/data/providers.py +90 -65
- vortex/nwp/data/query.py +71 -82
- vortex/nwp/data/stores.py +49 -36
- vortex/nwp/data/surfex.py +136 -137
- vortex/nwp/syntax/__init__.py +1 -1
- vortex/nwp/syntax/stdattrs.py +173 -111
- vortex/nwp/tools/__init__.py +2 -2
- vortex/nwp/tools/addons.py +22 -17
- vortex/nwp/tools/agt.py +24 -12
- vortex/nwp/tools/bdap.py +16 -5
- vortex/nwp/tools/bdcp.py +4 -1
- vortex/nwp/tools/bdm.py +3 -0
- vortex/nwp/tools/bdmp.py +14 -9
- vortex/nwp/tools/conftools.py +728 -378
- vortex/nwp/tools/drhook.py +12 -8
- vortex/nwp/tools/grib.py +65 -39
- vortex/nwp/tools/gribdiff.py +22 -17
- vortex/nwp/tools/ifstools.py +82 -42
- vortex/nwp/tools/igastuff.py +167 -143
- vortex/nwp/tools/mars.py +14 -2
- vortex/nwp/tools/odb.py +234 -125
- vortex/nwp/tools/partitioning.py +61 -37
- vortex/nwp/tools/satrad.py +27 -12
- vortex/nwp/util/async.py +83 -55
- vortex/nwp/util/beacon.py +10 -10
- vortex/nwp/util/diffpygram.py +174 -86
- vortex/nwp/util/ens.py +144 -63
- vortex/nwp/util/hooks.py +30 -19
- vortex/nwp/util/taskdeco.py +28 -24
- vortex/nwp/util/usepygram.py +278 -172
- vortex/nwp/util/usetnt.py +31 -17
- vortex/sessions.py +72 -39
- vortex/syntax/__init__.py +1 -1
- vortex/syntax/stdattrs.py +410 -171
- vortex/syntax/stddeco.py +31 -22
- vortex/toolbox.py +327 -192
- vortex/tools/__init__.py +11 -2
- vortex/tools/actions.py +110 -121
- vortex/tools/addons.py +111 -92
- vortex/tools/arm.py +42 -22
- vortex/tools/compression.py +72 -69
- vortex/tools/date.py +11 -4
- vortex/tools/delayedactions.py +242 -132
- vortex/tools/env.py +75 -47
- vortex/tools/folder.py +342 -171
- vortex/tools/grib.py +341 -162
- vortex/tools/lfi.py +423 -216
- vortex/tools/listings.py +109 -40
- vortex/tools/names.py +218 -156
- vortex/tools/net.py +655 -299
- vortex/tools/parallelism.py +93 -61
- vortex/tools/prestaging.py +55 -31
- vortex/tools/schedulers.py +172 -105
- vortex/tools/services.py +403 -334
- vortex/tools/storage.py +293 -358
- vortex/tools/surfex.py +24 -24
- vortex/tools/systems.py +1234 -643
- vortex/tools/targets.py +156 -100
- vortex/util/__init__.py +1 -1
- vortex/util/config.py +378 -327
- vortex/util/empty.py +2 -2
- vortex/util/helpers.py +56 -24
- vortex/util/introspection.py +18 -12
- vortex/util/iosponge.py +8 -4
- vortex/util/roles.py +4 -6
- vortex/util/storefunctions.py +39 -13
- vortex/util/structs.py +3 -3
- vortex/util/worker.py +29 -17
- vortex_nwp-2.1.0.dist-info/METADATA +67 -0
- vortex_nwp-2.1.0.dist-info/RECORD +144 -0
- {vortex_nwp-2.0.0b1.dist-info → vortex_nwp-2.1.0.dist-info}/WHEEL +1 -1
- vortex/layout/appconf.py +0 -109
- vortex/layout/jobs.py +0 -1276
- vortex/layout/nodes.py +0 -1424
- vortex/layout/subjobs.py +0 -464
- vortex_nwp-2.0.0b1.dist-info/METADATA +0 -50
- vortex_nwp-2.0.0b1.dist-info/RECORD +0 -146
- {vortex_nwp-2.0.0b1.dist-info → vortex_nwp-2.1.0.dist-info/licenses}/LICENSE +0 -0
- {vortex_nwp-2.0.0b1.dist-info → vortex_nwp-2.1.0.dist-info}/top_level.txt +0 -0
vortex/nwp/syntax/stdattrs.py
CHANGED
|
@@ -4,25 +4,32 @@ of attributes description that could be used in the footprint definition of any
|
|
|
4
4
|
class which follow the :class:`footprints.Footprint` syntax.
|
|
5
5
|
"""
|
|
6
6
|
|
|
7
|
-
|
|
7
|
+
import re
|
|
8
|
+
from functools import total_ordering
|
|
9
|
+
|
|
8
10
|
import footprints
|
|
11
|
+
from bronx.stdtypes.date import Time
|
|
12
|
+
|
|
13
|
+
from vortex.syntax.stddeco import namebuilding_append
|
|
9
14
|
|
|
10
|
-
#: Export
|
|
11
|
-
__all__ = []
|
|
15
|
+
#: Export some new class for attributes in footprint objects, eg : GenvKey
|
|
16
|
+
__all__ = ["GenvKey", "GenvDomain"]
|
|
12
17
|
|
|
13
18
|
#: Usual Footprint for a single member (in an algo Component)
|
|
14
19
|
a_algo_member = dict(
|
|
15
|
-
info=(
|
|
16
|
-
|
|
20
|
+
info=(
|
|
21
|
+
"The current member's number "
|
|
22
|
+
+ "(may be omitted in deterministic configurations)."
|
|
23
|
+
),
|
|
17
24
|
optional=True,
|
|
18
|
-
type=int
|
|
25
|
+
type=int,
|
|
19
26
|
)
|
|
20
27
|
|
|
21
28
|
#: Usual Footprint of the ``outputid`` attribute.
|
|
22
29
|
algo_member = footprints.Footprint(attr=dict(member=a_algo_member))
|
|
23
30
|
|
|
24
31
|
#: Known OOPS testcomponent ``run``
|
|
25
|
-
known_oops_testcomponent_runs = [
|
|
32
|
+
known_oops_testcomponent_runs = ["ootestcomponent", "testcomponent", "testvar"]
|
|
26
33
|
|
|
27
34
|
#: Usual definition of the ``run`` attribute for OOPS binaries.
|
|
28
35
|
a_oops_run = dict(
|
|
@@ -30,41 +37,51 @@ a_oops_run = dict(
|
|
|
30
37
|
optional=False,
|
|
31
38
|
)
|
|
32
39
|
#: Usual Footprint of the ``run`` attribute for OOPS binaries.
|
|
33
|
-
oops_run = footprints.Footprint(
|
|
40
|
+
oops_run = footprints.Footprint(
|
|
41
|
+
info="OOPS kind of run", attr=dict(run=a_oops_run)
|
|
42
|
+
)
|
|
34
43
|
|
|
35
44
|
#: Usual definition of the ``test_type`` attribute.
|
|
36
45
|
a_oops_test_type = dict(
|
|
37
|
-
info=
|
|
46
|
+
info="Sub-test or family of sub-tests to be ran.",
|
|
38
47
|
optional=False,
|
|
39
48
|
)
|
|
40
49
|
#: Usual Footprint of the ``test_type`` attribute.
|
|
41
|
-
oops_test_type = footprints.Footprint(
|
|
50
|
+
oops_test_type = footprints.Footprint(
|
|
51
|
+
info="OOPS type of test", attr=dict(test_type=a_oops_test_type)
|
|
52
|
+
)
|
|
42
53
|
|
|
43
54
|
#: Usual definition of the ``expected_target`` attribute.
|
|
44
55
|
an_oops_expected_target = dict(
|
|
45
|
-
info=(
|
|
56
|
+
info=("Expected target for the test success"),
|
|
46
57
|
type=footprints.FPDict,
|
|
47
58
|
optional=True,
|
|
48
|
-
default=None
|
|
59
|
+
default=None,
|
|
49
60
|
)
|
|
50
61
|
#: Usual Footprint of the ``expected_target`` attribute.
|
|
51
|
-
oops_expected_target = footprints.Footprint(
|
|
62
|
+
oops_expected_target = footprints.Footprint(
|
|
63
|
+
attr=dict(expected_target=an_oops_expected_target)
|
|
64
|
+
)
|
|
52
65
|
|
|
53
66
|
#: Usual Footprint of a combined lists of members and terms
|
|
54
67
|
oops_members_terms_lists = footprints.Footprint(
|
|
55
68
|
info="Abstract footprint for a members/terms list.",
|
|
56
69
|
attr=dict(
|
|
57
70
|
members=dict(
|
|
58
|
-
info=
|
|
71
|
+
info="A list of members.",
|
|
59
72
|
type=footprints.FPList,
|
|
60
73
|
),
|
|
61
74
|
terms=dict(
|
|
62
|
-
info=
|
|
75
|
+
info="A list of effective terms.",
|
|
63
76
|
type=footprints.FPList,
|
|
64
77
|
optional=True,
|
|
65
|
-
default=footprints.FPList(
|
|
78
|
+
default=footprints.FPList(
|
|
79
|
+
[
|
|
80
|
+
Time(0),
|
|
81
|
+
]
|
|
82
|
+
),
|
|
66
83
|
),
|
|
67
|
-
)
|
|
84
|
+
),
|
|
68
85
|
)
|
|
69
86
|
|
|
70
87
|
#: Usual definition of the ``outputid`` attribute
|
|
@@ -79,15 +96,20 @@ outputid = footprints.Footprint(attr=dict(outputid=a_outputid))
|
|
|
79
96
|
|
|
80
97
|
def _apply_outputid(cls):
|
|
81
98
|
"""Decorator that tweak the class in order to add OUTPUTID on the namelist"""
|
|
82
|
-
orig_pnd = getattr(cls,
|
|
99
|
+
orig_pnd = getattr(cls, "prepare_namelist_delta", None)
|
|
83
100
|
if orig_pnd is None:
|
|
84
|
-
raise ImportError(
|
|
101
|
+
raise ImportError(
|
|
102
|
+
"_apply_outputid can not be applied on {!s}".format(cls)
|
|
103
|
+
)
|
|
85
104
|
|
|
86
105
|
def prepare_namelist_delta(self, rh, namcontents, namlocal):
|
|
87
106
|
namw = orig_pnd(self, rh, namcontents, namlocal)
|
|
88
|
-
if self.outputid is not None and any(
|
|
89
|
-
|
|
90
|
-
|
|
107
|
+
if self.outputid is not None and any(
|
|
108
|
+
["OUTPUTID" in nam_b.macros() for nam_b in namcontents.values()]
|
|
109
|
+
):
|
|
110
|
+
self._set_nam_macro(
|
|
111
|
+
namcontents, namlocal, "OUTPUTID", self.outputid
|
|
112
|
+
)
|
|
91
113
|
namw = True
|
|
92
114
|
return namw
|
|
93
115
|
|
|
@@ -96,34 +118,31 @@ def _apply_outputid(cls):
|
|
|
96
118
|
|
|
97
119
|
|
|
98
120
|
#: Decorated footprint for the ``outputid`` attribute
|
|
99
|
-
outputid_deco = footprints.DecorativeFootprint(
|
|
100
|
-
|
|
121
|
+
outputid_deco = footprints.DecorativeFootprint(
|
|
122
|
+
outputid,
|
|
123
|
+
decorator=[
|
|
124
|
+
_apply_outputid,
|
|
125
|
+
],
|
|
126
|
+
)
|
|
101
127
|
|
|
102
128
|
|
|
103
129
|
def show():
|
|
104
130
|
"""Returns available items and their type."""
|
|
105
131
|
dmod = globals()
|
|
106
|
-
for stda in sorted(
|
|
107
|
-
|
|
108
|
-
|
|
132
|
+
for stda in sorted(
|
|
133
|
+
filter(
|
|
134
|
+
lambda x: x.startswith("a_")
|
|
135
|
+
or isinstance(dmod[x], footprints.Footprint),
|
|
136
|
+
dmod.keys(),
|
|
137
|
+
)
|
|
138
|
+
):
|
|
139
|
+
print(
|
|
140
|
+
"{} ( {} ) :\n {}\n".format(
|
|
141
|
+
stda, type(dmod[stda]).__name__, dmod[stda]
|
|
142
|
+
)
|
|
143
|
+
)
|
|
109
144
|
|
|
110
145
|
|
|
111
|
-
"""
|
|
112
|
-
This module provides some pre-defined attributes descriptions or combined sets
|
|
113
|
-
of attributes description that could be used in the footprint definition of any
|
|
114
|
-
class which follow the :class:`footprints.Footprint` syntax.
|
|
115
|
-
"""
|
|
116
|
-
|
|
117
|
-
import re
|
|
118
|
-
from functools import total_ordering
|
|
119
|
-
|
|
120
|
-
import footprints
|
|
121
|
-
|
|
122
|
-
from vortex.syntax.stddeco import namebuilding_append
|
|
123
|
-
|
|
124
|
-
#: Export some new class for attributes in footprint objects, eg : GenvKey
|
|
125
|
-
__all__ = ['GenvKey', 'GenvDomain']
|
|
126
|
-
|
|
127
146
|
domain_remap = dict()
|
|
128
147
|
|
|
129
148
|
|
|
@@ -141,18 +160,18 @@ class GenvKey(str):
|
|
|
141
160
|
|
|
142
161
|
def __new__(cls, value):
|
|
143
162
|
"""Proxy to ``str.__new___`` with attributes inside brackets translated to lower case."""
|
|
144
|
-
return str.__new__(cls, re.sub(r
|
|
163
|
+
return str.__new__(cls, re.sub(r"\[\w+\]", _lowerattr, value.upper()))
|
|
145
164
|
|
|
146
165
|
|
|
147
|
-
a_gvar = dict(
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
166
|
+
a_gvar = dict(
|
|
167
|
+
info="The key that identifies the resource in the Genv database.",
|
|
168
|
+
type=GenvKey,
|
|
169
|
+
optional=True,
|
|
170
|
+
doc_visibility=footprints.doc.visibility.ADVANCED,
|
|
171
|
+
)
|
|
152
172
|
|
|
153
173
|
#: Usual definition of the ``genv`` attribute.
|
|
154
|
-
gvar = footprints.Footprint(info=
|
|
155
|
-
attr=dict(gvar=a_gvar))
|
|
174
|
+
gvar = footprints.Footprint(info="A GENV access key", attr=dict(gvar=a_gvar))
|
|
156
175
|
|
|
157
176
|
|
|
158
177
|
class GenvDomain(str):
|
|
@@ -166,15 +185,18 @@ class GenvDomain(str):
|
|
|
166
185
|
return str.__new__(cls, domain_remap.get(value, value))
|
|
167
186
|
|
|
168
187
|
|
|
169
|
-
a_gdomain = dict(
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
188
|
+
a_gdomain = dict(
|
|
189
|
+
info="The resource's geographical domain name in the Genv database.",
|
|
190
|
+
type=GenvDomain,
|
|
191
|
+
optional=True,
|
|
192
|
+
default="[geometry::area]",
|
|
193
|
+
doc_visibility=footprints.doc.visibility.ADVANCED,
|
|
194
|
+
)
|
|
174
195
|
|
|
175
196
|
#: Usual definition of the ``gdomain`` attribute.
|
|
176
|
-
gdomain = footprints.Footprint(
|
|
177
|
-
|
|
197
|
+
gdomain = footprints.Footprint(
|
|
198
|
+
info="A domain name in GCO convention", attr=dict(gdomain=a_gdomain)
|
|
199
|
+
)
|
|
178
200
|
|
|
179
201
|
|
|
180
202
|
@total_ordering
|
|
@@ -187,22 +209,33 @@ class ArpIfsSimplifiedCycle:
|
|
|
187
209
|
|
|
188
210
|
It can be used in a footprint specification.
|
|
189
211
|
"""
|
|
190
|
-
|
|
212
|
+
|
|
213
|
+
_cy_re = re.compile(
|
|
214
|
+
r"(?:u(?:env|get):)?(?:cy|al)(\d+)(?:t(\d{1,3}))?(?=_|@|\.|$)(?:.*?(?:[_-]op(\d{1,3})))?"
|
|
215
|
+
)
|
|
191
216
|
_hash_shift = 10000
|
|
192
217
|
|
|
193
218
|
def __init__(self, cyclestr):
|
|
194
219
|
cy_match = self._cy_re.match(cyclestr)
|
|
195
220
|
if cy_match:
|
|
196
221
|
self._number = int(cy_match.group(1))
|
|
197
|
-
self._toulouse = (
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
222
|
+
self._toulouse = (
|
|
223
|
+
int(cy_match.group(2)) + 1
|
|
224
|
+
if cy_match.group(2) is not None
|
|
225
|
+
else 0
|
|
226
|
+
)
|
|
227
|
+
self._op = (
|
|
228
|
+
int(cy_match.group(3)) + 1
|
|
229
|
+
if cy_match.group(3) is not None
|
|
230
|
+
else 0
|
|
231
|
+
)
|
|
201
232
|
else:
|
|
202
|
-
raise ValueError(
|
|
233
|
+
raise ValueError("Malformed cycle: {}".format(cyclestr))
|
|
203
234
|
|
|
204
235
|
def __hash__(self):
|
|
205
|
-
return (
|
|
236
|
+
return (
|
|
237
|
+
self._number * self._hash_shift + self._toulouse
|
|
238
|
+
) * self._hash_shift + self._op
|
|
206
239
|
|
|
207
240
|
def __eq__(self, other):
|
|
208
241
|
if not isinstance(other, ArpIfsSimplifiedCycle):
|
|
@@ -218,39 +251,51 @@ class ArpIfsSimplifiedCycle:
|
|
|
218
251
|
return hash(self) > hash(other)
|
|
219
252
|
|
|
220
253
|
def __str__(self):
|
|
221
|
-
return (
|
|
222
|
-
|
|
223
|
-
|
|
254
|
+
return (
|
|
255
|
+
"cy{:d}".format(self._number)
|
|
256
|
+
+ ("t{:d}".format(self._toulouse - 1) if self._toulouse else "")
|
|
257
|
+
+ ("_op{:d}".format(self._op - 1) if self._op else "")
|
|
258
|
+
)
|
|
224
259
|
|
|
225
260
|
def __repr__(self):
|
|
226
|
-
return
|
|
261
|
+
return "<{} | {!s}>".format(
|
|
262
|
+
object.__repr__(self).lstrip("<").rstrip(">"), self
|
|
263
|
+
)
|
|
227
264
|
|
|
228
265
|
def export_dict(self):
|
|
229
266
|
"""The pure dict/json output is the raw integer"""
|
|
230
267
|
return str(self)
|
|
231
268
|
|
|
232
269
|
|
|
233
|
-
a_arpifs_cycle = dict(
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
270
|
+
a_arpifs_cycle = dict(
|
|
271
|
+
info="An Arpege/IFS cycle name",
|
|
272
|
+
type=ArpIfsSimplifiedCycle,
|
|
273
|
+
optional=True,
|
|
274
|
+
default="cy40", # For "old" Olive configurations to keep working
|
|
275
|
+
)
|
|
238
276
|
|
|
239
277
|
#: Usual definition of the ``cycle`` attribute.
|
|
240
|
-
arpifs_cycle = footprints.Footprint(
|
|
241
|
-
|
|
278
|
+
arpifs_cycle = footprints.Footprint(
|
|
279
|
+
info="An abstract arpifs_cycle in GCO convention",
|
|
280
|
+
attr=dict(cycle=a_arpifs_cycle),
|
|
281
|
+
)
|
|
242
282
|
|
|
243
|
-
uget_sloppy_id_regex = re.compile(
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
uget_id_regex =
|
|
283
|
+
uget_sloppy_id_regex = re.compile(
|
|
284
|
+
r"(?P<shortuget>(?P<id>\S+)@(?P<location>[-\w]+))"
|
|
285
|
+
)
|
|
286
|
+
uget_id_regex = (
|
|
287
|
+
r"(?P<fulluget>u(?:get|env):" + uget_sloppy_id_regex.pattern + ")"
|
|
288
|
+
)
|
|
289
|
+
uget_id_regex_only = re.compile("^" + uget_id_regex + "$")
|
|
290
|
+
uget_id_regex = re.compile(r"\b" + uget_id_regex + r"\b")
|
|
247
291
|
|
|
248
292
|
|
|
249
293
|
class GgetId(str):
|
|
250
294
|
"""Basestring wrapper for Gget Ids."""
|
|
295
|
+
|
|
251
296
|
def __new__(cls, value):
|
|
252
297
|
if uget_id_regex_only.match(value):
|
|
253
|
-
raise ValueError(
|
|
298
|
+
raise ValueError("A GgetId cannot look like a UgetId !")
|
|
254
299
|
return str.__new__(cls, value)
|
|
255
300
|
|
|
256
301
|
|
|
@@ -265,12 +310,20 @@ class AbstractUgetId(str):
|
|
|
265
310
|
if not vmatch:
|
|
266
311
|
raise ValueError('Invalid UgetId (got "{:s}")'.format(value))
|
|
267
312
|
me = str.__new__(cls, value)
|
|
268
|
-
me._id = vmatch.group(
|
|
269
|
-
me._location = vmatch.group(
|
|
313
|
+
me._id = vmatch.group("id")
|
|
314
|
+
me._location = vmatch.group("location")
|
|
270
315
|
if me._location in set(cls._OUTCAST_LOCATIONS):
|
|
271
|
-
raise ValueError(
|
|
272
|
-
|
|
273
|
-
|
|
316
|
+
raise ValueError(
|
|
317
|
+
'Invalid UgetId (got "{:s}"). Outcast Location.'.format(value)
|
|
318
|
+
)
|
|
319
|
+
if cls._ALLOWED_LOCATIONS and me._location not in set(
|
|
320
|
+
cls._ALLOWED_LOCATIONS
|
|
321
|
+
):
|
|
322
|
+
raise ValueError(
|
|
323
|
+
'Invalid UgetId (got "{:s}"). Disallowed location'.format(
|
|
324
|
+
value
|
|
325
|
+
)
|
|
326
|
+
)
|
|
274
327
|
return me
|
|
275
328
|
|
|
276
329
|
@property
|
|
@@ -283,31 +336,30 @@ class AbstractUgetId(str):
|
|
|
283
336
|
|
|
284
337
|
@property
|
|
285
338
|
def short(self):
|
|
286
|
-
return self._id +
|
|
339
|
+
return self._id + "@" + self._location
|
|
287
340
|
|
|
288
341
|
def monthlyshort(self, month):
|
|
289
|
-
return self._id +
|
|
342
|
+
return self._id + ".m{:02d}".format(month) + "@" + self._location
|
|
290
343
|
|
|
291
344
|
|
|
292
345
|
class UgetId(AbstractUgetId):
|
|
293
|
-
|
|
294
|
-
_OUTCAST_LOCATIONS = ('demo', )
|
|
346
|
+
_OUTCAST_LOCATIONS = ("demo",)
|
|
295
347
|
|
|
296
348
|
|
|
297
349
|
def genv_ifs_compiler_convention(cls):
|
|
298
350
|
"""Add the necessary method to handle compiler version/option in Genv."""
|
|
299
|
-
original_gget_basename = getattr(cls,
|
|
351
|
+
original_gget_basename = getattr(cls, "gget_basename", None)
|
|
300
352
|
if original_gget_basename is not None:
|
|
301
353
|
|
|
302
354
|
def gget_basename(self):
|
|
303
355
|
"""GGET specific naming convention."""
|
|
304
356
|
b_dict = original_gget_basename(self)
|
|
305
|
-
if getattr(self,
|
|
306
|
-
b_dict[
|
|
307
|
-
if getattr(self,
|
|
308
|
-
b_dict[
|
|
309
|
-
if getattr(self,
|
|
310
|
-
b_dict[
|
|
357
|
+
if getattr(self, "compiler_version", None):
|
|
358
|
+
b_dict["compiler_version"] = self.compiler_version
|
|
359
|
+
if getattr(self, "compiler_option", None):
|
|
360
|
+
b_dict["compiler_option"] = self.compiler_option
|
|
361
|
+
if getattr(self, "cycle", None):
|
|
362
|
+
b_dict["cycle"] = self.cycle
|
|
311
363
|
return b_dict
|
|
312
364
|
|
|
313
365
|
cls.gget_basename = gget_basename
|
|
@@ -316,15 +368,13 @@ def genv_ifs_compiler_convention(cls):
|
|
|
316
368
|
|
|
317
369
|
#: Usual definition of the ``compiler_version`` and ``compiler_option`` attributes.
|
|
318
370
|
gmkpack_compiler_identification = footprints.Footprint(
|
|
319
|
-
info=
|
|
371
|
+
info="Add the compiler version/option in the footprint",
|
|
320
372
|
attr=dict(
|
|
321
373
|
compiler_version=dict(
|
|
322
|
-
info="The compiler version in gmkpack convention.",
|
|
323
|
-
optional=True
|
|
374
|
+
info="The compiler version in gmkpack convention.", optional=True
|
|
324
375
|
),
|
|
325
376
|
compiler_option=dict(
|
|
326
|
-
info="The compiler option in gmkpack convention.",
|
|
327
|
-
optional=True
|
|
377
|
+
info="The compiler option in gmkpack convention.", optional=True
|
|
328
378
|
),
|
|
329
379
|
),
|
|
330
380
|
)
|
|
@@ -333,21 +383,27 @@ gmkpack_compiler_identification = footprints.Footprint(
|
|
|
333
383
|
#: Usual definition of the ``compiler_version`` and ``compiler_option`` attributes + genv integration.
|
|
334
384
|
gmkpack_compiler_identification_deco = footprints.DecorativeFootprint(
|
|
335
385
|
gmkpack_compiler_identification,
|
|
336
|
-
decorator=[
|
|
386
|
+
decorator=[
|
|
387
|
+
genv_ifs_compiler_convention,
|
|
388
|
+
],
|
|
337
389
|
)
|
|
338
390
|
|
|
339
391
|
|
|
340
392
|
def genv_executable_flavour(cls):
|
|
341
393
|
"""Add the necessary method to the "flavour" in Genv."""
|
|
342
|
-
original_genv_basename = getattr(cls,
|
|
394
|
+
original_genv_basename = getattr(cls, "genv_basename", None)
|
|
343
395
|
if original_genv_basename is not None:
|
|
344
396
|
|
|
345
397
|
def genv_basename(self):
|
|
346
398
|
"""Just retrieve a potential gvar attribute."""
|
|
347
399
|
gvar = original_genv_basename(self)
|
|
348
|
-
if getattr(self,
|
|
349
|
-
gvar +=
|
|
350
|
-
|
|
400
|
+
if getattr(self, "flavour", None):
|
|
401
|
+
gvar += (
|
|
402
|
+
"_"
|
|
403
|
+
+ {"singleprecision": "SP"}.get(
|
|
404
|
+
self.flavour, self.flavour
|
|
405
|
+
).upper()
|
|
406
|
+
)
|
|
351
407
|
return gvar
|
|
352
408
|
|
|
353
409
|
cls.genv_basename = genv_basename
|
|
@@ -356,11 +412,13 @@ def genv_executable_flavour(cls):
|
|
|
356
412
|
|
|
357
413
|
#: Usual definition of the ``flavour`` attribute.
|
|
358
414
|
executable_flavour = footprints.Footprint(
|
|
359
|
-
info=
|
|
415
|
+
info="Add the executable flavour attribute to the resource",
|
|
360
416
|
attr=dict(
|
|
361
417
|
flavour=dict(
|
|
362
418
|
info="The executable flavour (This may influence the Genv's key choice).",
|
|
363
|
-
values=[
|
|
419
|
+
values=[
|
|
420
|
+
"singleprecision",
|
|
421
|
+
],
|
|
364
422
|
optional=True,
|
|
365
423
|
),
|
|
366
424
|
),
|
|
@@ -370,6 +428,10 @@ executable_flavour = footprints.Footprint(
|
|
|
370
428
|
#: Usual definition of the ``flavour``.
|
|
371
429
|
executable_flavour_deco = footprints.DecorativeFootprint(
|
|
372
430
|
executable_flavour,
|
|
373
|
-
decorator=[
|
|
374
|
-
|
|
431
|
+
decorator=[
|
|
432
|
+
genv_executable_flavour,
|
|
433
|
+
namebuilding_append(
|
|
434
|
+
"src", lambda self: [self.flavour], none_discard=True
|
|
435
|
+
),
|
|
436
|
+
],
|
|
375
437
|
)
|
vortex/nwp/tools/__init__.py
CHANGED
|
@@ -3,8 +3,8 @@ Standalone tools for NWP
|
|
|
3
3
|
"""
|
|
4
4
|
|
|
5
5
|
# Recursive inclusion of packages with potential FootprintBase classes
|
|
6
|
-
from . import conftools
|
|
7
|
-
from . import ifstools
|
|
6
|
+
from . import conftools as conftools
|
|
7
|
+
from . import ifstools as ifstools
|
|
8
8
|
|
|
9
9
|
#: Automatic export of data subpackage
|
|
10
10
|
__all__ = []
|
vortex/nwp/tools/addons.py
CHANGED
|
@@ -4,12 +4,12 @@ Various System needed for NWP applications.
|
|
|
4
4
|
|
|
5
5
|
from vortex.tools.addons import AddonGroup
|
|
6
6
|
|
|
7
|
-
#
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
7
|
+
# Import the proper Addon modules for footprints
|
|
8
|
+
from vortex.tools import folder as folder
|
|
9
|
+
from vortex.tools import lfi as lfi
|
|
10
|
+
from vortex.tools import grib as grib
|
|
11
|
+
from vortex.tools import listings as listings
|
|
12
|
+
from vortex.tools import surfex as surfex
|
|
13
13
|
|
|
14
14
|
#: No automatic export
|
|
15
15
|
__all__ = []
|
|
@@ -19,17 +19,22 @@ class NWPAddonsGroup(AddonGroup):
|
|
|
19
19
|
"""A set of usual NWP Addons."""
|
|
20
20
|
|
|
21
21
|
_footprint = dict(
|
|
22
|
-
info
|
|
23
|
-
attr
|
|
24
|
-
kind
|
|
25
|
-
values
|
|
22
|
+
info="Default NWP Addons",
|
|
23
|
+
attr=dict(
|
|
24
|
+
kind=dict(
|
|
25
|
+
values=[
|
|
26
|
+
"nwp",
|
|
27
|
+
],
|
|
26
28
|
),
|
|
27
|
-
)
|
|
29
|
+
),
|
|
28
30
|
)
|
|
29
31
|
|
|
30
|
-
_addonslist = (
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
32
|
+
_addonslist = (
|
|
33
|
+
"allfolders", # Folder like...
|
|
34
|
+
"lfi",
|
|
35
|
+
"iopoll", # Wonderful FA/LFI world...
|
|
36
|
+
"grib",
|
|
37
|
+
"gribapi", # GRIB stuff...
|
|
38
|
+
"arpifs_listings", # Obscure IFS/Arpege listings...
|
|
39
|
+
"sfx", # Surfex...
|
|
40
|
+
)
|
vortex/nwp/tools/agt.py
CHANGED
|
@@ -7,6 +7,7 @@ import collections
|
|
|
7
7
|
|
|
8
8
|
class AgtConfigurationError(Exception):
|
|
9
9
|
"""Specific Transfer Agent configuration error."""
|
|
10
|
+
|
|
10
11
|
pass
|
|
11
12
|
|
|
12
13
|
|
|
@@ -15,10 +16,10 @@ def agt_volatile_path(sh):
|
|
|
15
16
|
and hard-linking to it instead of expecting it to be present "later".
|
|
16
17
|
"""
|
|
17
18
|
config = sh.default_target.config
|
|
18
|
-
if not config.has_section(
|
|
19
|
+
if not config.has_section("agt"):
|
|
19
20
|
fmt = 'Missing section "agt" in configuration file\n"{}"'
|
|
20
21
|
raise AgtConfigurationError(fmt.format(config.file))
|
|
21
|
-
return config.get(
|
|
22
|
+
return config.get("agt", "agt_volatile")
|
|
22
23
|
|
|
23
24
|
|
|
24
25
|
def agt_actual_command(sh, binary_name, args, extraenv=None):
|
|
@@ -35,21 +36,32 @@ def agt_actual_command(sh, binary_name, args, extraenv=None):
|
|
|
35
36
|
:param extraenv: Additional environment variables to export (dictionary)
|
|
36
37
|
"""
|
|
37
38
|
config = sh.default_target.config
|
|
38
|
-
if not config.has_section(
|
|
39
|
+
if not config.has_section("agt"):
|
|
39
40
|
fmt = 'Missing section "agt" in configuration file\n"{}"'
|
|
40
41
|
raise AgtConfigurationError(fmt.format(config.file))
|
|
41
42
|
|
|
42
|
-
agt_path = sh.default_target.get(
|
|
43
|
+
agt_path = sh.default_target.get("agt_path", default=None)
|
|
43
44
|
agt_bin = sh.default_target.get(binary_name, default=None)
|
|
44
45
|
if not all([agt_path, agt_bin]):
|
|
45
46
|
fmt = 'Missing key "agt_path" or "{}" in configuration file\n"{}"'
|
|
46
47
|
raise AgtConfigurationError(fmt.format(binary_name, config.file))
|
|
47
|
-
cfgkeys = [
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
48
|
+
cfgkeys = [
|
|
49
|
+
"HOME_SOPRA",
|
|
50
|
+
"LD_LIBRARY_PATH",
|
|
51
|
+
"base_transfert_agent",
|
|
52
|
+
"DIAP_AGENT_NUMPROG_AGENT",
|
|
53
|
+
]
|
|
54
|
+
context = " ; ".join(
|
|
55
|
+
["export {}={}".format(key, config.get("agt", key)) for key in cfgkeys]
|
|
56
|
+
)
|
|
51
57
|
if extraenv is not None and isinstance(extraenv, collections.abc.Mapping):
|
|
52
|
-
context =
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
58
|
+
context = " ; ".join(
|
|
59
|
+
[
|
|
60
|
+
context,
|
|
61
|
+
]
|
|
62
|
+
+ [
|
|
63
|
+
"export {}={}".format(key.upper(), value)
|
|
64
|
+
for (key, value) in extraenv.items()
|
|
65
|
+
]
|
|
66
|
+
)
|
|
67
|
+
return "{} ; {} {}".format(context, sh.path.join(agt_path, agt_bin), args)
|
vortex/nwp/tools/bdap.py
CHANGED
|
@@ -8,16 +8,19 @@ __all__ = []
|
|
|
8
8
|
|
|
9
9
|
class BDAPError(Exception):
|
|
10
10
|
"""General BDAP error."""
|
|
11
|
+
|
|
11
12
|
pass
|
|
12
13
|
|
|
13
14
|
|
|
14
15
|
class BDAPRequestConfigurationError(BDAPError):
|
|
15
16
|
"""Specific Transfer Agent configuration error."""
|
|
17
|
+
|
|
16
18
|
pass
|
|
17
19
|
|
|
18
20
|
|
|
19
21
|
class BDAPGetError(BDAPError):
|
|
20
22
|
"""Generic BDAP get error."""
|
|
23
|
+
|
|
21
24
|
pass
|
|
22
25
|
|
|
23
26
|
|
|
@@ -38,11 +41,19 @@ def BDAPrequest_actual_command(command, date, term, query, int_extraenv=False):
|
|
|
38
41
|
"""
|
|
39
42
|
|
|
40
43
|
# Environment variable to specify the date of the file
|
|
41
|
-
context =
|
|
44
|
+
context = " ; ".join(
|
|
45
|
+
["export {}={}".format("dmt_date_pivot".upper(), date.ymdhms)]
|
|
46
|
+
)
|
|
42
47
|
# Extra environment variables (integration BDAP)
|
|
43
48
|
if int_extraenv:
|
|
44
|
-
context =
|
|
45
|
-
|
|
46
|
-
|
|
49
|
+
context = " ; ".join(
|
|
50
|
+
[context]
|
|
51
|
+
+ [
|
|
52
|
+
"export {}={}".format(
|
|
53
|
+
"db_file_bdap".upper(),
|
|
54
|
+
"/usr/local/sopra/neons_db_bdap.intgr",
|
|
55
|
+
)
|
|
56
|
+
]
|
|
57
|
+
)
|
|
47
58
|
# Return the command to be launched
|
|
48
|
-
return
|
|
59
|
+
return "{} ; {} {:d} {}".format(context, command, term.hour, query)
|