vortex-nwp 2.0.0b1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (146) hide show
  1. vortex/__init__.py +135 -0
  2. vortex/algo/__init__.py +12 -0
  3. vortex/algo/components.py +2136 -0
  4. vortex/algo/mpitools.py +1648 -0
  5. vortex/algo/mpitools_templates/envelope_wrapper_default.tpl +27 -0
  6. vortex/algo/mpitools_templates/envelope_wrapper_mpiauto.tpl +29 -0
  7. vortex/algo/mpitools_templates/wrapstd_wrapper_default.tpl +18 -0
  8. vortex/algo/serversynctools.py +170 -0
  9. vortex/config.py +115 -0
  10. vortex/data/__init__.py +13 -0
  11. vortex/data/abstractstores.py +1572 -0
  12. vortex/data/containers.py +780 -0
  13. vortex/data/contents.py +596 -0
  14. vortex/data/executables.py +284 -0
  15. vortex/data/flow.py +113 -0
  16. vortex/data/geometries.ini +2689 -0
  17. vortex/data/geometries.py +703 -0
  18. vortex/data/handlers.py +1021 -0
  19. vortex/data/outflow.py +67 -0
  20. vortex/data/providers.py +465 -0
  21. vortex/data/resources.py +201 -0
  22. vortex/data/stores.py +1271 -0
  23. vortex/gloves.py +282 -0
  24. vortex/layout/__init__.py +27 -0
  25. vortex/layout/appconf.py +109 -0
  26. vortex/layout/contexts.py +511 -0
  27. vortex/layout/dataflow.py +1069 -0
  28. vortex/layout/jobs.py +1276 -0
  29. vortex/layout/monitor.py +833 -0
  30. vortex/layout/nodes.py +1424 -0
  31. vortex/layout/subjobs.py +464 -0
  32. vortex/nwp/__init__.py +11 -0
  33. vortex/nwp/algo/__init__.py +12 -0
  34. vortex/nwp/algo/assim.py +483 -0
  35. vortex/nwp/algo/clim.py +920 -0
  36. vortex/nwp/algo/coupling.py +609 -0
  37. vortex/nwp/algo/eda.py +632 -0
  38. vortex/nwp/algo/eps.py +613 -0
  39. vortex/nwp/algo/forecasts.py +745 -0
  40. vortex/nwp/algo/fpserver.py +927 -0
  41. vortex/nwp/algo/ifsnaming.py +403 -0
  42. vortex/nwp/algo/ifsroot.py +311 -0
  43. vortex/nwp/algo/monitoring.py +202 -0
  44. vortex/nwp/algo/mpitools.py +554 -0
  45. vortex/nwp/algo/odbtools.py +974 -0
  46. vortex/nwp/algo/oopsroot.py +735 -0
  47. vortex/nwp/algo/oopstests.py +186 -0
  48. vortex/nwp/algo/request.py +579 -0
  49. vortex/nwp/algo/stdpost.py +1285 -0
  50. vortex/nwp/data/__init__.py +12 -0
  51. vortex/nwp/data/assim.py +392 -0
  52. vortex/nwp/data/boundaries.py +261 -0
  53. vortex/nwp/data/climfiles.py +539 -0
  54. vortex/nwp/data/configfiles.py +149 -0
  55. vortex/nwp/data/consts.py +929 -0
  56. vortex/nwp/data/ctpini.py +133 -0
  57. vortex/nwp/data/diagnostics.py +181 -0
  58. vortex/nwp/data/eda.py +148 -0
  59. vortex/nwp/data/eps.py +383 -0
  60. vortex/nwp/data/executables.py +1039 -0
  61. vortex/nwp/data/fields.py +96 -0
  62. vortex/nwp/data/gridfiles.py +308 -0
  63. vortex/nwp/data/logs.py +551 -0
  64. vortex/nwp/data/modelstates.py +334 -0
  65. vortex/nwp/data/monitoring.py +220 -0
  66. vortex/nwp/data/namelists.py +644 -0
  67. vortex/nwp/data/obs.py +748 -0
  68. vortex/nwp/data/oopsexec.py +72 -0
  69. vortex/nwp/data/providers.py +182 -0
  70. vortex/nwp/data/query.py +217 -0
  71. vortex/nwp/data/stores.py +147 -0
  72. vortex/nwp/data/surfex.py +338 -0
  73. vortex/nwp/syntax/__init__.py +9 -0
  74. vortex/nwp/syntax/stdattrs.py +375 -0
  75. vortex/nwp/tools/__init__.py +10 -0
  76. vortex/nwp/tools/addons.py +35 -0
  77. vortex/nwp/tools/agt.py +55 -0
  78. vortex/nwp/tools/bdap.py +48 -0
  79. vortex/nwp/tools/bdcp.py +38 -0
  80. vortex/nwp/tools/bdm.py +21 -0
  81. vortex/nwp/tools/bdmp.py +49 -0
  82. vortex/nwp/tools/conftools.py +1311 -0
  83. vortex/nwp/tools/drhook.py +62 -0
  84. vortex/nwp/tools/grib.py +268 -0
  85. vortex/nwp/tools/gribdiff.py +99 -0
  86. vortex/nwp/tools/ifstools.py +163 -0
  87. vortex/nwp/tools/igastuff.py +249 -0
  88. vortex/nwp/tools/mars.py +56 -0
  89. vortex/nwp/tools/odb.py +548 -0
  90. vortex/nwp/tools/partitioning.py +234 -0
  91. vortex/nwp/tools/satrad.py +56 -0
  92. vortex/nwp/util/__init__.py +6 -0
  93. vortex/nwp/util/async.py +184 -0
  94. vortex/nwp/util/beacon.py +40 -0
  95. vortex/nwp/util/diffpygram.py +359 -0
  96. vortex/nwp/util/ens.py +198 -0
  97. vortex/nwp/util/hooks.py +128 -0
  98. vortex/nwp/util/taskdeco.py +81 -0
  99. vortex/nwp/util/usepygram.py +591 -0
  100. vortex/nwp/util/usetnt.py +87 -0
  101. vortex/proxy.py +6 -0
  102. vortex/sessions.py +341 -0
  103. vortex/syntax/__init__.py +9 -0
  104. vortex/syntax/stdattrs.py +628 -0
  105. vortex/syntax/stddeco.py +176 -0
  106. vortex/toolbox.py +982 -0
  107. vortex/tools/__init__.py +11 -0
  108. vortex/tools/actions.py +457 -0
  109. vortex/tools/addons.py +297 -0
  110. vortex/tools/arm.py +76 -0
  111. vortex/tools/compression.py +322 -0
  112. vortex/tools/date.py +20 -0
  113. vortex/tools/ddhpack.py +10 -0
  114. vortex/tools/delayedactions.py +672 -0
  115. vortex/tools/env.py +513 -0
  116. vortex/tools/folder.py +663 -0
  117. vortex/tools/grib.py +559 -0
  118. vortex/tools/lfi.py +746 -0
  119. vortex/tools/listings.py +354 -0
  120. vortex/tools/names.py +575 -0
  121. vortex/tools/net.py +1790 -0
  122. vortex/tools/odb.py +10 -0
  123. vortex/tools/parallelism.py +336 -0
  124. vortex/tools/prestaging.py +186 -0
  125. vortex/tools/rawfiles.py +10 -0
  126. vortex/tools/schedulers.py +413 -0
  127. vortex/tools/services.py +871 -0
  128. vortex/tools/storage.py +1061 -0
  129. vortex/tools/surfex.py +61 -0
  130. vortex/tools/systems.py +3396 -0
  131. vortex/tools/targets.py +384 -0
  132. vortex/util/__init__.py +9 -0
  133. vortex/util/config.py +1071 -0
  134. vortex/util/empty.py +24 -0
  135. vortex/util/helpers.py +184 -0
  136. vortex/util/introspection.py +63 -0
  137. vortex/util/iosponge.py +76 -0
  138. vortex/util/roles.py +51 -0
  139. vortex/util/storefunctions.py +103 -0
  140. vortex/util/structs.py +26 -0
  141. vortex/util/worker.py +150 -0
  142. vortex_nwp-2.0.0b1.dist-info/LICENSE +517 -0
  143. vortex_nwp-2.0.0b1.dist-info/METADATA +50 -0
  144. vortex_nwp-2.0.0b1.dist-info/RECORD +146 -0
  145. vortex_nwp-2.0.0b1.dist-info/WHEEL +5 -0
  146. vortex_nwp-2.0.0b1.dist-info/top_level.txt +1 -0
@@ -0,0 +1,284 @@
1
+ """
2
+ Abstract for any "Resource" class that deal with some kind of :class:`Script` or
3
+ :class:`Binary` executable.
4
+
5
+ Mode specialised version are also provided for various physical models:
6
+
7
+ * :class:`NWPModel`;
8
+ * :class:`OceanographicModel`;
9
+ * :class:`SurfaceModel`;
10
+ * :class:`ChemistryModel`.
11
+
12
+ """
13
+
14
+ from bronx.syntax import mktuple
15
+ import footprints
16
+
17
+ from .resources import Resource
18
+ from vortex.syntax.stdattrs import model_deco
19
+ from vortex.util.config import JacketConfigParser
20
+
21
+ #: No automatic export
22
+ __all__ = []
23
+
24
+
25
+ class Jacket:
26
+ """The class definition of in and out resources from a given executable."""
27
+ def __init__(self, afile=None):
28
+ if afile:
29
+ self.config = JacketConfigParser(afile)
30
+ self.virtual = False
31
+ else:
32
+ self.virtual = True
33
+ self._initfile = afile
34
+
35
+ def as_dump(self):
36
+ return "file={!r}".format(self._initfile)
37
+
38
+ def export_dict(self):
39
+ return self._initfile
40
+
41
+
42
+ class Executable(Resource):
43
+ """Abstract class for resources that could be executed."""
44
+
45
+ _abstract = True
46
+ _footprint = dict(
47
+ info = 'Miscellaneaous executable resource',
48
+ attr = dict(
49
+ cycle = dict(
50
+ info = "Any kind of cycle name",
51
+ optional = True,
52
+ default = None,
53
+ access = 'rwx',
54
+ ),
55
+ kind = dict(
56
+ info = "The resource's kind.",
57
+ doc_zorder = 90,
58
+ ),
59
+ nativefmt = dict(
60
+ doc_visibility = footprints.doc.visibility.GURU,
61
+ ),
62
+ clscontents = dict(
63
+ doc_visibility = footprints.doc.visibility.GURU,
64
+ )
65
+ )
66
+ )
67
+
68
+ def stdin_text(self, **opts):
69
+ """Abstract method."""
70
+ return None
71
+
72
+
73
+ class Script(Executable):
74
+ """Basic interpreted executable associated to a specific language."""
75
+
76
+ _footprint = dict(
77
+ attr = dict(
78
+ rawopts = dict(
79
+ info = "Options that will be passed directly to the script",
80
+ optional = True,
81
+ default = '',
82
+ ),
83
+ language = dict(
84
+ info = "The programming language",
85
+ values = ['perl', 'python', 'ksh', 'bash', 'sh', 'awk'],
86
+ ),
87
+ kind = dict(
88
+ optional = True,
89
+ default = 'script',
90
+ values = ['script'],
91
+ )
92
+ ),
93
+ fastkeys = {'language'},
94
+ )
95
+
96
+ @property
97
+ def realkind(self):
98
+ return 'script'
99
+
100
+ def command_line(self, **opts):
101
+ """Returns optional attribute :attr:`rawopts`."""
102
+ if self.rawopts is None:
103
+ return ''
104
+ else:
105
+ return self.rawopts
106
+
107
+
108
+ class GnuScript(Executable):
109
+ """Basic interpreted executable with standard command line arguments."""
110
+
111
+ _footprint = dict(
112
+ attr = dict(
113
+ language = dict(
114
+ info = "The programming language",
115
+ values = ['perl', 'python', 'ksh', 'bash', 'sh', 'awk'],
116
+ ),
117
+ kind = dict(
118
+ default = 'gnuscript',
119
+ values = ['gnuscript', 'argscript'],
120
+ )
121
+ ),
122
+ fastkeys = {'kind', 'language'},
123
+ )
124
+
125
+ @property
126
+ def realkind(self):
127
+ return 'script'
128
+
129
+ def command_line(self, **opts):
130
+ """Returns a blank separated list of options."""
131
+ return ' '.join(['--' + k + ' ' + ' '.join([str(x)
132
+ for x in mktuple(v)])
133
+ for k, v in opts.items()])
134
+
135
+
136
+ class Binary(Executable):
137
+ """Basic compiled executable."""
138
+
139
+ _abstract = True
140
+ _footprint = dict(
141
+ attr = dict(
142
+ static = dict(
143
+ info = "Statically linked binary.",
144
+ type = bool,
145
+ optional = True,
146
+ doc_visibility = footprints.doc.visibility.ADVANCED,
147
+ ),
148
+ jacket = dict(
149
+ type = Jacket,
150
+ optional = True,
151
+ default = Jacket(),
152
+ doc_visibility = footprints.doc.visibility.ADVANCED,
153
+ )
154
+ )
155
+ )
156
+
157
+ @property
158
+ def realkind(self):
159
+ return 'binary'
160
+
161
+ def guess_binary_sources(self, provider): # @UnusedVariable
162
+ """A list of path that contains source files (for debugging purposes)."""
163
+ return []
164
+
165
+
166
+ class BlackBox(Binary):
167
+ """Binary resource with explicit command line options."""
168
+
169
+ _footprint = dict(
170
+ attr = dict(
171
+ binopts = dict(
172
+ info = "Options that will be passed directly to the binary",
173
+ optional = True,
174
+ default = '',
175
+ ),
176
+ kind = dict(
177
+ values = ['binbox', 'blackbox'],
178
+ remap = dict(binbox = 'blackbox'),
179
+ ),
180
+ )
181
+ )
182
+
183
+ def command_line(self, **opts):
184
+ """Returns current attribute :attr:`binopts`."""
185
+ return self.binopts
186
+
187
+
188
+ class NWPModel(Binary):
189
+ """Base class for any Numerical Weather Prediction Model."""
190
+
191
+ _abstract = True
192
+ _footprint = [
193
+ model_deco,
194
+ dict(
195
+ info = 'NWP Model',
196
+ attr = dict(
197
+ kind = dict(
198
+ values = ['nwpmodel']
199
+ )
200
+ )
201
+ )
202
+ ]
203
+
204
+ @property
205
+ def realkind(self):
206
+ return 'nwpmodel'
207
+
208
+ def command_line(self, **opts):
209
+ """Abstract method."""
210
+ return ''
211
+
212
+
213
+ class OceanographicModel(Binary):
214
+ """Base class for any Oceanographic Model."""
215
+
216
+ _abstract = True
217
+ _footprint = [
218
+ model_deco,
219
+ dict(
220
+ info = 'Oceanographic Model',
221
+ attr = dict(
222
+ kind = dict(
223
+ values = ['oceanmodel']
224
+ )
225
+ )
226
+ )
227
+ ]
228
+
229
+ @property
230
+ def realkind(self):
231
+ return 'oceanmodel'
232
+
233
+ def command_line(self, **opts):
234
+ """Abstract method."""
235
+ return ''
236
+
237
+
238
+ class SurfaceModel(Binary):
239
+
240
+ _abstract = True
241
+ _footprint = [
242
+ model_deco,
243
+ dict(
244
+ info = 'Model used for the Safran-Surfex-Mepra chain.',
245
+ attr = dict(
246
+ kind = dict(
247
+ values = ['surfacemodel', 'snowmodel'],
248
+ remap = dict(autoremap = 'first'),
249
+ ),
250
+ ),
251
+ )
252
+ ]
253
+
254
+ @property
255
+ def realkind(self):
256
+ return 'surfacemodel'
257
+
258
+ def command_line(self, **opts):
259
+ """Abstract method."""
260
+ return ''
261
+
262
+
263
+ class ChemistryModel(Binary):
264
+
265
+ _abstract = True
266
+ _footprint = [
267
+ model_deco,
268
+ dict(
269
+ info = 'Base class for Chemistry models.',
270
+ attr = dict(
271
+ kind = dict(
272
+ values = ['chemistrymodel'],
273
+ ),
274
+ ),
275
+ )
276
+ ]
277
+
278
+ @property
279
+ def realkind(self):
280
+ return 'chemistrymodel'
281
+
282
+ def command_line(self, **opts):
283
+ """Abstract method."""
284
+ return ''
vortex/data/flow.py ADDED
@@ -0,0 +1,113 @@
1
+ """
2
+ Abstract class for any flow-dependant "Resource".
3
+
4
+ By "flow-dependant", we mean a "Resource" that has to be described by ``model``,
5
+ ``date`` and ``cutoff`` attributes. See :class:`FlowResource`
6
+
7
+ The :class:`GeoFlowResource` abstract class is to be used for flow-dependant
8
+ resources associated with an horizontal geometry.
9
+ """
10
+
11
+ from .resources import Resource
12
+ from .geometries import hgeometry_deco
13
+ from .contents import FormatAdapter
14
+
15
+ from vortex.syntax.stdattrs import model_deco, date_deco, dateperiod_deco, cutoff_deco, term_deco
16
+ from vortex.syntax.stddeco import namebuilding_insert
17
+
18
+ #: No automatic export
19
+ __all__ = []
20
+
21
+
22
+ class FlowResource(Resource):
23
+ """Abstract resource bound to a model, a date and a cutoff."""
24
+
25
+ _abstract = True
26
+ _footprint = [model_deco, date_deco, cutoff_deco]
27
+
28
+
29
+ @namebuilding_insert('radical', lambda s: s.nickname)
30
+ class UnknownFlow(FlowResource):
31
+
32
+ _footprint = [
33
+ term_deco,
34
+ dict(
35
+ info = 'Unknown assumed NWP Flow-Resource (development only !)',
36
+ attr = dict(
37
+ unknownflow = dict(
38
+ info = "Activate the unknown flow resource",
39
+ type = bool
40
+ ),
41
+ term = dict(
42
+ optional = True,
43
+ ),
44
+ nickname = dict(
45
+ info = "The string that serves the purpose of Vortex's basename radical",
46
+ optional = True,
47
+ default = 'unknown'
48
+ ),
49
+ clscontents = dict(
50
+ default = FormatAdapter
51
+ ),
52
+ ),
53
+ fastkeys = {'unknownflow'},
54
+ )
55
+ ]
56
+
57
+ _extension_remap = {k: None for k in ('auto', 'autoconfig', 'foo', 'unknown')}
58
+
59
+ def olive_basename(self):
60
+ target = self.nickname
61
+ if self.term is not None:
62
+ target += '+' + self.term.fmth
63
+ return target
64
+
65
+
66
+ class GeoFlowResource(FlowResource):
67
+ """Class which is a :class:`FlowResource` bound to a geometry."""
68
+
69
+ _abstract = True
70
+ _footprint = [
71
+ hgeometry_deco,
72
+ dict(
73
+ attr = dict(
74
+ clscontents = dict(
75
+ default = FormatAdapter,
76
+ ),
77
+ )
78
+ )
79
+ ]
80
+
81
+
82
+ class PeriodFlowResource(Resource):
83
+ """Abstract resource bound to a model, a begindate, enddate and a cutoff."""
84
+
85
+ _abstract = True
86
+ _footprint = [model_deco, dateperiod_deco, cutoff_deco]
87
+
88
+ _footprint = [
89
+ model_deco, dateperiod_deco, cutoff_deco,
90
+ dict(
91
+ attr = dict(
92
+ cutoff = dict(
93
+ optional = True,
94
+ ),
95
+ )
96
+ )
97
+ ]
98
+
99
+
100
+ class GeoPeriodFlowResource(PeriodFlowResource):
101
+ """Class which is a :class:`PeriodFlowResource` bound to a geometry."""
102
+
103
+ _abstract = True
104
+ _footprint = [
105
+ hgeometry_deco,
106
+ dict(
107
+ attr = dict(
108
+ clscontents = dict(
109
+ default = FormatAdapter,
110
+ ),
111
+ )
112
+ )
113
+ ]