vortex-nwp 2.0.0b1__py3-none-any.whl → 2.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (141) hide show
  1. vortex/__init__.py +75 -47
  2. vortex/algo/__init__.py +3 -2
  3. vortex/algo/components.py +944 -618
  4. vortex/algo/mpitools.py +802 -497
  5. vortex/algo/mpitools_templates/__init__.py +1 -0
  6. vortex/algo/serversynctools.py +34 -33
  7. vortex/config.py +19 -22
  8. vortex/data/__init__.py +9 -3
  9. vortex/data/abstractstores.py +593 -655
  10. vortex/data/containers.py +217 -162
  11. vortex/data/contents.py +65 -39
  12. vortex/data/executables.py +93 -102
  13. vortex/data/flow.py +40 -34
  14. vortex/data/geometries.py +228 -132
  15. vortex/data/handlers.py +436 -227
  16. vortex/data/outflow.py +15 -15
  17. vortex/data/providers.py +185 -163
  18. vortex/data/resources.py +48 -42
  19. vortex/data/stores.py +540 -417
  20. vortex/data/sync_templates/__init__.py +0 -0
  21. vortex/gloves.py +114 -87
  22. vortex/layout/__init__.py +1 -8
  23. vortex/layout/contexts.py +150 -84
  24. vortex/layout/dataflow.py +353 -202
  25. vortex/layout/monitor.py +264 -128
  26. vortex/nwp/__init__.py +5 -2
  27. vortex/nwp/algo/__init__.py +14 -5
  28. vortex/nwp/algo/assim.py +205 -151
  29. vortex/nwp/algo/clim.py +683 -517
  30. vortex/nwp/algo/coupling.py +447 -225
  31. vortex/nwp/algo/eda.py +437 -229
  32. vortex/nwp/algo/eps.py +403 -231
  33. vortex/nwp/algo/forecasts.py +416 -275
  34. vortex/nwp/algo/fpserver.py +683 -307
  35. vortex/nwp/algo/ifsnaming.py +205 -145
  36. vortex/nwp/algo/ifsroot.py +215 -122
  37. vortex/nwp/algo/monitoring.py +137 -76
  38. vortex/nwp/algo/mpitools.py +330 -190
  39. vortex/nwp/algo/odbtools.py +637 -353
  40. vortex/nwp/algo/oopsroot.py +454 -273
  41. vortex/nwp/algo/oopstests.py +90 -56
  42. vortex/nwp/algo/request.py +287 -206
  43. vortex/nwp/algo/stdpost.py +878 -522
  44. vortex/nwp/data/__init__.py +22 -4
  45. vortex/nwp/data/assim.py +125 -137
  46. vortex/nwp/data/boundaries.py +121 -68
  47. vortex/nwp/data/climfiles.py +193 -211
  48. vortex/nwp/data/configfiles.py +73 -69
  49. vortex/nwp/data/consts.py +426 -401
  50. vortex/nwp/data/ctpini.py +59 -43
  51. vortex/nwp/data/diagnostics.py +94 -66
  52. vortex/nwp/data/eda.py +50 -51
  53. vortex/nwp/data/eps.py +195 -146
  54. vortex/nwp/data/executables.py +440 -434
  55. vortex/nwp/data/fields.py +63 -48
  56. vortex/nwp/data/gridfiles.py +183 -111
  57. vortex/nwp/data/logs.py +250 -217
  58. vortex/nwp/data/modelstates.py +180 -151
  59. vortex/nwp/data/monitoring.py +72 -99
  60. vortex/nwp/data/namelists.py +254 -202
  61. vortex/nwp/data/obs.py +400 -308
  62. vortex/nwp/data/oopsexec.py +22 -20
  63. vortex/nwp/data/providers.py +90 -65
  64. vortex/nwp/data/query.py +71 -82
  65. vortex/nwp/data/stores.py +49 -36
  66. vortex/nwp/data/surfex.py +136 -137
  67. vortex/nwp/syntax/__init__.py +1 -1
  68. vortex/nwp/syntax/stdattrs.py +173 -111
  69. vortex/nwp/tools/__init__.py +2 -2
  70. vortex/nwp/tools/addons.py +22 -17
  71. vortex/nwp/tools/agt.py +24 -12
  72. vortex/nwp/tools/bdap.py +16 -5
  73. vortex/nwp/tools/bdcp.py +4 -1
  74. vortex/nwp/tools/bdm.py +3 -0
  75. vortex/nwp/tools/bdmp.py +14 -9
  76. vortex/nwp/tools/conftools.py +728 -378
  77. vortex/nwp/tools/drhook.py +12 -8
  78. vortex/nwp/tools/grib.py +65 -39
  79. vortex/nwp/tools/gribdiff.py +22 -17
  80. vortex/nwp/tools/ifstools.py +82 -42
  81. vortex/nwp/tools/igastuff.py +167 -143
  82. vortex/nwp/tools/mars.py +14 -2
  83. vortex/nwp/tools/odb.py +234 -125
  84. vortex/nwp/tools/partitioning.py +61 -37
  85. vortex/nwp/tools/satrad.py +27 -12
  86. vortex/nwp/util/async.py +83 -55
  87. vortex/nwp/util/beacon.py +10 -10
  88. vortex/nwp/util/diffpygram.py +174 -86
  89. vortex/nwp/util/ens.py +144 -63
  90. vortex/nwp/util/hooks.py +30 -19
  91. vortex/nwp/util/taskdeco.py +28 -24
  92. vortex/nwp/util/usepygram.py +278 -172
  93. vortex/nwp/util/usetnt.py +31 -17
  94. vortex/sessions.py +72 -39
  95. vortex/syntax/__init__.py +1 -1
  96. vortex/syntax/stdattrs.py +410 -171
  97. vortex/syntax/stddeco.py +31 -22
  98. vortex/toolbox.py +327 -192
  99. vortex/tools/__init__.py +11 -2
  100. vortex/tools/actions.py +110 -121
  101. vortex/tools/addons.py +111 -92
  102. vortex/tools/arm.py +42 -22
  103. vortex/tools/compression.py +72 -69
  104. vortex/tools/date.py +11 -4
  105. vortex/tools/delayedactions.py +242 -132
  106. vortex/tools/env.py +75 -47
  107. vortex/tools/folder.py +342 -171
  108. vortex/tools/grib.py +341 -162
  109. vortex/tools/lfi.py +423 -216
  110. vortex/tools/listings.py +109 -40
  111. vortex/tools/names.py +218 -156
  112. vortex/tools/net.py +655 -299
  113. vortex/tools/parallelism.py +93 -61
  114. vortex/tools/prestaging.py +55 -31
  115. vortex/tools/schedulers.py +172 -105
  116. vortex/tools/services.py +403 -334
  117. vortex/tools/storage.py +293 -358
  118. vortex/tools/surfex.py +24 -24
  119. vortex/tools/systems.py +1234 -643
  120. vortex/tools/targets.py +156 -100
  121. vortex/util/__init__.py +1 -1
  122. vortex/util/config.py +378 -327
  123. vortex/util/empty.py +2 -2
  124. vortex/util/helpers.py +56 -24
  125. vortex/util/introspection.py +18 -12
  126. vortex/util/iosponge.py +8 -4
  127. vortex/util/roles.py +4 -6
  128. vortex/util/storefunctions.py +39 -13
  129. vortex/util/structs.py +3 -3
  130. vortex/util/worker.py +29 -17
  131. vortex_nwp-2.1.0.dist-info/METADATA +67 -0
  132. vortex_nwp-2.1.0.dist-info/RECORD +144 -0
  133. {vortex_nwp-2.0.0b1.dist-info → vortex_nwp-2.1.0.dist-info}/WHEEL +1 -1
  134. vortex/layout/appconf.py +0 -109
  135. vortex/layout/jobs.py +0 -1276
  136. vortex/layout/nodes.py +0 -1424
  137. vortex/layout/subjobs.py +0 -464
  138. vortex_nwp-2.0.0b1.dist-info/METADATA +0 -50
  139. vortex_nwp-2.0.0b1.dist-info/RECORD +0 -146
  140. {vortex_nwp-2.0.0b1.dist-info → vortex_nwp-2.1.0.dist-info/licenses}/LICENSE +0 -0
  141. {vortex_nwp-2.0.0b1.dist-info → vortex_nwp-2.1.0.dist-info}/top_level.txt +0 -0
@@ -6,9 +6,17 @@ import json
6
6
 
7
7
  import footprints
8
8
 
9
- from vortex.algo.components import AlgoComponentDecoMixin, algo_component_deco_mixin_autodoc
9
+ from vortex.algo.components import (
10
+ AlgoComponentDecoMixin,
11
+ algo_component_deco_mixin_autodoc,
12
+ )
10
13
  from ..syntax.stdattrs import oops_test_type, oops_expected_target
11
- from .oopsroot import OOPSParallel, OOPSODB, OOPSMembersTermsDecoMixin, OOPSMembersTermsDetectDecoMixin
14
+ from .oopsroot import (
15
+ OOPSParallel,
16
+ OOPSODB,
17
+ OOPSMembersTermsDecoMixin,
18
+ OOPSMembersTermsDetectDecoMixin,
19
+ )
12
20
 
13
21
  #: No automatic export
14
22
  __all__ = []
@@ -25,14 +33,14 @@ class _OOPSTestDecoMixin(AlgoComponentDecoMixin):
25
33
  the dictionary that is used to build the binary' command line.
26
34
  """
27
35
 
28
- _MIXIN_EXTRA_FOOTPRINTS = (oops_test_type, )
36
+ _MIXIN_EXTRA_FOOTPRINTS = (oops_test_type,)
29
37
 
30
38
  def _ooptest_cli_opts_extend(self, prev):
31
39
  """Prepare options for the resource's command line."""
32
- prev['test_type'] = self.test_type
40
+ prev["test_type"] = self.test_type
33
41
  return prev
34
42
 
35
- _MIXIN_CLI_OPTS_EXTEND = (_ooptest_cli_opts_extend, )
43
+ _MIXIN_CLI_OPTS_EXTEND = (_ooptest_cli_opts_extend,)
36
44
 
37
45
 
38
46
  @algo_component_deco_mixin_autodoc
@@ -61,11 +69,13 @@ class _OOPSTestExpTargetDecoMixin(AlgoComponentDecoMixin):
61
69
  # if attribute 'expected_target' is attribute and given to the algo, use it
62
70
  target = self._set_expected_target_from_attribute()
63
71
  # else, go find Reference summary in effective inputs
64
- if target is not None and target.get('from') == 'reference_summary':
72
+ if target is not None and target.get("from") == "reference_summary":
65
73
  target = self._set_expected_target_from_reference_summary()
66
74
  # Else, default to be sure to pass any in-binary-test
67
75
  if target is None:
68
- target = self._set_expected_target_default() # CLEANME: to be removed after CY47 ?
76
+ target = (
77
+ self._set_expected_target_default()
78
+ ) # CLEANME: to be removed after CY47 ?
69
79
  # Then in the end, export variable
70
80
  target = json.dumps(target)
71
81
  logger.info("Expected Target for Test: " + target)
@@ -73,88 +83,112 @@ class _OOPSTestExpTargetDecoMixin(AlgoComponentDecoMixin):
73
83
 
74
84
  def _set_expected_target_from_attribute(self):
75
85
  """Read target in Algo attribute."""
76
- if hasattr(self, 'expected_target'):
86
+ if hasattr(self, "expected_target"):
77
87
  if self.expected_target is not None:
78
88
  target = self.expected_target
79
- logger.info('Set EXPECTED_RESULT from Attribute')
89
+ logger.info("Set EXPECTED_RESULT from Attribute")
80
90
  return target
81
91
 
82
92
  def _set_expected_target_from_reference_summary(self):
83
93
  """Read target in ReferenceSummary effective input"""
84
94
  target = None
85
- ref_summary = [s for s in self.context.sequence.effective_inputs(role=('Reference',))
86
- if s.rh.resource.kind == 'taskinfo']
95
+ ref_summary = [
96
+ s
97
+ for s in self.context.sequence.effective_inputs(
98
+ role=("Reference",)
99
+ )
100
+ if s.rh.resource.kind == "taskinfo"
101
+ ]
87
102
  if len(ref_summary) > 0:
88
103
  ref_summary = ref_summary[0].rh.contents.data
89
- target = ref_summary.get('oops:' + self.test_type,
90
- {}).get('as EXPECTED_RESULT', None)
104
+ target = ref_summary.get("oops:" + self.test_type, {}).get(
105
+ "as EXPECTED_RESULT", None
106
+ )
91
107
  if target is not None:
92
- logger.info('Set EXPECTED_RESULT from Reference summary')
108
+ logger.info("Set EXPECTED_RESULT from Reference summary")
93
109
  return target
94
110
 
95
- def _set_expected_target_default(self): # CLEANME: to be removed after CY47 ?
111
+ def _set_expected_target_default(
112
+ self,
113
+ ): # CLEANME: to be removed after CY47 ?
96
114
  """Set default, for binary not to crash before CY47."""
97
- target = {'significant_digits': '-9',
98
- 'expected_Jo': '9999',
99
- 'expected_variances': '9999',
100
- 'expected_diff': '9999'}
101
- logger.info('Set default EXPECTED_RESULT')
115
+ target = {
116
+ "significant_digits": "-9",
117
+ "expected_Jo": "9999",
118
+ "expected_variances": "9999",
119
+ "expected_diff": "9999",
120
+ }
121
+ logger.info("Set default EXPECTED_RESULT")
102
122
  return target
103
123
 
104
124
  def _ooptest_exptarget_prepare_hook(self, rh, opts):
105
125
  """Call set_expected_target juste after prepare."""
106
126
  self.set_expected_target()
107
127
 
108
- _MIXIN_PREPARE_HOOKS = (_ooptest_exptarget_prepare_hook, )
128
+ _MIXIN_PREPARE_HOOKS = (_ooptest_exptarget_prepare_hook,)
109
129
 
110
130
 
111
- class OOPSTest(OOPSParallel, _OOPSTestDecoMixin, _OOPSTestExpTargetDecoMixin,
112
- OOPSMembersTermsDetectDecoMixin):
131
+ class OOPSTest(
132
+ OOPSParallel,
133
+ _OOPSTestDecoMixin,
134
+ _OOPSTestExpTargetDecoMixin,
135
+ OOPSMembersTermsDetectDecoMixin,
136
+ ):
113
137
  """OOPS Tests without ODB."""
114
138
 
115
139
  _footprint = dict(
116
- info = "OOPS Test run.",
117
- attr = dict(
118
- kind = dict(
119
- values = ['ootest'],
140
+ info="OOPS Test run.",
141
+ attr=dict(
142
+ kind=dict(
143
+ values=["ootest"],
120
144
  ),
121
- test_type = dict(
122
- outcast = ['ensemble/build', ]
145
+ test_type=dict(
146
+ outcast=[
147
+ "ensemble/build",
148
+ ]
123
149
  ),
124
- )
150
+ ),
125
151
  )
126
152
 
127
153
 
128
- class OOPSTestEnsBuild(OOPSParallel, _OOPSTestDecoMixin, OOPSMembersTermsDecoMixin):
154
+ class OOPSTestEnsBuild(
155
+ OOPSParallel, _OOPSTestDecoMixin, OOPSMembersTermsDecoMixin
156
+ ):
129
157
  """OOPS Tests without ODB: ensemble/build specific case"""
130
158
 
131
159
  _footprint = dict(
132
- info = "OOPS Test run.",
133
- attr = dict(
134
- kind = dict(
135
- values = ['ootest'],
160
+ info="OOPS Test run.",
161
+ attr=dict(
162
+ kind=dict(
163
+ values=["ootest"],
136
164
  ),
137
- test_type = dict(
138
- values = ['ensemble/build', ]
165
+ test_type=dict(
166
+ values=[
167
+ "ensemble/build",
168
+ ]
139
169
  ),
140
- )
170
+ ),
141
171
  )
142
172
 
143
173
 
144
- class OOPSObsOpTest(OOPSODB, _OOPSTestDecoMixin, _OOPSTestExpTargetDecoMixin,
145
- OOPSMembersTermsDetectDecoMixin):
174
+ class OOPSObsOpTest(
175
+ OOPSODB,
176
+ _OOPSTestDecoMixin,
177
+ _OOPSTestExpTargetDecoMixin,
178
+ OOPSMembersTermsDetectDecoMixin,
179
+ ):
146
180
  """OOPS Obs Operators Tests."""
147
181
 
148
182
  _footprint = dict(
149
- info = "OOPS Obs Operators Tests.",
150
- attr = dict(
151
- kind = dict(
152
- values = ['ootestobs'],
183
+ info="OOPS Obs Operators Tests.",
184
+ attr=dict(
185
+ kind=dict(
186
+ values=["ootestobs"],
153
187
  ),
154
- virtualdb = dict(
155
- default = 'ccma',
188
+ virtualdb=dict(
189
+ default="ccma",
156
190
  ),
157
- )
191
+ ),
158
192
  )
159
193
 
160
194
 
@@ -162,15 +196,15 @@ class OOPSecma2ccma(OOPSODB, _OOPSTestDecoMixin):
162
196
  """OOPS Test ECMA 2 CCMA completer."""
163
197
 
164
198
  _footprint = dict(
165
- info = "OOPS ECMA 2 CCMA completer.",
166
- attr = dict(
167
- kind = dict(
168
- values = ['ootest2ccma'],
199
+ info="OOPS ECMA 2 CCMA completer.",
200
+ attr=dict(
201
+ kind=dict(
202
+ values=["ootest2ccma"],
169
203
  ),
170
- virtualdb = dict(
171
- values = ['ecma'],
204
+ virtualdb=dict(
205
+ values=["ecma"],
172
206
  ),
173
- )
207
+ ),
174
208
  )
175
209
 
176
210
  def postfix(self, rh, opts):
@@ -182,5 +216,5 @@ class OOPSecma2ccma(OOPSODB, _OOPSTestDecoMixin):
182
216
  """Make the appropriate renaming of files in ECMA to CCMA."""
183
217
  for e in self.lookupodb():
184
218
  edir = e.rh.container.localpath()
185
- self.odb.change_layout('ECMA', 'CCMA', edir)
186
- self.system.mv(edir, edir.replace('ECMA', 'CCMA'))
219
+ self.odb.change_layout("ECMA", "CCMA", edir)
220
+ self.system.mv(edir, edir.replace("ECMA", "CCMA"))