vortex-nwp 2.0.0b1__py3-none-any.whl → 2.0.0b2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (139) hide show
  1. vortex/__init__.py +59 -45
  2. vortex/algo/__init__.py +3 -2
  3. vortex/algo/components.py +940 -614
  4. vortex/algo/mpitools.py +802 -497
  5. vortex/algo/serversynctools.py +34 -33
  6. vortex/config.py +19 -22
  7. vortex/data/__init__.py +9 -3
  8. vortex/data/abstractstores.py +593 -655
  9. vortex/data/containers.py +217 -162
  10. vortex/data/contents.py +65 -39
  11. vortex/data/executables.py +93 -102
  12. vortex/data/flow.py +40 -34
  13. vortex/data/geometries.py +228 -132
  14. vortex/data/handlers.py +428 -225
  15. vortex/data/outflow.py +15 -15
  16. vortex/data/providers.py +185 -163
  17. vortex/data/resources.py +48 -42
  18. vortex/data/stores.py +544 -413
  19. vortex/gloves.py +114 -87
  20. vortex/layout/__init__.py +1 -8
  21. vortex/layout/contexts.py +150 -84
  22. vortex/layout/dataflow.py +353 -202
  23. vortex/layout/monitor.py +264 -128
  24. vortex/nwp/__init__.py +5 -2
  25. vortex/nwp/algo/__init__.py +14 -5
  26. vortex/nwp/algo/assim.py +205 -151
  27. vortex/nwp/algo/clim.py +683 -517
  28. vortex/nwp/algo/coupling.py +447 -225
  29. vortex/nwp/algo/eda.py +437 -229
  30. vortex/nwp/algo/eps.py +403 -231
  31. vortex/nwp/algo/forecasts.py +420 -271
  32. vortex/nwp/algo/fpserver.py +683 -307
  33. vortex/nwp/algo/ifsnaming.py +205 -145
  34. vortex/nwp/algo/ifsroot.py +210 -122
  35. vortex/nwp/algo/monitoring.py +132 -76
  36. vortex/nwp/algo/mpitools.py +321 -191
  37. vortex/nwp/algo/odbtools.py +617 -353
  38. vortex/nwp/algo/oopsroot.py +449 -273
  39. vortex/nwp/algo/oopstests.py +90 -56
  40. vortex/nwp/algo/request.py +287 -206
  41. vortex/nwp/algo/stdpost.py +878 -522
  42. vortex/nwp/data/__init__.py +22 -4
  43. vortex/nwp/data/assim.py +125 -137
  44. vortex/nwp/data/boundaries.py +121 -68
  45. vortex/nwp/data/climfiles.py +193 -211
  46. vortex/nwp/data/configfiles.py +73 -69
  47. vortex/nwp/data/consts.py +426 -401
  48. vortex/nwp/data/ctpini.py +59 -43
  49. vortex/nwp/data/diagnostics.py +94 -66
  50. vortex/nwp/data/eda.py +50 -51
  51. vortex/nwp/data/eps.py +195 -146
  52. vortex/nwp/data/executables.py +440 -434
  53. vortex/nwp/data/fields.py +63 -48
  54. vortex/nwp/data/gridfiles.py +183 -111
  55. vortex/nwp/data/logs.py +250 -217
  56. vortex/nwp/data/modelstates.py +180 -151
  57. vortex/nwp/data/monitoring.py +72 -99
  58. vortex/nwp/data/namelists.py +254 -202
  59. vortex/nwp/data/obs.py +400 -308
  60. vortex/nwp/data/oopsexec.py +22 -20
  61. vortex/nwp/data/providers.py +90 -65
  62. vortex/nwp/data/query.py +71 -82
  63. vortex/nwp/data/stores.py +49 -36
  64. vortex/nwp/data/surfex.py +136 -137
  65. vortex/nwp/syntax/__init__.py +1 -1
  66. vortex/nwp/syntax/stdattrs.py +173 -111
  67. vortex/nwp/tools/__init__.py +2 -2
  68. vortex/nwp/tools/addons.py +22 -17
  69. vortex/nwp/tools/agt.py +24 -12
  70. vortex/nwp/tools/bdap.py +16 -5
  71. vortex/nwp/tools/bdcp.py +4 -1
  72. vortex/nwp/tools/bdm.py +3 -0
  73. vortex/nwp/tools/bdmp.py +14 -9
  74. vortex/nwp/tools/conftools.py +728 -378
  75. vortex/nwp/tools/drhook.py +12 -8
  76. vortex/nwp/tools/grib.py +65 -39
  77. vortex/nwp/tools/gribdiff.py +22 -17
  78. vortex/nwp/tools/ifstools.py +82 -42
  79. vortex/nwp/tools/igastuff.py +167 -143
  80. vortex/nwp/tools/mars.py +14 -2
  81. vortex/nwp/tools/odb.py +234 -125
  82. vortex/nwp/tools/partitioning.py +61 -37
  83. vortex/nwp/tools/satrad.py +27 -12
  84. vortex/nwp/util/async.py +83 -55
  85. vortex/nwp/util/beacon.py +10 -10
  86. vortex/nwp/util/diffpygram.py +174 -86
  87. vortex/nwp/util/ens.py +144 -63
  88. vortex/nwp/util/hooks.py +30 -19
  89. vortex/nwp/util/taskdeco.py +28 -24
  90. vortex/nwp/util/usepygram.py +278 -172
  91. vortex/nwp/util/usetnt.py +31 -17
  92. vortex/sessions.py +72 -39
  93. vortex/syntax/__init__.py +1 -1
  94. vortex/syntax/stdattrs.py +410 -171
  95. vortex/syntax/stddeco.py +31 -22
  96. vortex/toolbox.py +327 -192
  97. vortex/tools/__init__.py +11 -2
  98. vortex/tools/actions.py +125 -59
  99. vortex/tools/addons.py +111 -92
  100. vortex/tools/arm.py +42 -22
  101. vortex/tools/compression.py +72 -69
  102. vortex/tools/date.py +11 -4
  103. vortex/tools/delayedactions.py +242 -132
  104. vortex/tools/env.py +75 -47
  105. vortex/tools/folder.py +342 -171
  106. vortex/tools/grib.py +311 -149
  107. vortex/tools/lfi.py +423 -216
  108. vortex/tools/listings.py +109 -40
  109. vortex/tools/names.py +218 -156
  110. vortex/tools/net.py +632 -298
  111. vortex/tools/parallelism.py +93 -61
  112. vortex/tools/prestaging.py +55 -31
  113. vortex/tools/schedulers.py +172 -105
  114. vortex/tools/services.py +402 -333
  115. vortex/tools/storage.py +293 -358
  116. vortex/tools/surfex.py +24 -24
  117. vortex/tools/systems.py +1211 -631
  118. vortex/tools/targets.py +156 -100
  119. vortex/util/__init__.py +1 -1
  120. vortex/util/config.py +377 -327
  121. vortex/util/empty.py +2 -2
  122. vortex/util/helpers.py +56 -24
  123. vortex/util/introspection.py +18 -12
  124. vortex/util/iosponge.py +8 -4
  125. vortex/util/roles.py +4 -6
  126. vortex/util/storefunctions.py +39 -13
  127. vortex/util/structs.py +3 -3
  128. vortex/util/worker.py +29 -17
  129. vortex_nwp-2.0.0b2.dist-info/METADATA +66 -0
  130. vortex_nwp-2.0.0b2.dist-info/RECORD +142 -0
  131. {vortex_nwp-2.0.0b1.dist-info → vortex_nwp-2.0.0b2.dist-info}/WHEEL +1 -1
  132. vortex/layout/appconf.py +0 -109
  133. vortex/layout/jobs.py +0 -1276
  134. vortex/layout/nodes.py +0 -1424
  135. vortex/layout/subjobs.py +0 -464
  136. vortex_nwp-2.0.0b1.dist-info/METADATA +0 -50
  137. vortex_nwp-2.0.0b1.dist-info/RECORD +0 -146
  138. {vortex_nwp-2.0.0b1.dist-info → vortex_nwp-2.0.0b2.dist-info}/LICENSE +0 -0
  139. {vortex_nwp-2.0.0b1.dist-info → vortex_nwp-2.0.0b2.dist-info}/top_level.txt +0 -0
@@ -6,9 +6,17 @@ import json
6
6
 
7
7
  import footprints
8
8
 
9
- from vortex.algo.components import AlgoComponentDecoMixin, algo_component_deco_mixin_autodoc
9
+ from vortex.algo.components import (
10
+ AlgoComponentDecoMixin,
11
+ algo_component_deco_mixin_autodoc,
12
+ )
10
13
  from ..syntax.stdattrs import oops_test_type, oops_expected_target
11
- from .oopsroot import OOPSParallel, OOPSODB, OOPSMembersTermsDecoMixin, OOPSMembersTermsDetectDecoMixin
14
+ from .oopsroot import (
15
+ OOPSParallel,
16
+ OOPSODB,
17
+ OOPSMembersTermsDecoMixin,
18
+ OOPSMembersTermsDetectDecoMixin,
19
+ )
12
20
 
13
21
  #: No automatic export
14
22
  __all__ = []
@@ -25,14 +33,14 @@ class _OOPSTestDecoMixin(AlgoComponentDecoMixin):
25
33
  the dictionary that is used to build the binary' command line.
26
34
  """
27
35
 
28
- _MIXIN_EXTRA_FOOTPRINTS = (oops_test_type, )
36
+ _MIXIN_EXTRA_FOOTPRINTS = (oops_test_type,)
29
37
 
30
38
  def _ooptest_cli_opts_extend(self, prev):
31
39
  """Prepare options for the resource's command line."""
32
- prev['test_type'] = self.test_type
40
+ prev["test_type"] = self.test_type
33
41
  return prev
34
42
 
35
- _MIXIN_CLI_OPTS_EXTEND = (_ooptest_cli_opts_extend, )
43
+ _MIXIN_CLI_OPTS_EXTEND = (_ooptest_cli_opts_extend,)
36
44
 
37
45
 
38
46
  @algo_component_deco_mixin_autodoc
@@ -61,11 +69,13 @@ class _OOPSTestExpTargetDecoMixin(AlgoComponentDecoMixin):
61
69
  # if attribute 'expected_target' is attribute and given to the algo, use it
62
70
  target = self._set_expected_target_from_attribute()
63
71
  # else, go find Reference summary in effective inputs
64
- if target is not None and target.get('from') == 'reference_summary':
72
+ if target is not None and target.get("from") == "reference_summary":
65
73
  target = self._set_expected_target_from_reference_summary()
66
74
  # Else, default to be sure to pass any in-binary-test
67
75
  if target is None:
68
- target = self._set_expected_target_default() # CLEANME: to be removed after CY47 ?
76
+ target = (
77
+ self._set_expected_target_default()
78
+ ) # CLEANME: to be removed after CY47 ?
69
79
  # Then in the end, export variable
70
80
  target = json.dumps(target)
71
81
  logger.info("Expected Target for Test: " + target)
@@ -73,88 +83,112 @@ class _OOPSTestExpTargetDecoMixin(AlgoComponentDecoMixin):
73
83
 
74
84
  def _set_expected_target_from_attribute(self):
75
85
  """Read target in Algo attribute."""
76
- if hasattr(self, 'expected_target'):
86
+ if hasattr(self, "expected_target"):
77
87
  if self.expected_target is not None:
78
88
  target = self.expected_target
79
- logger.info('Set EXPECTED_RESULT from Attribute')
89
+ logger.info("Set EXPECTED_RESULT from Attribute")
80
90
  return target
81
91
 
82
92
  def _set_expected_target_from_reference_summary(self):
83
93
  """Read target in ReferenceSummary effective input"""
84
94
  target = None
85
- ref_summary = [s for s in self.context.sequence.effective_inputs(role=('Reference',))
86
- if s.rh.resource.kind == 'taskinfo']
95
+ ref_summary = [
96
+ s
97
+ for s in self.context.sequence.effective_inputs(
98
+ role=("Reference",)
99
+ )
100
+ if s.rh.resource.kind == "taskinfo"
101
+ ]
87
102
  if len(ref_summary) > 0:
88
103
  ref_summary = ref_summary[0].rh.contents.data
89
- target = ref_summary.get('oops:' + self.test_type,
90
- {}).get('as EXPECTED_RESULT', None)
104
+ target = ref_summary.get("oops:" + self.test_type, {}).get(
105
+ "as EXPECTED_RESULT", None
106
+ )
91
107
  if target is not None:
92
- logger.info('Set EXPECTED_RESULT from Reference summary')
108
+ logger.info("Set EXPECTED_RESULT from Reference summary")
93
109
  return target
94
110
 
95
- def _set_expected_target_default(self): # CLEANME: to be removed after CY47 ?
111
+ def _set_expected_target_default(
112
+ self,
113
+ ): # CLEANME: to be removed after CY47 ?
96
114
  """Set default, for binary not to crash before CY47."""
97
- target = {'significant_digits': '-9',
98
- 'expected_Jo': '9999',
99
- 'expected_variances': '9999',
100
- 'expected_diff': '9999'}
101
- logger.info('Set default EXPECTED_RESULT')
115
+ target = {
116
+ "significant_digits": "-9",
117
+ "expected_Jo": "9999",
118
+ "expected_variances": "9999",
119
+ "expected_diff": "9999",
120
+ }
121
+ logger.info("Set default EXPECTED_RESULT")
102
122
  return target
103
123
 
104
124
  def _ooptest_exptarget_prepare_hook(self, rh, opts):
105
125
  """Call set_expected_target juste after prepare."""
106
126
  self.set_expected_target()
107
127
 
108
- _MIXIN_PREPARE_HOOKS = (_ooptest_exptarget_prepare_hook, )
128
+ _MIXIN_PREPARE_HOOKS = (_ooptest_exptarget_prepare_hook,)
109
129
 
110
130
 
111
- class OOPSTest(OOPSParallel, _OOPSTestDecoMixin, _OOPSTestExpTargetDecoMixin,
112
- OOPSMembersTermsDetectDecoMixin):
131
+ class OOPSTest(
132
+ OOPSParallel,
133
+ _OOPSTestDecoMixin,
134
+ _OOPSTestExpTargetDecoMixin,
135
+ OOPSMembersTermsDetectDecoMixin,
136
+ ):
113
137
  """OOPS Tests without ODB."""
114
138
 
115
139
  _footprint = dict(
116
- info = "OOPS Test run.",
117
- attr = dict(
118
- kind = dict(
119
- values = ['ootest'],
140
+ info="OOPS Test run.",
141
+ attr=dict(
142
+ kind=dict(
143
+ values=["ootest"],
120
144
  ),
121
- test_type = dict(
122
- outcast = ['ensemble/build', ]
145
+ test_type=dict(
146
+ outcast=[
147
+ "ensemble/build",
148
+ ]
123
149
  ),
124
- )
150
+ ),
125
151
  )
126
152
 
127
153
 
128
- class OOPSTestEnsBuild(OOPSParallel, _OOPSTestDecoMixin, OOPSMembersTermsDecoMixin):
154
+ class OOPSTestEnsBuild(
155
+ OOPSParallel, _OOPSTestDecoMixin, OOPSMembersTermsDecoMixin
156
+ ):
129
157
  """OOPS Tests without ODB: ensemble/build specific case"""
130
158
 
131
159
  _footprint = dict(
132
- info = "OOPS Test run.",
133
- attr = dict(
134
- kind = dict(
135
- values = ['ootest'],
160
+ info="OOPS Test run.",
161
+ attr=dict(
162
+ kind=dict(
163
+ values=["ootest"],
136
164
  ),
137
- test_type = dict(
138
- values = ['ensemble/build', ]
165
+ test_type=dict(
166
+ values=[
167
+ "ensemble/build",
168
+ ]
139
169
  ),
140
- )
170
+ ),
141
171
  )
142
172
 
143
173
 
144
- class OOPSObsOpTest(OOPSODB, _OOPSTestDecoMixin, _OOPSTestExpTargetDecoMixin,
145
- OOPSMembersTermsDetectDecoMixin):
174
+ class OOPSObsOpTest(
175
+ OOPSODB,
176
+ _OOPSTestDecoMixin,
177
+ _OOPSTestExpTargetDecoMixin,
178
+ OOPSMembersTermsDetectDecoMixin,
179
+ ):
146
180
  """OOPS Obs Operators Tests."""
147
181
 
148
182
  _footprint = dict(
149
- info = "OOPS Obs Operators Tests.",
150
- attr = dict(
151
- kind = dict(
152
- values = ['ootestobs'],
183
+ info="OOPS Obs Operators Tests.",
184
+ attr=dict(
185
+ kind=dict(
186
+ values=["ootestobs"],
153
187
  ),
154
- virtualdb = dict(
155
- default = 'ccma',
188
+ virtualdb=dict(
189
+ default="ccma",
156
190
  ),
157
- )
191
+ ),
158
192
  )
159
193
 
160
194
 
@@ -162,15 +196,15 @@ class OOPSecma2ccma(OOPSODB, _OOPSTestDecoMixin):
162
196
  """OOPS Test ECMA 2 CCMA completer."""
163
197
 
164
198
  _footprint = dict(
165
- info = "OOPS ECMA 2 CCMA completer.",
166
- attr = dict(
167
- kind = dict(
168
- values = ['ootest2ccma'],
199
+ info="OOPS ECMA 2 CCMA completer.",
200
+ attr=dict(
201
+ kind=dict(
202
+ values=["ootest2ccma"],
169
203
  ),
170
- virtualdb = dict(
171
- values = ['ecma'],
204
+ virtualdb=dict(
205
+ values=["ecma"],
172
206
  ),
173
- )
207
+ ),
174
208
  )
175
209
 
176
210
  def postfix(self, rh, opts):
@@ -182,5 +216,5 @@ class OOPSecma2ccma(OOPSODB, _OOPSTestDecoMixin):
182
216
  """Make the appropriate renaming of files in ECMA to CCMA."""
183
217
  for e in self.lookupodb():
184
218
  edir = e.rh.container.localpath()
185
- self.odb.change_layout('ECMA', 'CCMA', edir)
186
- self.system.mv(edir, edir.replace('ECMA', 'CCMA'))
219
+ self.odb.change_layout("ECMA", "CCMA", edir)
220
+ self.system.mv(edir, edir.replace("ECMA", "CCMA"))