vortex-nwp 2.0.0b1__py3-none-any.whl → 2.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (141) hide show
  1. vortex/__init__.py +75 -47
  2. vortex/algo/__init__.py +3 -2
  3. vortex/algo/components.py +944 -618
  4. vortex/algo/mpitools.py +802 -497
  5. vortex/algo/mpitools_templates/__init__.py +1 -0
  6. vortex/algo/serversynctools.py +34 -33
  7. vortex/config.py +19 -22
  8. vortex/data/__init__.py +9 -3
  9. vortex/data/abstractstores.py +593 -655
  10. vortex/data/containers.py +217 -162
  11. vortex/data/contents.py +65 -39
  12. vortex/data/executables.py +93 -102
  13. vortex/data/flow.py +40 -34
  14. vortex/data/geometries.py +228 -132
  15. vortex/data/handlers.py +436 -227
  16. vortex/data/outflow.py +15 -15
  17. vortex/data/providers.py +185 -163
  18. vortex/data/resources.py +48 -42
  19. vortex/data/stores.py +540 -417
  20. vortex/data/sync_templates/__init__.py +0 -0
  21. vortex/gloves.py +114 -87
  22. vortex/layout/__init__.py +1 -8
  23. vortex/layout/contexts.py +150 -84
  24. vortex/layout/dataflow.py +353 -202
  25. vortex/layout/monitor.py +264 -128
  26. vortex/nwp/__init__.py +5 -2
  27. vortex/nwp/algo/__init__.py +14 -5
  28. vortex/nwp/algo/assim.py +205 -151
  29. vortex/nwp/algo/clim.py +683 -517
  30. vortex/nwp/algo/coupling.py +447 -225
  31. vortex/nwp/algo/eda.py +437 -229
  32. vortex/nwp/algo/eps.py +403 -231
  33. vortex/nwp/algo/forecasts.py +416 -275
  34. vortex/nwp/algo/fpserver.py +683 -307
  35. vortex/nwp/algo/ifsnaming.py +205 -145
  36. vortex/nwp/algo/ifsroot.py +215 -122
  37. vortex/nwp/algo/monitoring.py +137 -76
  38. vortex/nwp/algo/mpitools.py +330 -190
  39. vortex/nwp/algo/odbtools.py +637 -353
  40. vortex/nwp/algo/oopsroot.py +454 -273
  41. vortex/nwp/algo/oopstests.py +90 -56
  42. vortex/nwp/algo/request.py +287 -206
  43. vortex/nwp/algo/stdpost.py +878 -522
  44. vortex/nwp/data/__init__.py +22 -4
  45. vortex/nwp/data/assim.py +125 -137
  46. vortex/nwp/data/boundaries.py +121 -68
  47. vortex/nwp/data/climfiles.py +193 -211
  48. vortex/nwp/data/configfiles.py +73 -69
  49. vortex/nwp/data/consts.py +426 -401
  50. vortex/nwp/data/ctpini.py +59 -43
  51. vortex/nwp/data/diagnostics.py +94 -66
  52. vortex/nwp/data/eda.py +50 -51
  53. vortex/nwp/data/eps.py +195 -146
  54. vortex/nwp/data/executables.py +440 -434
  55. vortex/nwp/data/fields.py +63 -48
  56. vortex/nwp/data/gridfiles.py +183 -111
  57. vortex/nwp/data/logs.py +250 -217
  58. vortex/nwp/data/modelstates.py +180 -151
  59. vortex/nwp/data/monitoring.py +72 -99
  60. vortex/nwp/data/namelists.py +254 -202
  61. vortex/nwp/data/obs.py +400 -308
  62. vortex/nwp/data/oopsexec.py +22 -20
  63. vortex/nwp/data/providers.py +90 -65
  64. vortex/nwp/data/query.py +71 -82
  65. vortex/nwp/data/stores.py +49 -36
  66. vortex/nwp/data/surfex.py +136 -137
  67. vortex/nwp/syntax/__init__.py +1 -1
  68. vortex/nwp/syntax/stdattrs.py +173 -111
  69. vortex/nwp/tools/__init__.py +2 -2
  70. vortex/nwp/tools/addons.py +22 -17
  71. vortex/nwp/tools/agt.py +24 -12
  72. vortex/nwp/tools/bdap.py +16 -5
  73. vortex/nwp/tools/bdcp.py +4 -1
  74. vortex/nwp/tools/bdm.py +3 -0
  75. vortex/nwp/tools/bdmp.py +14 -9
  76. vortex/nwp/tools/conftools.py +728 -378
  77. vortex/nwp/tools/drhook.py +12 -8
  78. vortex/nwp/tools/grib.py +65 -39
  79. vortex/nwp/tools/gribdiff.py +22 -17
  80. vortex/nwp/tools/ifstools.py +82 -42
  81. vortex/nwp/tools/igastuff.py +167 -143
  82. vortex/nwp/tools/mars.py +14 -2
  83. vortex/nwp/tools/odb.py +234 -125
  84. vortex/nwp/tools/partitioning.py +61 -37
  85. vortex/nwp/tools/satrad.py +27 -12
  86. vortex/nwp/util/async.py +83 -55
  87. vortex/nwp/util/beacon.py +10 -10
  88. vortex/nwp/util/diffpygram.py +174 -86
  89. vortex/nwp/util/ens.py +144 -63
  90. vortex/nwp/util/hooks.py +30 -19
  91. vortex/nwp/util/taskdeco.py +28 -24
  92. vortex/nwp/util/usepygram.py +278 -172
  93. vortex/nwp/util/usetnt.py +31 -17
  94. vortex/sessions.py +72 -39
  95. vortex/syntax/__init__.py +1 -1
  96. vortex/syntax/stdattrs.py +410 -171
  97. vortex/syntax/stddeco.py +31 -22
  98. vortex/toolbox.py +327 -192
  99. vortex/tools/__init__.py +11 -2
  100. vortex/tools/actions.py +110 -121
  101. vortex/tools/addons.py +111 -92
  102. vortex/tools/arm.py +42 -22
  103. vortex/tools/compression.py +72 -69
  104. vortex/tools/date.py +11 -4
  105. vortex/tools/delayedactions.py +242 -132
  106. vortex/tools/env.py +75 -47
  107. vortex/tools/folder.py +342 -171
  108. vortex/tools/grib.py +341 -162
  109. vortex/tools/lfi.py +423 -216
  110. vortex/tools/listings.py +109 -40
  111. vortex/tools/names.py +218 -156
  112. vortex/tools/net.py +655 -299
  113. vortex/tools/parallelism.py +93 -61
  114. vortex/tools/prestaging.py +55 -31
  115. vortex/tools/schedulers.py +172 -105
  116. vortex/tools/services.py +403 -334
  117. vortex/tools/storage.py +293 -358
  118. vortex/tools/surfex.py +24 -24
  119. vortex/tools/systems.py +1234 -643
  120. vortex/tools/targets.py +156 -100
  121. vortex/util/__init__.py +1 -1
  122. vortex/util/config.py +378 -327
  123. vortex/util/empty.py +2 -2
  124. vortex/util/helpers.py +56 -24
  125. vortex/util/introspection.py +18 -12
  126. vortex/util/iosponge.py +8 -4
  127. vortex/util/roles.py +4 -6
  128. vortex/util/storefunctions.py +39 -13
  129. vortex/util/structs.py +3 -3
  130. vortex/util/worker.py +29 -17
  131. vortex_nwp-2.1.0.dist-info/METADATA +67 -0
  132. vortex_nwp-2.1.0.dist-info/RECORD +144 -0
  133. {vortex_nwp-2.0.0b1.dist-info → vortex_nwp-2.1.0.dist-info}/WHEEL +1 -1
  134. vortex/layout/appconf.py +0 -109
  135. vortex/layout/jobs.py +0 -1276
  136. vortex/layout/nodes.py +0 -1424
  137. vortex/layout/subjobs.py +0 -464
  138. vortex_nwp-2.0.0b1.dist-info/METADATA +0 -50
  139. vortex_nwp-2.0.0b1.dist-info/RECORD +0 -146
  140. {vortex_nwp-2.0.0b1.dist-info → vortex_nwp-2.1.0.dist-info/licenses}/LICENSE +0 -0
  141. {vortex_nwp-2.0.0b1.dist-info → vortex_nwp-2.1.0.dist-info}/top_level.txt +0 -0
vortex/nwp/tools/odb.py CHANGED
@@ -8,7 +8,12 @@ from bronx.fancies import loggers
8
8
  from bronx.stdtypes import date as bdate
9
9
  import footprints
10
10
 
11
- from vortex.algo.components import AlgoComponentDecoMixin, AlgoComponentError, algo_component_deco_mixin_autodoc
11
+ from vortex.algo.components import (
12
+ AlgoComponentDecoMixin,
13
+ AlgoComponentError,
14
+ algo_component_deco_mixin_autodoc,
15
+ )
16
+ from vortex import config
12
17
  from vortex.layout.dataflow import intent
13
18
 
14
19
  from ..syntax.stdattrs import ArpIfsSimplifiedCycle
@@ -22,16 +27,18 @@ logger = loggers.getLogger(__name__)
22
27
  class TimeSlots:
23
28
  """Handling of assimilation time slots."""
24
29
 
25
- def __init__(self, nslot=7, start='-PT3H', window='PT6H', chunk=None, center=True):
30
+ def __init__(
31
+ self, nslot=7, start="-PT3H", window="PT6H", chunk=None, center=True
32
+ ):
26
33
  if isinstance(nslot, str):
27
- info = [x.strip() for x in nslot.split('/')]
34
+ info = [x.strip() for x in nslot.split("/")]
28
35
  nslot = info[0]
29
36
  if len(info) > 1:
30
37
  start = info[1]
31
38
  if len(info) > 2:
32
39
  window = info[2]
33
40
  if len(info) > 3:
34
- if re.match('^regular', info[3]):
41
+ if re.match("^regular", info[3]):
35
42
  center = False
36
43
  else:
37
44
  chunk = info[3]
@@ -41,7 +48,9 @@ class TimeSlots:
41
48
  self.window = bdate.Period(window)
42
49
  if chunk is None:
43
50
  cslot = self.nslot - 1 if self.center else self.nslot
44
- chunk = 'PT' + str((self.window.length // max(1, cslot)) // 60) + 'M'
51
+ chunk = (
52
+ "PT" + str((self.window.length // max(1, cslot)) // 60) + "M"
53
+ )
45
54
  self.chunk = self.window if self.nslot < 2 else bdate.Period(chunk)
46
55
 
47
56
  def __eq__(self, other):
@@ -50,31 +59,38 @@ class TimeSlots:
50
59
  other = TimeSlots(other)
51
60
  except ValueError:
52
61
  pass
53
- return (isinstance(other, TimeSlots) and
54
- self.nslot == other.nslot and self.center == other.center and
55
- self.start == other.start and self.window == other.window and
56
- self.chunk == other.chunk)
62
+ return (
63
+ isinstance(other, TimeSlots)
64
+ and self.nslot == other.nslot
65
+ and self.center == other.center
66
+ and self.start == other.start
67
+ and self.window == other.window
68
+ and self.chunk == other.chunk
69
+ )
57
70
 
58
71
  def __str__(self):
59
- chunky = self.chunk.isoformat() if self.center else 'regular'
60
- return '{0.nslot:d}/{1:s}/{2:s}/{3:s}'.format(self,
61
- self.start.isoformat(),
62
- self.window.isoformat(),
63
- chunky)
72
+ chunky = self.chunk.isoformat() if self.center else "regular"
73
+ return "{0.nslot:d}/{1:s}/{2:s}/{3:s}".format(
74
+ self, self.start.isoformat(), self.window.isoformat(), chunky
75
+ )
64
76
 
65
77
  def __repr__(self, *args, **kwargs):
66
- return super().__repr__()[:-1] + ' | {!s}>'.format(self)
78
+ return super().__repr__()[:-1] + " | {!s}>".format(self)
67
79
 
68
80
  def as_slots(self):
69
81
  """Return a list of slots in seconds."""
70
82
  if self.center:
71
- slots = [self.chunk.length, ] * self.nslot
83
+ slots = [
84
+ self.chunk.length,
85
+ ] * self.nslot
72
86
  nb = self.window.length // self.chunk.length
73
87
  if nb != self.nslot:
74
88
  slots[0] = slots[-1] = self.chunk.length // 2
75
89
  else:
76
90
  islot = self.window.length // self.nslot
77
- slots = [islot, ] * self.nslot
91
+ slots = [
92
+ islot,
93
+ ] * self.nslot
78
94
  return slots
79
95
 
80
96
  def as_centers_fromstart(self):
@@ -85,7 +101,9 @@ class TimeSlots:
85
101
  for i in range(len(slots)):
86
102
  fromstart.append(acc + slots[i] / 2)
87
103
  acc += slots[i]
88
- if self.center and (self.window.length // self.chunk.length != self.nslot):
104
+ if self.center and (
105
+ self.window.length // self.chunk.length != self.nslot
106
+ ):
89
107
  fromstart[0] = 0
90
108
  fromstart[-1] = self.window.length
91
109
  return [bdate.Period(seconds=t) for t in fromstart]
@@ -93,7 +111,9 @@ class TimeSlots:
93
111
  def as_bounds(self, date):
94
112
  """Return time slots as a list of compact date values."""
95
113
  date = bdate.Date(date)
96
- boundlist = [date + self.start, ]
114
+ boundlist = [
115
+ date + self.start,
116
+ ]
97
117
  for x in self.as_slots():
98
118
  boundlist.append(boundlist[-1] + x)
99
119
  boundlist = [x.compact() for x in boundlist]
@@ -111,16 +131,21 @@ class TimeSlots:
111
131
 
112
132
  def as_environment(self):
113
133
  """Return a dictionary of ready-to-export variables that describe the timeslots."""
114
- thelen = self.chunk.length // 60 if self.center and self.nslot > 1 else 0
115
- return dict(BATOR_WINDOW_LEN=self.window.length // 60,
116
- BATOR_WINDOW_SHIFT=int(self.start.total_seconds()) // 60,
117
- BATOR_SLOT_LEN=thelen, BATOR_CENTER_LEN=thelen)
134
+ thelen = (
135
+ self.chunk.length // 60 if self.center and self.nslot > 1 else 0
136
+ )
137
+ return dict(
138
+ BATOR_WINDOW_LEN=self.window.length // 60,
139
+ BATOR_WINDOW_SHIFT=int(self.start.total_seconds()) // 60,
140
+ BATOR_SLOT_LEN=thelen,
141
+ BATOR_CENTER_LEN=thelen,
142
+ )
118
143
 
119
144
  def as_file(self, date, filename):
120
145
  """Fill the specified ``filename`` wih the current list of time slots at this ``date``."""
121
- with open(filename, 'w') as fd:
146
+ with open(filename, "w") as fd:
122
147
  for x in self.as_bounds(date):
123
- fd.write(str(x) + '\n')
148
+ fd.write(str(x) + "\n")
124
149
  nbx = fd.tell()
125
150
  return nbx
126
151
 
@@ -136,15 +161,23 @@ class OdbDriver:
136
161
  self.cycle = cycle
137
162
  self.sh = sh
138
163
  if self.sh is None:
139
- logger.critical('%s created with a proper shell access [%s]', self.__class__, self)
164
+ logger.critical(
165
+ "%s created with a proper shell access [%s]",
166
+ self.__class__,
167
+ self,
168
+ )
140
169
  self.env = env
141
170
  if self.env is None:
142
- logger.critical('%s created with a proper environment access [%s]', self.__class__, self)
171
+ logger.critical(
172
+ "%s created with a proper environment access [%s]",
173
+ self.__class__,
174
+ self,
175
+ )
143
176
 
144
- def setup(self, date, npool=1, nslot=1, iomethod=1, layout='ecma'):
177
+ def setup(self, date, npool=1, nslot=1, iomethod=1, layout="ecma"):
145
178
  """Setup given environment with default ODB env variables."""
146
179
 
147
- logger.info('ODB: generic setup called.'),
180
+ (logger.info("ODB: generic setup called."),)
148
181
  self.env.update(
149
182
  ODB_CMA=layout.upper(),
150
183
  ODB_IO_METHOD=iomethod,
@@ -156,7 +189,7 @@ class OdbDriver:
156
189
  ODB_REPRODUCIBLE_SEQNO=4,
157
190
  ODB_STATIC_LINKING=1,
158
191
  ODB_ANALYSIS_DATE=date.ymd,
159
- ODB_ANALYSIS_TIME=date.hm + '00',
192
+ ODB_ANALYSIS_TIME=date.hm + "00",
160
193
  TO_ODB_ECMWF=0,
161
194
  TO_ODB_SWAPOUT=0,
162
195
  )
@@ -167,15 +200,17 @@ class OdbDriver:
167
200
  ODB_IO_FILESIZE=128,
168
201
  )
169
202
 
170
- if self.sh.path.exists('IOASSIGN'):
203
+ if self.sh.path.exists("IOASSIGN"):
171
204
  self.env.default(
172
- IOASSIGN=self.sh.path.abspath('IOASSIGN'),
205
+ IOASSIGN=self.sh.path.abspath("IOASSIGN"),
173
206
  )
174
207
 
175
208
  def force_overwrite_method(self):
176
209
  """Force ODB_OVERWRITE_METHOD if necessary."""
177
- if not int(self.env.get('ODB_OVERWRITE_METHOD', 0)):
178
- logger.info('ODB: Some input ODB databases are read-only. Setting ODB_OVERWRITE_METHOD to 1.')
210
+ if not int(self.env.get("ODB_OVERWRITE_METHOD", 0)):
211
+ logger.info(
212
+ "ODB: Some input ODB databases are read-only. Setting ODB_OVERWRITE_METHOD to 1."
213
+ )
179
214
  self.env.ODB_OVERWRITE_METHOD = 1
180
215
 
181
216
  def _process_layout_dbpath(self, layout, dbpath=None):
@@ -191,19 +226,19 @@ class OdbDriver:
191
226
  if env is None:
192
227
  env = self.env
193
228
  layout, dbpath, _ = self._process_layout_dbpath(layout, dbpath)
194
- logger.info('ODB: Fix %s path: %s', layout, dbpath)
195
- env['ODB_SRCPATH_{:s}'.format(layout)] = dbpath
196
- env['ODB_DATAPATH_{:s}'.format(layout)] = dbpath
229
+ logger.info("ODB: Fix %s path: %s", layout, dbpath)
230
+ env["ODB_SRCPATH_{:s}".format(layout)] = dbpath
231
+ env["ODB_DATAPATH_{:s}".format(layout)] = dbpath
197
232
 
198
233
  @property
199
234
  def _default_iocreate_path(self):
200
235
  """The location to the default create_ioassign utility."""
201
236
  return self.sh.path.join(
202
- from_config(section="nwp-tools", key="odb"),
203
- get_from_config_w_default(
237
+ config.from_config(section="nwp-tools", key="odb"),
238
+ config.get_from_config_w_default(
204
239
  section="nwp-tools",
205
240
  key="iocreate_cmd",
206
- default="create_ioassign"
241
+ default="create_ioassign",
207
242
  ),
208
243
  )
209
244
 
@@ -211,16 +246,22 @@ class OdbDriver:
211
246
  def _default_iomerge_path(self):
212
247
  """The location to the default merge_ioassign utility."""
213
248
  return self.sh.path.join(
214
- from_config(section="nwp-tools", key="odb"),
215
- get_from_config_w_default(
249
+ config.from_config(section="nwp-tools", key="odb"),
250
+ config.get_from_config_w_default(
216
251
  section="nwp-tools",
217
252
  key="iomerge_cmd",
218
- default="merge_ioassign"
253
+ default="merge_ioassign",
219
254
  ),
220
255
  )
221
256
 
222
- def ioassign_create(self, ioassign='ioassign.x', npool=1, layout='ecma',
223
- dbpath=None, iocreate_path=None):
257
+ def ioassign_create(
258
+ self,
259
+ ioassign="ioassign.x",
260
+ npool=1,
261
+ layout="ecma",
262
+ dbpath=None,
263
+ iocreate_path=None,
264
+ ):
224
265
  """Build IO-Assign table."""
225
266
  layout, dbpath, _ = self._process_layout_dbpath(layout, dbpath)
226
267
  if iocreate_path is None:
@@ -229,17 +270,28 @@ class OdbDriver:
229
270
  self.sh.xperm(ioassign, force=True)
230
271
  self.sh.mkdir(dbpath)
231
272
  with self.env.clone() as lenv:
232
- lenv['ODB_IOASSIGN_BINARY'] = ioassign
273
+ lenv["ODB_IOASSIGN_BINARY"] = ioassign
233
274
  self.fix_db_path(layout, dbpath, env=lenv)
234
- self.sh.spawn([iocreate_path,
235
- '-d' + dbpath,
236
- '-l' + layout,
237
- '-n' + str(npool)],
238
- output=False)
275
+ self.sh.spawn(
276
+ [
277
+ iocreate_path,
278
+ "-d" + dbpath,
279
+ "-l" + layout,
280
+ "-n" + str(npool),
281
+ ],
282
+ output=False,
283
+ )
239
284
  return dbpath
240
285
 
241
- def ioassign_merge(self, ioassign='ioassign.x', layout='ecma', odbnames=None,
242
- dbpath=None, iomerge_path=None, iocreate_path=None):
286
+ def ioassign_merge(
287
+ self,
288
+ ioassign="ioassign.x",
289
+ layout="ecma",
290
+ odbnames=None,
291
+ dbpath=None,
292
+ iomerge_path=None,
293
+ iocreate_path=None,
294
+ ):
243
295
  """Build IO-Assign table."""
244
296
  layout, dbpath, thispwd = self._process_layout_dbpath(layout, dbpath)
245
297
  if iomerge_path is None:
@@ -250,35 +302,39 @@ class OdbDriver:
250
302
  ioassign = self.sh.path.abspath(ioassign)
251
303
  self.sh.xperm(ioassign, force=True)
252
304
  with self.sh.cdcontext(dbpath, create=True):
253
- iocmd.extend(['-d', thispwd])
305
+ iocmd.extend(["-d", thispwd])
254
306
  for dbname in odbnames:
255
- iocmd.extend(['-t', dbname])
307
+ iocmd.extend(["-t", dbname])
256
308
  with self.env.clone() as lenv:
257
- lenv['ODB_IOASSIGN_BINARY'] = ioassign
258
- if 'ODB_IOCREATE_COMMAND' not in lenv:
259
- lenv['ODB_IOCREATE_COMMAND'] = iocreate_path
309
+ lenv["ODB_IOASSIGN_BINARY"] = ioassign
310
+ if "ODB_IOCREATE_COMMAND" not in lenv:
311
+ lenv["ODB_IOCREATE_COMMAND"] = iocreate_path
260
312
  self.fix_db_path(layout, dbpath, env=lenv)
261
313
  self.sh.spawn(iocmd, output=False)
262
314
  return dbpath
263
315
 
264
316
  def _ioassign_process(self, dbpaths, wmode):
265
- with open('IOASSIGN', wmode) as fhgather:
317
+ with open("IOASSIGN", wmode) as fhgather:
266
318
  for dbpath in dbpaths:
267
- with open(self.sh.path.join(dbpath, 'IOASSIGN')) as fhlay:
319
+ with open(self.sh.path.join(dbpath, "IOASSIGN")) as fhlay:
268
320
  for line in fhlay:
269
321
  fhgather.write(line)
270
322
 
271
323
  def ioassign_gather(self, *dbpaths):
272
324
  """Gather IOASSIGN data from **dbpaths** databases and create a global IOASSIGN file."""
273
- logger.info('ODB: creating a global IOASSIGN file from: %s',
274
- ','.join([self.sh.path.basename(db) for db in dbpaths]))
275
- self._ioassign_process(dbpaths, 'w')
325
+ logger.info(
326
+ "ODB: creating a global IOASSIGN file from: %s",
327
+ ",".join([self.sh.path.basename(db) for db in dbpaths]),
328
+ )
329
+ self._ioassign_process(dbpaths, "w")
276
330
 
277
331
  def ioassign_append(self, *dbpaths):
278
332
  """Append IOASSIGN data from **dbpaths** databases into the global IOASSIGN file."""
279
- logger.info('ODB: extending the IOASSIGN file with: %s',
280
- ','.join([self.sh.path.basename(db) for db in dbpaths]))
281
- self._ioassign_process(dbpaths, 'a')
333
+ logger.info(
334
+ "ODB: extending the IOASSIGN file with: %s",
335
+ ",".join([self.sh.path.basename(db) for db in dbpaths]),
336
+ )
337
+ self._ioassign_process(dbpaths, "a")
282
338
 
283
339
  def shuffle_setup(self, slots, mergedirect=False, ccmadirect=False):
284
340
  """Setup environment variables to control ODB shuffle behaviour.
@@ -286,8 +342,11 @@ class OdbDriver:
286
342
  :param bool mergedirect: Run the shuffle procedure on the input database.
287
343
  :param bool ccmadirect: Create a CCMA database at the end of the run.
288
344
  """
289
- logger.info('ODB: shuffle_setup: mergedirect=%s, ccmadirect=%s',
290
- str(mergedirect), str(ccmadirect))
345
+ logger.info(
346
+ "ODB: shuffle_setup: mergedirect=%s, ccmadirect=%s",
347
+ str(mergedirect),
348
+ str(ccmadirect),
349
+ )
291
350
  if mergedirect or ccmadirect:
292
351
  self.env.update(
293
352
  ODB_CCMA_TSLOTS=slots.nslot,
@@ -304,38 +363,63 @@ class OdbDriver:
304
363
  def create_poolmask(self, layout, dbpath=None):
305
364
  """Request the poolmask file creation."""
306
365
  layout, dbpath, _ = self._process_layout_dbpath(layout, dbpath)
307
- logger.info('ODB: requesting poolmask file for: %s (layout=%s).', dbpath, layout)
366
+ logger.info(
367
+ "ODB: requesting poolmask file for: %s (layout=%s).",
368
+ dbpath,
369
+ layout,
370
+ )
308
371
  self.env.update(
309
372
  ODB_CCMA_CREATE_POOLMASK=1,
310
- ODB_CCMA_POOLMASK_FILE=self.sh.path.join(dbpath, layout + '.poolmask'),
373
+ ODB_CCMA_POOLMASK_FILE=self.sh.path.join(
374
+ dbpath, layout + ".poolmask"
375
+ ),
311
376
  )
312
377
 
313
378
  def change_layout(self, layout, layout_new, dbpath=None):
314
379
  """Make the appropriate renaming of files in ECMA to CCMA."""
315
380
  layout, dbpath, _ = self._process_layout_dbpath(layout, dbpath)
316
381
  layout_new = layout_new.upper()
317
- logger.info('ODB: changing layout (%s -> %s) for %s.', layout, layout_new, dbpath)
382
+ logger.info(
383
+ "ODB: changing layout (%s -> %s) for %s.",
384
+ layout,
385
+ layout_new,
386
+ dbpath,
387
+ )
318
388
  to_cleanup = set()
319
389
  for f in self.sh.ls(dbpath):
320
390
  if self.sh.path.islink(self.sh.path.join(dbpath, f)):
321
391
  fullpath = self.sh.path.join(dbpath, f)
322
392
  target = self.sh.readlink(fullpath)
323
393
  self.sh.unlink(fullpath)
324
- self.sh.symlink(target.replace(layout, layout_new),
325
- fullpath.replace(layout, layout_new))
394
+ self.sh.symlink(
395
+ target.replace(layout, layout_new),
396
+ fullpath.replace(layout, layout_new),
397
+ )
326
398
  continue
327
- if f in [n.format(layout) for n in ('{:s}.dd', '{:s}.flags')]:
328
- self.sh.mv(self.sh.path.join(dbpath, f),
329
- self.sh.path.join(dbpath, f.replace(layout, layout_new)))
330
- if f in [n.format(layout) for n in ('{:s}.iomap', '{:s}.sch',
331
- '{:s}.IOASSIGN', 'IOASSIGN.{:s}', 'IOASSIGN')]:
332
- tmp_target = self.sh.path.join(dbpath, f + '.tmp_new')
399
+ if f in [n.format(layout) for n in ("{:s}.dd", "{:s}.flags")]:
400
+ self.sh.mv(
401
+ self.sh.path.join(dbpath, f),
402
+ self.sh.path.join(dbpath, f.replace(layout, layout_new)),
403
+ )
404
+ if f in [
405
+ n.format(layout)
406
+ for n in (
407
+ "{:s}.iomap",
408
+ "{:s}.sch",
409
+ "{:s}.IOASSIGN",
410
+ "IOASSIGN.{:s}",
411
+ "IOASSIGN",
412
+ )
413
+ ]:
414
+ tmp_target = self.sh.path.join(dbpath, f + ".tmp_new")
333
415
  with open(self.sh.path.join(dbpath, f)) as inodb:
334
- with open(tmp_target, 'w') as outodb:
416
+ with open(tmp_target, "w") as outodb:
335
417
  for line in inodb:
336
418
  outodb.write(line.replace(layout, layout_new))
337
- self.sh.mv(tmp_target,
338
- self.sh.path.join(dbpath, f.replace(layout, layout_new)))
419
+ self.sh.mv(
420
+ tmp_target,
421
+ self.sh.path.join(dbpath, f.replace(layout, layout_new)),
422
+ )
339
423
  if layout in f:
340
424
  to_cleanup.add(self.sh.path.join(dbpath, f))
341
425
  for f_name in to_cleanup:
@@ -347,44 +431,45 @@ odbmix_attributes = footprints.Footprint(
347
431
  info="Abstract ODB footprints' attributes.",
348
432
  attr=dict(
349
433
  npool=dict(
350
- info='The number of pool(s) in the ODB database.',
434
+ info="The number of pool(s) in the ODB database.",
351
435
  type=int,
352
436
  optional=True,
353
437
  default=1,
354
438
  ),
355
439
  iomethod=dict(
356
- info='The io_method of the ODB database.',
440
+ info="The io_method of the ODB database.",
357
441
  type=int,
358
442
  optional=True,
359
443
  default=1,
360
444
  doc_zorder=-50,
361
445
  ),
362
446
  slots=dict(
363
- info='The timeslots of the assimilation window.',
447
+ info="The timeslots of the assimilation window.",
364
448
  type=TimeSlots,
365
449
  optional=True,
366
- default=TimeSlots(7, chunk='PT1H'),
450
+ default=TimeSlots(7, chunk="PT1H"),
367
451
  ),
368
452
  virtualdb=dict(
369
- info='The type of the virtual ODB database.',
453
+ info="The type of the virtual ODB database.",
370
454
  optional=True,
371
- default='ecma',
372
- access='rwx',
455
+ default="ecma",
456
+ access="rwx",
373
457
  doc_visibility=footprints.doc.visibility.ADVANCED,
374
458
  ),
375
459
  date=dict(
376
- info='The current run date.',
460
+ info="The current run date.",
377
461
  optional=True,
378
- access='rwx',
462
+ access="rwx",
379
463
  type=bdate.Date,
380
464
  doc_zorder=-50,
381
465
  ),
382
466
  ioassign=dict(
383
- info='The path to the ioassign binary (needed for merge/create actions',
467
+ info="The path to the ioassign binary (needed for merge/create actions",
384
468
  optional=True,
385
- default='ioassign.x',
386
- )
387
- ))
469
+ default="ioassign.x",
470
+ ),
471
+ ),
472
+ )
388
473
 
389
474
 
390
475
  @algo_component_deco_mixin_autodoc
@@ -402,12 +487,14 @@ class OdbComponentDecoMixin(AlgoComponentDecoMixin):
402
487
  manually if needed.
403
488
  """
404
489
 
405
- _MIXIN_EXTRA_FOOTPRINTS = [odbmix_attributes, ]
490
+ _MIXIN_EXTRA_FOOTPRINTS = [
491
+ odbmix_attributes,
492
+ ]
406
493
 
407
494
  def _odbobj_init(self, rh, opts): # @UnusedVariable
408
495
  """Setup the OdbDriver object."""
409
- cycle = ArpIfsSimplifiedCycle('cy01')
410
- if rh and hasattr(rh.resource, 'cycle'):
496
+ cycle = ArpIfsSimplifiedCycle("cy01")
497
+ if rh and hasattr(rh.resource, "cycle"):
411
498
  cycle = rh.resource.cycle
412
499
  self._odb = OdbDriver(
413
500
  cycle=cycle,
@@ -436,20 +523,23 @@ class OdbComponentDecoMixin(AlgoComponentDecoMixin):
436
523
  @property
437
524
  def odb(self):
438
525
  """Access to a properly initialised :class:`OdbDriver` object."""
439
- if not hasattr(self, '_odb'):
440
- raise RuntimeError('Uninitialised *odb* object.')
526
+ if not hasattr(self, "_odb"):
527
+ raise RuntimeError("Uninitialised *odb* object.")
441
528
  return self._odb
442
529
 
443
530
  def lookupodb(self, fatal=True):
444
531
  """Return a list of effective input resources which are odb observations."""
445
532
  allodb = [
446
- x for x in self.context.sequence.effective_inputs(kind='observations')
447
- if x.rh.container.actualfmt == 'odb'
533
+ x
534
+ for x in self.context.sequence.effective_inputs(
535
+ kind="observations"
536
+ )
537
+ if x.rh.container.actualfmt == "odb"
448
538
  ]
449
539
  allodb.sort(key=lambda s: s.rh.resource.part)
450
540
  if not allodb and fatal:
451
- logger.critical('Missing ODB input data for %s', self.fullname())
452
- raise ValueError('Missing ODB input data')
541
+ logger.critical("Missing ODB input data for %s", self.fullname())
542
+ raise ValueError("Missing ODB input data")
453
543
  return allodb
454
544
 
455
545
  def odb_date_and_layout_from_sections(self, odbsections):
@@ -465,23 +555,32 @@ class OdbComponentDecoMixin(AlgoComponentDecoMixin):
465
555
  raise AlgoComponentError("Inconsistent ODB dates")
466
556
  self.virtualdb = alllayouts.pop()
467
557
  self.date = alldates.pop()
468
- logger.info('ODB: Detected from ODB database(s). self.date=%s, self.virtualdb=%s.',
469
- self.date.stdvortex, self.virtualdb)
558
+ logger.info(
559
+ "ODB: Detected from ODB database(s). self.date=%s, self.virtualdb=%s.",
560
+ self.date.stdvortex,
561
+ self.virtualdb,
562
+ )
470
563
 
471
564
  def _odb_find_ioassign_script(self, purpose):
472
565
  """Look for ioassign script of *purpose" attribute, return path."""
473
- scripts = [x.rh.container.abspath
474
- for x in self.context.sequence.effective_inputs(kind='ioassign_script')
475
- if x.rh.resource.purpose == purpose]
566
+ scripts = [
567
+ x.rh.container.abspath
568
+ for x in self.context.sequence.effective_inputs(
569
+ kind="ioassign_script"
570
+ )
571
+ if x.rh.resource.purpose == purpose
572
+ ]
476
573
  if len(scripts) > 1:
477
- raise AlgoComponentError("More than one purpose={} ioassign_script found in resources.")
574
+ raise AlgoComponentError(
575
+ "More than one purpose={} ioassign_script found in resources."
576
+ )
478
577
  elif len(scripts) == 1:
479
578
  self.system.xperm(scripts[0], force=True)
480
579
  return scripts[0]
481
580
  else:
482
581
  return None
483
582
 
484
- def odb_merge_if_needed(self, odbsections, subdir='.'):
583
+ def odb_merge_if_needed(self, odbsections, subdir="."):
485
584
  """
486
585
  If multiple ODB databases are listed in the **odsections** section list,
487
586
  start an ODB merge.
@@ -489,10 +588,10 @@ class OdbComponentDecoMixin(AlgoComponentDecoMixin):
489
588
  :return: The path to the ODB database (the virtual database if a merge is
490
589
  performed).
491
590
  """
492
- if len(odbsections) > 1 or self.virtualdb.lower() == 'ecma':
493
- logger.info('ODB: merge for: %s.', self.virtualdb)
494
- iomerge_p = self._odb_find_ioassign_script('merge')
495
- iocreate_p = self._odb_find_ioassign_script('create')
591
+ if len(odbsections) > 1 or self.virtualdb.lower() == "ecma":
592
+ logger.info("ODB: merge for: %s.", self.virtualdb)
593
+ iomerge_p = self._odb_find_ioassign_script("merge")
594
+ iocreate_p = self._odb_find_ioassign_script("create")
496
595
  with self.system.cdcontext(subdir):
497
596
  virtualdb_path = self.odb.ioassign_merge(
498
597
  layout=self.virtualdb,
@@ -502,7 +601,9 @@ class OdbComponentDecoMixin(AlgoComponentDecoMixin):
502
601
  iocreate_path=iocreate_p,
503
602
  )
504
603
  else:
505
- virtualdb_path = self.system.path.abspath(odbsections[0].rh.container.localpath())
604
+ virtualdb_path = self.system.path.abspath(
605
+ odbsections[0].rh.container.localpath()
606
+ )
506
607
  return virtualdb_path
507
608
 
508
609
  def odb_create_db(self, layout, dbpath=None):
@@ -517,28 +618,36 @@ class OdbComponentDecoMixin(AlgoComponentDecoMixin):
517
618
  npool=self.npool,
518
619
  ioassign=self._x_ioassign,
519
620
  dbpath=dbpath,
520
- iocreate_path=self._odb_find_ioassign_script('create')
621
+ iocreate_path=self._odb_find_ioassign_script("create"),
521
622
  )
522
- logger.info('ODB: database created: %s (layout=%s).', dbout, layout)
623
+ logger.info("ODB: database created: %s (layout=%s).", dbout, layout)
523
624
  return dbout
524
625
 
525
626
  def odb_handle_raw_dbs(self):
526
627
  """Look for extras ODB raw databases and fix the environment accordingly."""
527
628
  odbraw = [
528
- x.rh for x in self.context.sequence.effective_inputs(kind='odbraw')
529
- if x.rh.container.actualfmt == 'odb'
629
+ x.rh
630
+ for x in self.context.sequence.effective_inputs(kind="odbraw")
631
+ if x.rh.container.actualfmt == "odb"
530
632
  ]
531
633
  if not odbraw:
532
- logger.error('No ODB raw databases found.')
634
+ logger.error("No ODB raw databases found.")
533
635
  else:
534
636
  for rraw in odbraw:
535
637
  rawpath = rraw.container.localpath()
536
638
  self.odb.fix_db_path(rraw.resource.layout, rawpath)
537
- for badlink in [bl
538
- for bl in self.system.glob(self.system.path.join(rawpath, '*.h'))
539
- if self.system.path.islink(bl) and not self.system.path.exists(bl)]:
639
+ for badlink in [
640
+ bl
641
+ for bl in self.system.glob(
642
+ self.system.path.join(rawpath, "*.h")
643
+ )
644
+ if self.system.path.islink(bl)
645
+ and not self.system.path.exists(bl)
646
+ ]:
540
647
  self.system.unlink(badlink)
541
- self.odb.ioassign_append(* [rraw.container.localpath() for rraw in odbraw])
648
+ self.odb.ioassign_append(
649
+ *[rraw.container.localpath() for rraw in odbraw]
650
+ )
542
651
  return odbraw
543
652
 
544
653
  def odb_rw_or_overwrite_method(self, *dbsections):