vortex-nwp 2.0.0b1__py3-none-any.whl → 2.0.0b2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (139) hide show
  1. vortex/__init__.py +59 -45
  2. vortex/algo/__init__.py +3 -2
  3. vortex/algo/components.py +940 -614
  4. vortex/algo/mpitools.py +802 -497
  5. vortex/algo/serversynctools.py +34 -33
  6. vortex/config.py +19 -22
  7. vortex/data/__init__.py +9 -3
  8. vortex/data/abstractstores.py +593 -655
  9. vortex/data/containers.py +217 -162
  10. vortex/data/contents.py +65 -39
  11. vortex/data/executables.py +93 -102
  12. vortex/data/flow.py +40 -34
  13. vortex/data/geometries.py +228 -132
  14. vortex/data/handlers.py +428 -225
  15. vortex/data/outflow.py +15 -15
  16. vortex/data/providers.py +185 -163
  17. vortex/data/resources.py +48 -42
  18. vortex/data/stores.py +544 -413
  19. vortex/gloves.py +114 -87
  20. vortex/layout/__init__.py +1 -8
  21. vortex/layout/contexts.py +150 -84
  22. vortex/layout/dataflow.py +353 -202
  23. vortex/layout/monitor.py +264 -128
  24. vortex/nwp/__init__.py +5 -2
  25. vortex/nwp/algo/__init__.py +14 -5
  26. vortex/nwp/algo/assim.py +205 -151
  27. vortex/nwp/algo/clim.py +683 -517
  28. vortex/nwp/algo/coupling.py +447 -225
  29. vortex/nwp/algo/eda.py +437 -229
  30. vortex/nwp/algo/eps.py +403 -231
  31. vortex/nwp/algo/forecasts.py +420 -271
  32. vortex/nwp/algo/fpserver.py +683 -307
  33. vortex/nwp/algo/ifsnaming.py +205 -145
  34. vortex/nwp/algo/ifsroot.py +210 -122
  35. vortex/nwp/algo/monitoring.py +132 -76
  36. vortex/nwp/algo/mpitools.py +321 -191
  37. vortex/nwp/algo/odbtools.py +617 -353
  38. vortex/nwp/algo/oopsroot.py +449 -273
  39. vortex/nwp/algo/oopstests.py +90 -56
  40. vortex/nwp/algo/request.py +287 -206
  41. vortex/nwp/algo/stdpost.py +878 -522
  42. vortex/nwp/data/__init__.py +22 -4
  43. vortex/nwp/data/assim.py +125 -137
  44. vortex/nwp/data/boundaries.py +121 -68
  45. vortex/nwp/data/climfiles.py +193 -211
  46. vortex/nwp/data/configfiles.py +73 -69
  47. vortex/nwp/data/consts.py +426 -401
  48. vortex/nwp/data/ctpini.py +59 -43
  49. vortex/nwp/data/diagnostics.py +94 -66
  50. vortex/nwp/data/eda.py +50 -51
  51. vortex/nwp/data/eps.py +195 -146
  52. vortex/nwp/data/executables.py +440 -434
  53. vortex/nwp/data/fields.py +63 -48
  54. vortex/nwp/data/gridfiles.py +183 -111
  55. vortex/nwp/data/logs.py +250 -217
  56. vortex/nwp/data/modelstates.py +180 -151
  57. vortex/nwp/data/monitoring.py +72 -99
  58. vortex/nwp/data/namelists.py +254 -202
  59. vortex/nwp/data/obs.py +400 -308
  60. vortex/nwp/data/oopsexec.py +22 -20
  61. vortex/nwp/data/providers.py +90 -65
  62. vortex/nwp/data/query.py +71 -82
  63. vortex/nwp/data/stores.py +49 -36
  64. vortex/nwp/data/surfex.py +136 -137
  65. vortex/nwp/syntax/__init__.py +1 -1
  66. vortex/nwp/syntax/stdattrs.py +173 -111
  67. vortex/nwp/tools/__init__.py +2 -2
  68. vortex/nwp/tools/addons.py +22 -17
  69. vortex/nwp/tools/agt.py +24 -12
  70. vortex/nwp/tools/bdap.py +16 -5
  71. vortex/nwp/tools/bdcp.py +4 -1
  72. vortex/nwp/tools/bdm.py +3 -0
  73. vortex/nwp/tools/bdmp.py +14 -9
  74. vortex/nwp/tools/conftools.py +728 -378
  75. vortex/nwp/tools/drhook.py +12 -8
  76. vortex/nwp/tools/grib.py +65 -39
  77. vortex/nwp/tools/gribdiff.py +22 -17
  78. vortex/nwp/tools/ifstools.py +82 -42
  79. vortex/nwp/tools/igastuff.py +167 -143
  80. vortex/nwp/tools/mars.py +14 -2
  81. vortex/nwp/tools/odb.py +234 -125
  82. vortex/nwp/tools/partitioning.py +61 -37
  83. vortex/nwp/tools/satrad.py +27 -12
  84. vortex/nwp/util/async.py +83 -55
  85. vortex/nwp/util/beacon.py +10 -10
  86. vortex/nwp/util/diffpygram.py +174 -86
  87. vortex/nwp/util/ens.py +144 -63
  88. vortex/nwp/util/hooks.py +30 -19
  89. vortex/nwp/util/taskdeco.py +28 -24
  90. vortex/nwp/util/usepygram.py +278 -172
  91. vortex/nwp/util/usetnt.py +31 -17
  92. vortex/sessions.py +72 -39
  93. vortex/syntax/__init__.py +1 -1
  94. vortex/syntax/stdattrs.py +410 -171
  95. vortex/syntax/stddeco.py +31 -22
  96. vortex/toolbox.py +327 -192
  97. vortex/tools/__init__.py +11 -2
  98. vortex/tools/actions.py +125 -59
  99. vortex/tools/addons.py +111 -92
  100. vortex/tools/arm.py +42 -22
  101. vortex/tools/compression.py +72 -69
  102. vortex/tools/date.py +11 -4
  103. vortex/tools/delayedactions.py +242 -132
  104. vortex/tools/env.py +75 -47
  105. vortex/tools/folder.py +342 -171
  106. vortex/tools/grib.py +311 -149
  107. vortex/tools/lfi.py +423 -216
  108. vortex/tools/listings.py +109 -40
  109. vortex/tools/names.py +218 -156
  110. vortex/tools/net.py +632 -298
  111. vortex/tools/parallelism.py +93 -61
  112. vortex/tools/prestaging.py +55 -31
  113. vortex/tools/schedulers.py +172 -105
  114. vortex/tools/services.py +402 -333
  115. vortex/tools/storage.py +293 -358
  116. vortex/tools/surfex.py +24 -24
  117. vortex/tools/systems.py +1211 -631
  118. vortex/tools/targets.py +156 -100
  119. vortex/util/__init__.py +1 -1
  120. vortex/util/config.py +377 -327
  121. vortex/util/empty.py +2 -2
  122. vortex/util/helpers.py +56 -24
  123. vortex/util/introspection.py +18 -12
  124. vortex/util/iosponge.py +8 -4
  125. vortex/util/roles.py +4 -6
  126. vortex/util/storefunctions.py +39 -13
  127. vortex/util/structs.py +3 -3
  128. vortex/util/worker.py +29 -17
  129. vortex_nwp-2.0.0b2.dist-info/METADATA +66 -0
  130. vortex_nwp-2.0.0b2.dist-info/RECORD +142 -0
  131. {vortex_nwp-2.0.0b1.dist-info → vortex_nwp-2.0.0b2.dist-info}/WHEEL +1 -1
  132. vortex/layout/appconf.py +0 -109
  133. vortex/layout/jobs.py +0 -1276
  134. vortex/layout/nodes.py +0 -1424
  135. vortex/layout/subjobs.py +0 -464
  136. vortex_nwp-2.0.0b1.dist-info/METADATA +0 -50
  137. vortex_nwp-2.0.0b1.dist-info/RECORD +0 -146
  138. {vortex_nwp-2.0.0b1.dist-info → vortex_nwp-2.0.0b2.dist-info}/LICENSE +0 -0
  139. {vortex_nwp-2.0.0b1.dist-info → vortex_nwp-2.0.0b2.dist-info}/top_level.txt +0 -0
vortex/data/stores.py CHANGED
@@ -8,6 +8,7 @@ Store objects use the :mod:`footprints` mechanism.
8
8
  import copy
9
9
  import ftplib
10
10
  import io
11
+ import os
11
12
  import re
12
13
 
13
14
  from bronx.fancies import loggers
@@ -15,12 +16,15 @@ import footprints
15
16
 
16
17
  from vortex import sessions
17
18
  from vortex import config
18
- from vortex.data.abstractstores import Store, ArchiveStore, ConfigurableArchiveStore, CacheStore
19
+ from vortex.data.abstractstores import (
20
+ Store,
21
+ ArchiveStore,
22
+ CacheStore,
23
+ )
19
24
  from vortex.data.abstractstores import MultiStore, PromiseStore
20
25
  from vortex.data.abstractstores import ARCHIVE_GET_INTENT_DEFAULT
21
26
  from vortex.layout import dataflow
22
27
  from vortex.syntax.stdattrs import hashalgo_avail_list
23
- from vortex.syntax.stdattrs import FreeXPid
24
28
  from vortex.syntax.stdattrs import DelayedEnvValue
25
29
  from vortex.tools.systems import ExecutionError
26
30
 
@@ -30,24 +34,35 @@ __all__ = []
30
34
  logger = loggers.getLogger(__name__)
31
35
 
32
36
 
37
+ def get_cache_location():
38
+ try:
39
+ cacheloc = config.from_config(
40
+ section="data-tree",
41
+ key="rootdir",
42
+ )
43
+ except config.ConfigurationError:
44
+ cacheloc = os.path.join(os.environ["HOME"], ".vortex.d")
45
+ return cacheloc
46
+
47
+
33
48
  class MagicPlace(Store):
34
49
  """Somewhere, over the rainbow!"""
35
50
 
36
51
  _footprint = dict(
37
- info = 'Evanescent physical store',
38
- attr = dict(
39
- scheme = dict(
40
- values = ['magic'],
52
+ info="Evanescent physical store",
53
+ attr=dict(
54
+ scheme=dict(
55
+ values=["magic"],
41
56
  ),
42
57
  ),
43
- priority = dict(
44
- level = footprints.priorities.top.DEFAULT # @UndefinedVariable
45
- )
58
+ priority=dict(
59
+ level=footprints.priorities.top.DEFAULT # @UndefinedVariable
60
+ ),
46
61
  )
47
62
 
48
63
  @property
49
64
  def realkind(self):
50
- return 'magicstore'
65
+ return "magicstore"
51
66
 
52
67
  def has_fast_check(self):
53
68
  """A void check is very fast !"""
@@ -59,7 +74,7 @@ class MagicPlace(Store):
59
74
 
60
75
  def magiclocate(self, remote, options):
61
76
  """Void - Empty string returned."""
62
- return ''
77
+ return ""
63
78
 
64
79
  def magicget(self, remote, local, options):
65
80
  """Void - Always True."""
@@ -108,23 +123,23 @@ class FunctionStore(Store):
108
123
  """
109
124
 
110
125
  _footprint = dict(
111
- info = 'Dummy store that calls a function',
112
- attr = dict(
113
- scheme = dict(
114
- values = ['function'],
126
+ info="Dummy store that calls a function",
127
+ attr=dict(
128
+ scheme=dict(
129
+ values=["function"],
130
+ ),
131
+ netloc=dict(
132
+ values=[""],
115
133
  ),
116
- netloc = dict(
117
- values = [''],
118
- )
119
134
  ),
120
- priority = dict(
121
- level = footprints.priorities.top.DEFAULT # @UndefinedVariable
122
- )
135
+ priority=dict(
136
+ level=footprints.priorities.top.DEFAULT # @UndefinedVariable
137
+ ),
123
138
  )
124
139
 
125
140
  @property
126
141
  def realkind(self):
127
- return 'functionstore'
142
+ return "functionstore"
128
143
 
129
144
  def has_fast_check(self):
130
145
  """A void check is very fast !"""
@@ -136,20 +151,21 @@ class FunctionStore(Store):
136
151
 
137
152
  def functionlocate(self, remote, options):
138
153
  """The name of the function that will be called."""
139
- cleanname = remote['path'][1:]
140
- if cleanname.endswith('/'):
154
+ cleanname = remote["path"][1:]
155
+ if cleanname.endswith("/"):
141
156
  cleanname = cleanname[:-1]
142
157
  return cleanname
143
158
 
144
159
  def functionget(self, remote, local, options):
145
160
  """Calls the appropriate function and writes the result."""
146
161
  # Find the appropriate function
147
- cbfunc = self.system.import_function(self.functionlocate(remote,
148
- options))
162
+ cbfunc = self.system.import_function(
163
+ self.functionlocate(remote, options)
164
+ )
149
165
  # ... and call it
150
166
  opts = dict()
151
167
  opts.update(options)
152
- opts.update(remote['query'])
168
+ opts.update(remote["query"])
153
169
  try:
154
170
  fres = cbfunc(opts)
155
171
  except FunctionStoreCallbackError as e:
@@ -157,15 +173,15 @@ class FunctionStore(Store):
157
173
  logger.error("Here is the exception: %s", str(e))
158
174
  fres = None
159
175
  if fres is not None:
160
- if 'intent' in options and options['intent'] == dataflow.intent.IN:
161
- logger.info('Ignore intent <in> for function input.')
176
+ if "intent" in options and options["intent"] == dataflow.intent.IN:
177
+ logger.info("Ignore intent <in> for function input.")
162
178
  # Handle StringIO objects, by changing them to ByteIOs...
163
179
  if isinstance(fres, io.StringIO):
164
180
  s_fres = fres
165
181
  s_fres.seek(0)
166
182
  fres = io.BytesIO()
167
183
  for l in s_fres:
168
- fres.write(l.encode(encoding='utf-8'))
184
+ fres.write(l.encode(encoding="utf-8"))
169
185
  fres.seek(0)
170
186
  # NB: fres should be a file like object (BytesIO will do the trick)
171
187
  return self.system.cp(fres, local)
@@ -187,30 +203,30 @@ class Finder(Store):
187
203
  """The most usual store: your current filesystem!"""
188
204
 
189
205
  _footprint = dict(
190
- info = 'Miscellaneous file access',
191
- attr = dict(
192
- scheme = dict(
193
- values = ['file', 'ftp', 'symlink', 'rcp', 'scp'],
206
+ info="Miscellaneous file access",
207
+ attr=dict(
208
+ scheme=dict(
209
+ values=["file", "ftp", "symlink", "rcp", "scp"],
194
210
  ),
195
- netloc = dict(
196
- outcast = ['oper.inline.fr'],
211
+ netloc=dict(
212
+ outcast=["oper.inline.fr"],
197
213
  ),
198
- storehash = dict(
199
- values = hashalgo_avail_list,
214
+ storehash=dict(
215
+ values=hashalgo_avail_list,
200
216
  ),
201
217
  ),
202
- priority = dict(
203
- level = footprints.priorities.top.DEFAULT # @UndefinedVariable
204
- )
218
+ priority=dict(
219
+ level=footprints.priorities.top.DEFAULT # @UndefinedVariable
220
+ ),
205
221
  )
206
222
 
207
223
  def __init__(self, *args, **kw):
208
- logger.debug('Finder store init %s', self.__class__)
224
+ logger.debug("Finder store init %s", self.__class__)
209
225
  super().__init__(*args, **kw)
210
226
 
211
227
  @property
212
228
  def realkind(self):
213
- return 'finder'
229
+ return "finder"
214
230
 
215
231
  def hostname(self):
216
232
  """Returns the current :attr:`netloc`."""
@@ -218,21 +234,28 @@ class Finder(Store):
218
234
 
219
235
  def fullpath(self, remote):
220
236
  """Return actual path unless explicitly defined as relative path."""
221
- if remote['query'].get('relative', False):
222
- return remote['path'].lstrip('/')
237
+ if remote["query"].get("relative", False):
238
+ return remote["path"].lstrip("/")
223
239
  else:
224
- return remote['path']
240
+ return remote["path"]
225
241
 
226
242
  def _localtarfix(self, local):
227
- if (isinstance(local, str) and self.system.path.isfile(local) and
228
- self.system.is_tarfile(local)):
229
- destdir = self.system.path.dirname(self.system.path.realpath(local))
243
+ if (
244
+ isinstance(local, str)
245
+ and self.system.path.isfile(local)
246
+ and self.system.is_tarfile(local)
247
+ ):
248
+ destdir = self.system.path.dirname(
249
+ self.system.path.realpath(local)
250
+ )
230
251
  try:
231
252
  self.system.smartuntar(local, destdir)
232
253
  except ExecutionError:
233
254
  if not self.system.is_tarname(local):
234
- logger.warning("An automatic untar was attempted but it failed. " +
235
- "Maybe the system's is_tarfile got it wrong ?")
255
+ logger.warning(
256
+ "An automatic untar was attempted but it failed. "
257
+ + "Maybe the system's is_tarfile got it wrong ?"
258
+ )
236
259
  else:
237
260
  raise
238
261
 
@@ -251,10 +274,12 @@ class Finder(Store):
251
274
  def fileget(self, remote, local, options):
252
275
  """Delegates to ``system`` the copy of ``remote`` to ``local``."""
253
276
  rpath = self.fullpath(remote)
254
- logger.info('fileget on %s (to: %s)', rpath, local)
255
- if 'intent' in options and options['intent'] == dataflow.intent.IN:
256
- logger.info('Ignore intent <in> for remote input %s', rpath)
257
- rc = self.system.cp(rpath, local, fmt=options.get('fmt'), intent=dataflow.intent.INOUT)
277
+ logger.info("fileget on %s (to: %s)", rpath, local)
278
+ if "intent" in options and options["intent"] == dataflow.intent.IN:
279
+ logger.info("Ignore intent <in> for remote input %s", rpath)
280
+ rc = self.system.cp(
281
+ rpath, local, fmt=options.get("fmt"), intent=dataflow.intent.INOUT
282
+ )
258
283
  rc = rc and self._hash_get_check(self.fileget, remote, local, options)
259
284
  if rc:
260
285
  self._localtarfix(local)
@@ -263,8 +288,8 @@ class Finder(Store):
263
288
  def fileput(self, local, remote, options):
264
289
  """Delegates to ``system`` the copy of ``local`` to ``remote``."""
265
290
  rpath = self.fullpath(remote)
266
- logger.info('fileput to %s (from: %s)', rpath, local)
267
- rc = self.system.cp(local, rpath, fmt=options.get('fmt'))
291
+ logger.info("fileput to %s (from: %s)", rpath, local)
292
+ rc = self.system.cp(local, rpath, fmt=options.get("fmt"))
268
293
  return rc and self._hash_put(self.fileput, local, remote, options)
269
294
 
270
295
  def filedelete(self, remote, options):
@@ -272,10 +297,13 @@ class Finder(Store):
272
297
  rc = None
273
298
  if self.filecheck(remote, options):
274
299
  rpath = self.fullpath(remote)
275
- logger.info('filedelete on %s', rpath)
276
- rc = self.system.remove(rpath, fmt=options.get('fmt'))
300
+ logger.info("filedelete on %s", rpath)
301
+ rc = self.system.remove(rpath, fmt=options.get("fmt"))
277
302
  else:
278
- logger.error('Try to remove a non-existing resource <%s>', self.fullpath(remote))
303
+ logger.error(
304
+ "Try to remove a non-existing resource <%s>",
305
+ self.fullpath(remote),
306
+ )
279
307
  return rc
280
308
 
281
309
  symlinkcheck = filecheck
@@ -283,34 +311,40 @@ class Finder(Store):
283
311
 
284
312
  def symlinkget(self, remote, local, options):
285
313
  rpath = self.fullpath(remote)
286
- if 'intent' in options and options['intent'] == dataflow.intent.INOUT:
287
- logger.error('It is unsafe to have a symlink with intent=inout: %s', rpath)
314
+ if "intent" in options and options["intent"] == dataflow.intent.INOUT:
315
+ logger.error(
316
+ "It is unsafe to have a symlink with intent=inout: %s", rpath
317
+ )
288
318
  return False
289
319
  rc = self.system.remove(local)
290
320
  self.system.symlink(rpath, local)
291
321
  return rc and self.system.path.exists(local)
292
322
 
293
323
  def symlinkput(self, local, remote, options):
294
- logger.error("The Finder store with scheme:symlink is not able to perform Puts.")
324
+ logger.error(
325
+ "The Finder store with scheme:symlink is not able to perform Puts."
326
+ )
295
327
  return False
296
328
 
297
329
  def symlinkdelete(self, remote, options):
298
- logger.error("The Finder store with scheme:symlink is not able to perform Deletes.")
330
+ logger.error(
331
+ "The Finder store with scheme:symlink is not able to perform Deletes."
332
+ )
299
333
  return False
300
334
 
301
335
  def _ftpinfos(self, remote, **kwargs):
302
336
  args = kwargs.copy()
303
- args['hostname'] = self.hostname()
304
- args['logname'] = remote['username']
337
+ args["hostname"] = self.hostname()
338
+ args["logname"] = remote["username"]
305
339
  port = self.hostname().netport
306
340
  if port is not None:
307
- args['port'] = port
341
+ args["port"] = port
308
342
  return args
309
343
 
310
344
  def ftpcheck(self, remote, options):
311
345
  """Delegates to ``system.ftp`` a distant check."""
312
346
  rc = None
313
- ftp = self.system.ftp(** self._ftpinfos(remote))
347
+ ftp = self.system.ftp(**self._ftpinfos(remote))
314
348
  if ftp:
315
349
  try:
316
350
  rc = ftp.size(self.fullpath(remote))
@@ -324,7 +358,7 @@ class Finder(Store):
324
358
 
325
359
  def ftplocate(self, remote, options):
326
360
  """Delegates to ``system`` qualified name creation."""
327
- ftp = self.system.ftp(** self._ftpinfos(remote, delayed=True))
361
+ ftp = self.system.ftp(**self._ftpinfos(remote, delayed=True))
328
362
  if ftp:
329
363
  rloc = ftp.netpath(self.fullpath(remote))
330
364
  ftp.close()
@@ -335,13 +369,15 @@ class Finder(Store):
335
369
  def ftpget(self, remote, local, options):
336
370
  """Delegates to ``system`` the file transfer of ``remote`` to ``local``."""
337
371
  rpath = self.fullpath(remote)
338
- logger.info('ftpget on ftp://%s/%s (to: %s)', self.hostname(), rpath, local)
372
+ logger.info(
373
+ "ftpget on ftp://%s/%s (to: %s)", self.hostname(), rpath, local
374
+ )
339
375
  rc = self.system.smartftget(
340
376
  rpath,
341
377
  local,
342
- fmt=options.get('fmt'),
378
+ fmt=options.get("fmt"),
343
379
  # ftp control
344
- ** self._ftpinfos(remote)
380
+ **self._ftpinfos(remote),
345
381
  )
346
382
  rc = rc and self._hash_get_check(self.ftpget, remote, local, options)
347
383
  if rc:
@@ -352,14 +388,16 @@ class Finder(Store):
352
388
  """Delegates to ``system`` the file transfer of ``local`` to ``remote``."""
353
389
  rpath = self.fullpath(remote)
354
390
  put_opts = dict()
355
- put_opts['fmt'] = options.get('fmt')
356
- put_opts['sync'] = options.get('enforcesync', False)
357
- logger.info('ftpput to ftp://%s/%s (from: %s)', self.hostname(), rpath, local)
391
+ put_opts["fmt"] = options.get("fmt")
392
+ put_opts["sync"] = options.get("enforcesync", False)
393
+ logger.info(
394
+ "ftpput to ftp://%s/%s (from: %s)", self.hostname(), rpath, local
395
+ )
358
396
  rc = self.system.smartftput(
359
397
  local,
360
398
  rpath,
361
399
  # ftp control
362
- ** self._ftpinfos(remote, ** put_opts)
400
+ **self._ftpinfos(remote, **put_opts),
363
401
  )
364
402
  return rc and self._hash_put(self.ftpput, local, remote, options)
365
403
 
@@ -368,7 +406,9 @@ class Finder(Store):
368
406
  rc = None
369
407
  actualpath = self.fullpath(remote)
370
408
  if self.ftpcheck(remote, options):
371
- logger.info('ftpdelete on ftp://%s/%s', self.hostname(), actualpath)
409
+ logger.info(
410
+ "ftpdelete on ftp://%s/%s", self.hostname(), actualpath
411
+ )
372
412
  ftp = self.system.ftp(**self._ftpinfos(remote))
373
413
  if ftp:
374
414
  try:
@@ -376,14 +416,16 @@ class Finder(Store):
376
416
  finally:
377
417
  ftp.close()
378
418
  else:
379
- logger.error('Try to remove a non-existing resource <%s>', actualpath)
419
+ logger.error(
420
+ "Try to remove a non-existing resource <%s>", actualpath
421
+ )
380
422
  return rc
381
423
 
382
424
 
383
425
  class _VortexStackedStorageMixin:
384
426
  """Mixin class that adds utility functions to work with stacked data."""
385
427
 
386
- _STACKED_RE = re.compile('stacked-')
428
+ _STACKED_RE = re.compile("stacked-")
387
429
 
388
430
  @property
389
431
  def stackedstore(self):
@@ -391,25 +433,31 @@ class _VortexStackedStorageMixin:
391
433
  return self._STACKED_RE.search(self.netloc)
392
434
 
393
435
  def _stacked_remainder(self, remote, stackpath):
394
- path_remainder = remote['path'].strip('/').split('/')
395
- for a_spath in stackpath.split('/'):
436
+ path_remainder = remote["path"].strip("/").split("/")
437
+ for a_spath in stackpath.split("/"):
396
438
  if path_remainder and path_remainder[0] == a_spath:
397
439
  del path_remainder[0]
398
440
  else:
399
441
  break
400
- return '/'.join(path_remainder)
442
+ return "/".join(path_remainder)
401
443
 
402
444
  def _stacked_xremote(self, remote):
403
445
  """The path to **remote** with its stack."""
404
446
  if self.stackedstore:
405
447
  remote = remote.copy()
406
- remote['query'] = remote['query'].copy()
407
- stackpath = remote['query'].pop('stackpath', (None, ))[0]
408
- stackfmt = remote['query'].pop('stackfmt', (None, ))[0]
448
+ remote["query"] = remote["query"].copy()
449
+ stackpath = remote["query"].pop("stackpath", (None,))[0]
450
+ stackfmt = remote["query"].pop("stackfmt", (None,))[0]
409
451
  if stackpath is None or stackfmt is None:
410
- raise ValueError('"stackpath" and "stackfmt" are not available in the query.')
452
+ raise ValueError(
453
+ '"stackpath" and "stackfmt" are not available in the query.'
454
+ )
411
455
  else:
412
- remote['path'] = stackpath + '/' + self._stacked_remainder(remote, stackpath)
456
+ remote["path"] = (
457
+ stackpath
458
+ + "/"
459
+ + self._stacked_remainder(remote, stackpath)
460
+ )
413
461
  return remote
414
462
 
415
463
  def _stacked_xegglocate(self, remote):
@@ -423,14 +471,16 @@ class _VortexStackedStorageMixin:
423
471
 
424
472
  """
425
473
  remote = remote.copy()
426
- remote['query'] = remote['query'].copy()
427
- stackpath = remote['query'].pop('stackpath', (None, ))[0].strip('/')
428
- stackfmt = remote['query'].pop('stackfmt', (None, ))[0]
474
+ remote["query"] = remote["query"].copy()
475
+ stackpath = remote["query"].pop("stackpath", (None,))[0].strip("/")
476
+ stackfmt = remote["query"].pop("stackfmt", (None,))[0]
429
477
  if stackpath is None or stackfmt is None:
430
- raise ValueError('"stackpath" and "stackfmt" are not available in the query.')
478
+ raise ValueError(
479
+ '"stackpath" and "stackfmt" are not available in the query.'
480
+ )
431
481
  else:
432
482
  resource_remainder = self._stacked_remainder(remote, stackpath)
433
- remote['path'] = '/' + stackpath
483
+ remote["path"] = "/" + stackpath
434
484
  return remote, stackfmt, resource_remainder
435
485
 
436
486
 
@@ -438,11 +488,13 @@ _vortex_readonly_store = footprints.Footprint(
438
488
  info="Abstract store' readonly=True attribute",
439
489
  attr=dict(
440
490
  readonly=dict(
441
- values=[True, ],
491
+ values=[
492
+ True,
493
+ ],
442
494
  optional=True,
443
- default=True
495
+ default=True,
444
496
  )
445
- )
497
+ ),
446
498
  )
447
499
 
448
500
 
@@ -451,37 +503,38 @@ class _VortexBaseArchiveStore(ArchiveStore, _VortexStackedStorageMixin):
451
503
 
452
504
  _abstract = True
453
505
  _footprint = dict(
454
- info = 'VORTEX archive access',
455
- attr = dict(
456
- scheme = dict(
457
- values = ['vortex'],
458
- ),
459
- netloc = dict(
506
+ info="VORTEX archive access",
507
+ attr=dict(
508
+ scheme=dict(
509
+ values=["vortex"],
460
510
  ),
461
- storehead = dict(
462
- optional = True,
463
- default = 'vortex',
464
- outcast = ['xp'],
511
+ netloc=dict(),
512
+ storehead=dict(
513
+ optional=True,
514
+ default="vortex",
515
+ outcast=["xp"],
465
516
  ),
466
- )
517
+ ),
467
518
  )
468
519
 
469
- _STACKS_AUTOREFILL_CRIT = 'stacked-archive-smart'
520
+ _STACKS_AUTOREFILL_CRIT = "stacked-archive-smart"
470
521
 
471
522
  def __init__(self, *args, **kw):
472
- logger.debug('Vortex archive store init %s', self.__class__)
523
+ logger.debug("Vortex archive store init %s", self.__class__)
473
524
  super().__init__(*args, **kw)
474
525
 
475
526
  def remap_read(self, remote, options):
476
527
  """Remap actual remote path to distant store path for intrusive actions."""
477
- return copy.copy(remote)
528
+ raise NotImplementedError
478
529
 
479
530
  def remap_list(self, remote, options):
480
531
  """Reformulates the remote path to compatible vortex namespace."""
481
- if len(remote['path'].split('/')) >= 4:
532
+ if len(remote["path"].split("/")) >= 4:
482
533
  return self.remap_read(remote, options)
483
534
  else:
484
- logger.critical('The << %s >> path is not listable.', remote['path'])
535
+ logger.critical(
536
+ "The << %s >> path is not listable.", remote["path"]
537
+ )
485
538
  return None
486
539
 
487
540
  remap_write = remap_read
@@ -490,7 +543,7 @@ class _VortexBaseArchiveStore(ArchiveStore, _VortexStackedStorageMixin):
490
543
  def stacks_autorefill(self):
491
544
  """Where to refill a stack retrieved from the archive."""
492
545
  if self._STACKS_AUTOREFILL_CRIT in self.netloc:
493
- return self.netloc.replace(self._STACKS_AUTOREFILL_CRIT, 'cache')
546
+ return self.netloc.replace(self._STACKS_AUTOREFILL_CRIT, "cache")
494
547
  else:
495
548
  return None
496
549
 
@@ -500,24 +553,34 @@ class _VortexBaseArchiveStore(ArchiveStore, _VortexStackedStorageMixin):
500
553
  rundir = sessions.current().context.rundir
501
554
  if not rundir:
502
555
  rundir = self.system.pwd()
503
- rundir = self.system.path.join(rundir, 'vortex_stacks_xeggs')
504
- target = self.system.path.join(rundir, * remote['path'].strip('/').split('/'))
556
+ rundir = self.system.path.join(rundir, "vortex_stacks_xeggs")
557
+ target = self.system.path.join(
558
+ rundir, *remote["path"].strip("/").split("/")
559
+ )
505
560
  targetopts = dict(fmt=remotefmt, intent=dataflow.intent.IN)
506
561
  if self.system.path.exists(target):
507
- logger.info("Stack previously retrieved (in %s). Using it.", target)
562
+ logger.info(
563
+ "Stack previously retrieved (in %s). Using it.", target
564
+ )
508
565
  rc = True
509
566
  else:
510
567
  if result_id:
511
- rc = self._vortexfinaliseget(result_id, remote, target, targetopts)
568
+ rc = self._vortexfinaliseget(
569
+ result_id, remote, target, targetopts
570
+ )
512
571
  else:
513
572
  rc = self._vortexget(remote, target, targetopts)
514
573
  if rc and self.stacks_autorefill:
515
- rstore = footprints.proxy.store(scheme=self.scheme, netloc=self.stacks_autorefill)
574
+ rstore = footprints.proxy.store(
575
+ scheme=self.scheme, netloc=self.stacks_autorefill
576
+ )
516
577
  logger.info("Refilling the stack egg to [%s]", rstore)
517
578
  try:
518
579
  rstore.put(target, remote.copy(), targetopts)
519
580
  except (ExecutionError, OSError) as e:
520
- logger.error("An ExecutionError happened during the refill: %s", str(e))
581
+ logger.error(
582
+ "An ExecutionError happened during the refill: %s", str(e)
583
+ )
521
584
  logger.error("This error is ignored... but that's ugly !")
522
585
  return rc, target, remainder
523
586
 
@@ -526,11 +589,15 @@ class _VortexBaseArchiveStore(ArchiveStore, _VortexStackedStorageMixin):
526
589
  if self.stackedstore:
527
590
  s_remote, s_remotefmt, _ = self._stacked_xegglocate(remote)
528
591
  options = options.copy()
529
- options['fmt'] = s_remotefmt
592
+ options["fmt"] = s_remotefmt
530
593
  rc = self._vortexcheck(s_remote, options)
531
594
  if rc:
532
- rc, target, remainder = self._vortex_stacked_egg_retrieve(remote)
533
- rc = rc and self.system.path.exists(self.system.path.join(target, remainder))
595
+ rc, target, remainder = self._vortex_stacked_egg_retrieve(
596
+ remote
597
+ )
598
+ rc = rc and self.system.path.exists(
599
+ self.system.path.join(target, remainder)
600
+ )
534
601
  return rc
535
602
  else:
536
603
  return self._vortexcheck(remote, options)
@@ -545,7 +612,7 @@ class _VortexBaseArchiveStore(ArchiveStore, _VortexStackedStorageMixin):
545
612
  if self.stackedstore:
546
613
  remote, s_remotefmt, _ = self._stacked_xegglocate(remote)
547
614
  options = options.copy()
548
- options['fmt'] = s_remotefmt
615
+ options["fmt"] = s_remotefmt
549
616
  return self._vortexlocate(remote, options)
550
617
 
551
618
  def _vortexlocate(self, remote, options):
@@ -573,7 +640,7 @@ class _VortexBaseArchiveStore(ArchiveStore, _VortexStackedStorageMixin):
573
640
  if self.stackedstore:
574
641
  remote, s_remotefmt, _ = self._stacked_xegglocate(remote)
575
642
  options = options.copy()
576
- options['fmt'] = s_remotefmt
643
+ options["fmt"] = s_remotefmt
577
644
  return self._vortexprestageinfo(remote, options)
578
645
 
579
646
  def _vortexprestageinfo(self, remote, options):
@@ -585,10 +652,12 @@ class _VortexBaseArchiveStore(ArchiveStore, _VortexStackedStorageMixin):
585
652
  """Vortex' archive get sequence."""
586
653
  if self.stackedstore:
587
654
  rc, target, remainder = self._vortex_stacked_egg_retrieve(remote)
588
- rc = rc and self.system.cp(self.system.path.join(target, remainder), local,
589
- fmt=options.get('fmt'),
590
- intent=options.get('intent',
591
- ARCHIVE_GET_INTENT_DEFAULT))
655
+ rc = rc and self.system.cp(
656
+ self.system.path.join(target, remainder),
657
+ local,
658
+ fmt=options.get("fmt"),
659
+ intent=options.get("intent", ARCHIVE_GET_INTENT_DEFAULT),
660
+ )
592
661
  return rc
593
662
  else:
594
663
  return self._vortexget(remote, local, options)
@@ -603,7 +672,7 @@ class _VortexBaseArchiveStore(ArchiveStore, _VortexStackedStorageMixin):
603
672
  if self.stackedstore:
604
673
  s_remote, s_remotefmt, _ = self._stacked_xegglocate(remote)
605
674
  targetopts = dict(fmt=s_remotefmt, intent=dataflow.intent.IN)
606
- return self._vortexearlyget(s_remote, 'somelocalfile', targetopts)
675
+ return self._vortexearlyget(s_remote, "somelocalfile", targetopts)
607
676
  else:
608
677
  return self._vortexearlyget(remote, local, options)
609
678
 
@@ -615,11 +684,15 @@ class _VortexBaseArchiveStore(ArchiveStore, _VortexStackedStorageMixin):
615
684
  def vortexfinaliseget(self, result_id, remote, local, options):
616
685
  """Vortex' archive finaliseget sequence."""
617
686
  if self.stackedstore:
618
- rc, target, remainder = self._vortex_stacked_egg_retrieve(remote, result_id=result_id)
619
- rc = rc and self.system.cp(self.system.path.join(target, remainder), local,
620
- fmt=options.get('fmt'),
621
- intent=options.get('intent',
622
- ARCHIVE_GET_INTENT_DEFAULT))
687
+ rc, target, remainder = self._vortex_stacked_egg_retrieve(
688
+ remote, result_id=result_id
689
+ )
690
+ rc = rc and self.system.cp(
691
+ self.system.path.join(target, remainder),
692
+ local,
693
+ fmt=options.get("fmt"),
694
+ intent=options.get("intent", ARCHIVE_GET_INTENT_DEFAULT),
695
+ )
623
696
  return rc
624
697
  else:
625
698
  return self._vortexfinaliseget(result_id, remote, local, options)
@@ -655,30 +728,31 @@ class VortexStdBaseArchiveStore(_VortexBaseArchiveStore):
655
728
  """
656
729
 
657
730
  _footprint = dict(
658
- info = 'VORTEX archive access for casual experiments',
659
- attr = dict(
660
- netloc = dict(
661
- values = ['vortex.archive-legacy.fr'],
731
+ info="VORTEX archive access for casual experiments",
732
+ attr=dict(
733
+ netloc=dict(
734
+ values=["vortex.archive-legacy.fr"],
662
735
  ),
663
- )
736
+ ),
664
737
  )
665
738
 
666
- @property
667
- def _actual_mappingroot(self):
668
- """Read the get entry point form configuration."""
669
- return config.from_config(
670
- section="storage", key="rootdir",
671
- )
672
-
673
739
  def remap_read(self, remote, options):
674
740
  """Reformulates the remote path to compatible vortex namespace."""
675
741
  remote = copy.copy(remote)
676
- xpath = remote['path'].split('/')
677
- actual_mappingroot = self._actual_mappingroot
678
- if not self.storeroot and actual_mappingroot:
679
- remote['root'] = actual_mappingroot
680
- xpath[3:4] = list(xpath[3])
681
- remote['path'] = self.system.path.join(*xpath)
742
+ try:
743
+ remote["root"] = config.from_config(
744
+ section="storage",
745
+ key="rootdir",
746
+ )
747
+ except config.ConfigurationError as e:
748
+ msg = (
749
+ "Trying to write to archive but location is not configured.\n"
750
+ 'Make sure key "rootdir" is defined in storage section of '
751
+ "the configuration.\n"
752
+ "See https://vortex-nwp.readthedocs.io/en/latest/user-guide/configuration.html#storage"
753
+ )
754
+ logger.error(msg)
755
+ raise e
682
756
  return remote
683
757
 
684
758
 
@@ -693,70 +767,18 @@ class VortexStdStackedArchiveStore(VortexStdBaseArchiveStore):
693
767
  _footprint = [
694
768
  _vortex_readonly_store,
695
769
  dict(
696
- attr = dict(
697
- netloc = dict(
698
- values = ['vortex.stacked-archive-legacy.fr',
699
- 'vortex.stacked-archive-smart.fr'],
770
+ attr=dict(
771
+ netloc=dict(
772
+ values=[
773
+ "vortex.stacked-archive-legacy.fr",
774
+ "vortex.stacked-archive-smart.fr",
775
+ ],
700
776
  ),
701
777
  )
702
- )
778
+ ),
703
779
  ]
704
780
 
705
781
 
706
- class VortexFreeStdBaseArchiveStore(_VortexBaseArchiveStore, ConfigurableArchiveStore):
707
- """Archive for casual VORTEX experiments: Support for Free XPIDs.
708
-
709
- This 'archive-legacy' store looks into the resource 'main' location not
710
- into a potential stack.
711
- """
712
-
713
- #: Path to the vortex-free Store configuration file
714
- _store_global_config = '@store-vortex-free.ini'
715
- _datastore_id = 'store-vortex-free-conf'
716
-
717
- _footprint = dict(
718
- info = 'VORTEX archive access for casual experiments',
719
- attr = dict(
720
- netloc = dict(
721
- values = ['vortex-free.archive-legacy.fr'],
722
- ),
723
- )
724
- )
725
-
726
- def remap_read(self, remote, options):
727
- """Reformulates the remote path to compatible vortex namespace."""
728
- remote = copy.copy(remote)
729
- xpath = remote['path'].strip('/').split('/')
730
- f_xpid = FreeXPid(xpath[2])
731
- xpath[2] = f_xpid.id
732
- if 'root' not in remote:
733
- remote['root'] = self._actual_storeroot(f_xpid)
734
- remote['path'] = self.system.path.join(*xpath)
735
- return remote
736
-
737
- remap_write = remap_read
738
-
739
-
740
- class VortexFreeStdStackedArchiveStore(VortexFreeStdBaseArchiveStore):
741
- """Archive for casual VORTEX experiments: Support for Free XPIDs.
742
-
743
- This 'stacked-archive-legacy' or 'stacked-archive-smart' store looks into
744
- the stack associated to the resource. The '-smart' variant, has the ability
745
- to refill the whole stack into local cache (to be faster in the future).
746
- """
747
-
748
- _footprint = [
749
- _vortex_readonly_store,
750
- dict(
751
- attr = dict(
752
- netloc = dict(
753
- values = ['vortex-free.stacked-archive-legacy.fr',
754
- 'vortex-free.stacked-archive-smart.fr'],
755
- ),
756
- )
757
- )]
758
-
759
-
760
782
  class VortexOpBaseArchiveStore(_VortexBaseArchiveStore):
761
783
  """Archive for op VORTEX experiments.
762
784
 
@@ -765,39 +787,43 @@ class VortexOpBaseArchiveStore(_VortexBaseArchiveStore):
765
787
  """
766
788
 
767
789
  _footprint = dict(
768
- info = 'VORTEX archive access for op experiments',
769
- attr = dict(
770
- netloc = dict(
771
- values = ['vsop.archive-legacy.fr'],
790
+ info="VORTEX archive access for op experiments",
791
+ attr=dict(
792
+ netloc=dict(
793
+ values=["vsop.archive-legacy.fr"],
772
794
  ),
773
- storetrue = dict(
774
- default = DelayedEnvValue('op_archive', True),
795
+ storetrue=dict(
796
+ default=DelayedEnvValue("op_archive", True),
775
797
  ),
776
- )
798
+ ),
777
799
  )
778
800
 
779
- @property
780
- def _actual_storeroot(self):
781
- return (
782
- self.storeroot or
783
- config.from_config(
784
- section="storage", key="op_rootdir",
785
- )
786
- )
787
-
788
801
  def remap_read(self, remote, options):
789
802
  """Reformulates the remote path to compatible vortex namespace."""
790
803
  remote = copy.copy(remote)
791
- xpath = remote['path'].split('/')
792
- remote['root'] = self._actual_storeroot
793
- if len(xpath) >= 5 and re.match(r'^\d{8}T\d{2,4}', xpath[4]):
804
+ try:
805
+ remote["root"] = config.from_config(
806
+ section="storage",
807
+ key="op_rootdir",
808
+ )
809
+ except config.ConfigurationError as e:
810
+ msg = (
811
+ "Trying to write to operational data archive but location"
812
+ 'is not configured.\nMake sure key "rootdir" is defined in '
813
+ "the storage section of the configuration.\n"
814
+ "See https://vortex-nwp.readthedocs.io/en/latest/user-guide/configuration.html#storage"
815
+ )
816
+ logger.error(msg)
817
+ raise e
818
+ xpath = remote["path"].split("/")
819
+ if len(xpath) >= 5 and re.match(r"^\d{8}T\d{2,4}", xpath[4]):
794
820
  # If a date is detected
795
821
  vxdate = list(xpath[4])
796
- vxdate.insert(4, '/')
797
- vxdate.insert(7, '/')
798
- vxdate.insert(10, '/')
799
- xpath[4] = ''.join(vxdate)
800
- remote['path'] = self.system.path.join(*xpath)
822
+ vxdate.insert(4, "/")
823
+ vxdate.insert(7, "/")
824
+ vxdate.insert(10, "/")
825
+ xpath[4] = "".join(vxdate)
826
+ remote["path"] = self.system.path.join(*xpath)
801
827
  return remote
802
828
 
803
829
  remap_write = remap_read
@@ -814,13 +840,16 @@ class VortexOpStackedArchiveStore(VortexOpBaseArchiveStore):
814
840
  _footprint = [
815
841
  _vortex_readonly_store,
816
842
  dict(
817
- attr = dict(
818
- netloc = dict(
819
- values = ['vsop.stacked-archive-legacy.fr',
820
- 'vsop.stacked-archive-smart.fr'],
843
+ attr=dict(
844
+ netloc=dict(
845
+ values=[
846
+ "vsop.stacked-archive-legacy.fr",
847
+ "vsop.stacked-archive-smart.fr",
848
+ ],
821
849
  ),
822
850
  )
823
- )]
851
+ ),
852
+ ]
824
853
 
825
854
 
826
855
  class VortexArchiveStore(MultiStore):
@@ -835,42 +864,58 @@ class VortexArchiveStore(MultiStore):
835
864
  """
836
865
 
837
866
  _footprint = dict(
838
- info = 'VORTEX archive access',
839
- attr = dict(
840
- scheme = dict(
841
- values = ['vortex'],
867
+ info="VORTEX archive access",
868
+ attr=dict(
869
+ scheme=dict(
870
+ values=["vortex"],
842
871
  ),
843
- netloc = dict(
844
- values = ['vortex.archive.fr', 'vortex-free.archive.fr', 'vsop.archive.fr'],
872
+ netloc=dict(
873
+ values=[
874
+ "vortex.archive.fr",
875
+ "vortex-free.archive.fr",
876
+ "vsop.archive.fr",
877
+ ],
845
878
  ),
846
- refillstore = dict(
847
- default = False,
879
+ refillstore=dict(
880
+ default=False,
848
881
  ),
849
- storehead = dict(
850
- optional = True,
882
+ storehead=dict(
883
+ optional=True,
851
884
  ),
852
- storesync = dict(
853
- alias = ('archsync', 'synchro'),
854
- type = bool,
855
- optional = True,
885
+ storesync=dict(
886
+ alias=("archsync", "synchro"),
887
+ type=bool,
888
+ optional=True,
856
889
  ),
857
- )
890
+ ),
858
891
  )
859
892
 
860
893
  def filtered_readable_openedstores(self, remote):
861
894
  """Only use the stacked store if sensible."""
862
- ostores = [self.openedstores[0], ]
863
- ostores.extend([sto for sto in self.openedstores[1:]
864
- if not sto.stackedstore or 'stackpath' in remote['query']
865
- ])
895
+ ostores = [
896
+ self.openedstores[0],
897
+ ]
898
+ ostores.extend(
899
+ [
900
+ sto
901
+ for sto in self.openedstores[1:]
902
+ if not sto.stackedstore or "stackpath" in remote["query"]
903
+ ]
904
+ )
866
905
  return ostores
867
906
 
868
907
  def alternates_netloc(self):
869
908
  """Return netlocs describing both base and stacked archives."""
870
- netloc_m = re.match(r'(?P<base>v.*)\.archive\.(?P<country>\w+)', self.netloc)
909
+ netloc_m = re.match(
910
+ r"(?P<base>v.*)\.archive\.(?P<country>\w+)", self.netloc
911
+ )
871
912
  return [
872
- '{base:s}.archive-legacy.{country:s}'.format(** netloc_m.groupdict()),
873
- '{base:s}.stacked-archive-legacy.{country:s}'.format(** netloc_m.groupdict()),
913
+ "{base:s}.archive-legacy.{country:s}".format(
914
+ **netloc_m.groupdict()
915
+ ),
916
+ "{base:s}.stacked-archive-legacy.{country:s}".format(
917
+ **netloc_m.groupdict()
918
+ ),
874
919
  ]
875
920
 
876
921
  def alternates_fpextras(self):
@@ -883,26 +928,28 @@ class _VortexCacheBaseStore(CacheStore, _VortexStackedStorageMixin):
883
928
 
884
929
  _abstract = True
885
930
  _footprint = dict(
886
- info = 'VORTEX cache access',
887
- attr = dict(
888
- scheme = dict(
889
- values = ['vortex'],
931
+ info="VORTEX cache access",
932
+ attr=dict(
933
+ scheme=dict(
934
+ values=["vortex"],
890
935
  ),
891
- headdir = dict(
892
- default = "",
893
- outcast = ['xp', ],
936
+ headdir=dict(
937
+ default="",
938
+ outcast=[
939
+ "xp",
940
+ ],
894
941
  ),
895
- rtouch = dict(
896
- default = True,
942
+ rtouch=dict(
943
+ default=True,
897
944
  ),
898
- rtouchskip = dict(
899
- default = 3,
945
+ rtouchskip=dict(
946
+ default=3,
900
947
  ),
901
- )
948
+ ),
902
949
  )
903
950
 
904
951
  def __init__(self, *args, **kw):
905
- logger.debug('Vortex cache store init %s', self.__class__)
952
+ logger.debug("Vortex cache store init %s", self.__class__)
906
953
  del self.cache
907
954
  super().__init__(*args, **kw)
908
955
 
@@ -939,45 +986,56 @@ class VortexCacheMtStore(_VortexCacheBaseStore):
939
986
  """Some kind of MTOOL cache for VORTEX experiments."""
940
987
 
941
988
  _footprint = dict(
942
- info = 'VORTEX MTOOL like Cache access',
943
- attr = dict(
944
- netloc = dict(
945
- values = ['{:s}.{:s}cache-mt.fr'.format(v, s)
946
- for v in ('vortex', 'vortex-free', 'vsop') for s in ('', 'stacked-')]
947
- ),
948
- strategy = dict(
949
- default = 'mtool',
989
+ info="VORTEX MTOOL like Cache access",
990
+ attr=dict(
991
+ netloc=dict(
992
+ values=[
993
+ "{:s}.{:s}cache-mt.fr".format(v, s)
994
+ for v in ("vortex", "vortex-free", "vsop")
995
+ for s in ("", "stacked-")
996
+ ]
950
997
  ),
951
- )
998
+ ),
952
999
  )
953
1000
 
1001
+ def __init__(self, *args, **kw):
1002
+ super().__init__(*args, **kw)
1003
+ self.location = get_cache_location()
1004
+
954
1005
 
955
- # TODO Not sure this class is needed anymore
956
1006
  class VortexCacheOp2ResearchStore(_VortexCacheBaseStore):
957
1007
  """The DSI/OP VORTEX cache where researchers can get the freshest data."""
958
1008
 
959
1009
  _footprint = dict(
960
- info = 'VORTEX Mtool cache access',
961
- attr = dict(
962
- netloc = dict(
963
- values = [
964
- 'vsop.{:s}cache-op2r.fr'.format(s)
965
- for s in ('', 'stacked-')
1010
+ info="VORTEX Mtool cache access",
1011
+ attr=dict(
1012
+ netloc=dict(
1013
+ values=[
1014
+ "vsop.{:s}cache-op2r.fr".format(s)
1015
+ for s in ("", "stacked-")
966
1016
  ],
967
1017
  ),
968
- strategy = dict(
969
- default = 'op2r',
1018
+ readonly=dict(
1019
+ default=True,
970
1020
  ),
971
- readonly = dict(
972
- default = True,
973
- )
974
- )
1021
+ ),
975
1022
  )
976
1023
 
977
- @property
978
- def underlying_cache_kind(self):
979
- """The kind of cache that will be used."""
980
- return self.strategy
1024
+ def __init__(self, *args, **kw):
1025
+ super().__init__(*args, **kw)
1026
+ try:
1027
+ cachepath = config.from_config(
1028
+ section="data-tree",
1029
+ key="op_rootdir",
1030
+ )
1031
+ except config.ConfigurationError as e:
1032
+ logger.error(
1033
+ "Cannot use special experiment cache without providing",
1034
+ "cache location",
1035
+ )
1036
+ raise e
1037
+
1038
+ self.location = os.path.join(cachepath, "vortex")
981
1039
 
982
1040
 
983
1041
  class _AbstractVortexCacheMultiStore(MultiStore):
@@ -985,33 +1043,49 @@ class _AbstractVortexCacheMultiStore(MultiStore):
985
1043
 
986
1044
  _abstract = True
987
1045
  _footprint = dict(
988
- info = 'VORTEX cache access',
989
- attr = dict(
990
- scheme = dict(
991
- values = ['vortex'],
1046
+ info="VORTEX cache access",
1047
+ attr=dict(
1048
+ scheme=dict(
1049
+ values=["vortex"],
992
1050
  ),
993
- refillstore = dict(
994
- default = False,
995
- )
996
- )
1051
+ refillstore=dict(
1052
+ default=False,
1053
+ ),
1054
+ ),
997
1055
  )
998
1056
 
999
1057
  def filtered_readable_openedstores(self, remote):
1000
1058
  """Deals with stacked stores that are not always active."""
1001
- ostores = [self.openedstores[0], ]
1059
+ ostores = [
1060
+ self.openedstores[0],
1061
+ ]
1002
1062
  # TODO is the call to cache.allow_reads still required without
1003
1063
  # marketplace stores?
1004
- ostores.extend([sto for sto in self.openedstores[1:]
1005
- if ((not sto.stackedstore or 'stackpath' in remote['query']) and
1006
- sto.cache.allow_reads(remote['path']))
1007
- ])
1064
+ ostores.extend(
1065
+ [
1066
+ sto
1067
+ for sto in self.openedstores[1:]
1068
+ if (
1069
+ (not sto.stackedstore or "stackpath" in remote["query"])
1070
+ and sto.cache.allow_reads(remote["path"])
1071
+ )
1072
+ ]
1073
+ )
1008
1074
  return ostores
1009
1075
 
1010
1076
  def filtered_writeable_openedstores(self, remote):
1011
1077
  """never writes into stack stores."""
1012
- ostores = [self.openedstores[0], ]
1013
- ostores.extend([sto for sto in self.openedstores[1:]
1014
- if not sto.stackedstore and sto.cache.allow_writes(remote['path'])])
1078
+ ostores = [
1079
+ self.openedstores[0],
1080
+ ]
1081
+ ostores.extend(
1082
+ [
1083
+ sto
1084
+ for sto in self.openedstores[1:]
1085
+ if not sto.stackedstore
1086
+ and sto.cache.allow_writes(remote["path"])
1087
+ ]
1088
+ )
1015
1089
  return ostores
1016
1090
 
1017
1091
 
@@ -1019,18 +1093,27 @@ class VortexCacheStore(_AbstractVortexCacheMultiStore):
1019
1093
  """The go to store for data cached by VORTEX R&D experiments."""
1020
1094
 
1021
1095
  _footprint = dict(
1022
- attr = dict(
1023
- netloc = dict(
1024
- values = ['vortex.cache.fr', 'vortex-free.cache.fr', ],
1096
+ attr=dict(
1097
+ netloc=dict(
1098
+ values=[
1099
+ "vortex.cache.fr",
1100
+ "vortex-free.cache.fr",
1101
+ ],
1025
1102
  ),
1026
1103
  )
1027
1104
  )
1028
1105
 
1029
1106
  def alternates_netloc(self):
1030
1107
  """For Non-Op users, Op caches may be accessed in read-only mode."""
1031
- netloc_m = re.match(r'(?P<base>vortex.*)\.cache\.(?P<country>\w+)', self.netloc)
1032
- mt_netloc = '{base:s}.cache-mt.{country:s}'.format(** netloc_m.groupdict())
1033
- s_mt_netloc = '{base:s}.stacked-cache-mt.{country:s}'.format(** netloc_m.groupdict())
1108
+ netloc_m = re.match(
1109
+ r"(?P<base>vortex.*)\.cache\.(?P<country>\w+)", self.netloc
1110
+ )
1111
+ mt_netloc = "{base:s}.cache-mt.{country:s}".format(
1112
+ **netloc_m.groupdict()
1113
+ )
1114
+ s_mt_netloc = "{base:s}.stacked-cache-mt.{country:s}".format(
1115
+ **netloc_m.groupdict()
1116
+ )
1034
1117
  return [mt_netloc, s_mt_netloc]
1035
1118
 
1036
1119
 
@@ -1042,35 +1125,36 @@ class VortexVsopCacheStore(_AbstractVortexCacheMultiStore):
1042
1125
  """
1043
1126
 
1044
1127
  _footprint = dict(
1045
- info = 'VORTEX vsop magic cache access',
1046
- attr = dict(
1047
- netloc = dict(
1048
- values = ['vsop.cache.fr', ],
1128
+ info="VORTEX vsop magic cache access",
1129
+ attr=dict(
1130
+ netloc=dict(
1131
+ values=[
1132
+ "vsop.cache.fr",
1133
+ ],
1049
1134
  ),
1050
- glovekind = dict(
1051
- optional = True,
1052
- default = '[glove::realkind]',
1135
+ glovekind=dict(
1136
+ optional=True,
1137
+ default="[glove::realkind]",
1053
1138
  ),
1054
- )
1139
+ ),
1055
1140
  )
1056
1141
 
1057
1142
  def alternates_netloc(self):
1058
1143
  """For Non-Op users, Op caches may be accessed in read-only mode."""
1059
1144
  todo = [
1060
- 'vsop.cache-mt.fr',
1061
- 'vsop.stacked-cache-mt.fr',
1145
+ "vsop.cache-mt.fr",
1146
+ "vsop.stacked-cache-mt.fr",
1062
1147
  ]
1063
1148
 
1064
1149
  # Only set up op2r cache if the associated filepath
1065
1150
  # is configured
1066
- if (
1067
- (self.glovekind != 'opuser') and
1068
- config.is_defined(
1069
- section="data-tree", key="op_rootdir",
1070
- )
1151
+ if (self.glovekind != "opuser") and config.is_defined(
1152
+ section="data-tree",
1153
+ key="op_rootdir",
1071
1154
  ):
1072
1155
  todo += [
1073
- 'vsop.cache-op2r.fr', 'vsop.stacked-cache-op2r.fr',
1156
+ "vsop.cache-op2r.fr",
1157
+ "vsop.stacked-cache-op2r.fr",
1074
1158
  ]
1075
1159
  return todo
1076
1160
 
@@ -1080,30 +1164,44 @@ class _AbstractVortexStackMultiStore(MultiStore):
1080
1164
 
1081
1165
  _abstract = True
1082
1166
  _footprint = dict(
1083
- info = 'VORTEX stack access',
1084
- attr = dict(
1085
- scheme = dict(
1086
- values = ['vortex'],
1167
+ info="VORTEX stack access",
1168
+ attr=dict(
1169
+ scheme=dict(
1170
+ values=["vortex"],
1087
1171
  ),
1088
- refillstore = dict(
1089
- default = False,
1090
- )
1091
- )
1172
+ refillstore=dict(
1173
+ default=False,
1174
+ ),
1175
+ ),
1092
1176
  )
1093
1177
 
1094
1178
  # TODO is this still needed without marketplace stores?
1095
1179
  def filtered_readable_openedstores(self, remote):
1096
1180
  """Deals with marketplace stores that are not always active."""
1097
- ostores = [self.openedstores[0], ]
1098
- ostores.extend([sto for sto in self.openedstores[1:]
1099
- if sto.cache.allow_reads(remote['path'])])
1181
+ ostores = [
1182
+ self.openedstores[0],
1183
+ ]
1184
+ ostores.extend(
1185
+ [
1186
+ sto
1187
+ for sto in self.openedstores[1:]
1188
+ if sto.cache.allow_reads(remote["path"])
1189
+ ]
1190
+ )
1100
1191
  return ostores
1101
1192
 
1102
1193
  def filtered_writeable_openedstores(self, remote):
1103
1194
  """Deals with marketplace stores that are not always active."""
1104
- ostores = [self.openedstores[0], ]
1105
- ostores.extend([sto for sto in self.openedstores[1:]
1106
- if sto.cache.allow_writes(remote['path'])])
1195
+ ostores = [
1196
+ self.openedstores[0],
1197
+ ]
1198
+ ostores.extend(
1199
+ [
1200
+ sto
1201
+ for sto in self.openedstores[1:]
1202
+ if sto.cache.allow_writes(remote["path"])
1203
+ ]
1204
+ )
1107
1205
  return ostores
1108
1206
 
1109
1207
 
@@ -1111,18 +1209,22 @@ class VortexStackStore(_AbstractVortexStackMultiStore):
1111
1209
  """Store intended to read and write data into VORTEX R&D stacks."""
1112
1210
 
1113
1211
  _footprint = dict(
1114
- info = 'VORTEX stack access',
1115
- attr = dict(
1116
- netloc = dict(
1117
- values = ['vortex.stack.fr', 'vortex-free.stack.fr'],
1212
+ info="VORTEX stack access",
1213
+ attr=dict(
1214
+ netloc=dict(
1215
+ values=["vortex.stack.fr", "vortex-free.stack.fr"],
1118
1216
  ),
1119
- )
1217
+ ),
1120
1218
  )
1121
1219
 
1122
1220
  def alternates_netloc(self):
1123
1221
  """Go through the various stacked stores."""
1124
- netloc_m = re.match(r'(?P<base>vortex.*)\.stack\.(?P<country>\w+)', self.netloc)
1125
- s_mt_netloc = '{base:s}.stacked-cache-mt.{country:s}'.format(** netloc_m.groupdict())
1222
+ netloc_m = re.match(
1223
+ r"(?P<base>vortex.*)\.stack\.(?P<country>\w+)", self.netloc
1224
+ )
1225
+ s_mt_netloc = "{base:s}.stacked-cache-mt.{country:s}".format(
1226
+ **netloc_m.groupdict()
1227
+ )
1126
1228
  return [s_mt_netloc]
1127
1229
 
1128
1230
 
@@ -1130,22 +1232,24 @@ class VortexVsopStackStore(_AbstractVortexStackMultiStore):
1130
1232
  """Store intended to read and write data into VORTEX R&D stacks."""
1131
1233
 
1132
1234
  _footprint = dict(
1133
- info = 'VORTEX stack access',
1134
- attr = dict(
1135
- netloc = dict(
1136
- values = ['vsop.stack.fr'],
1235
+ info="VORTEX stack access",
1236
+ attr=dict(
1237
+ netloc=dict(
1238
+ values=["vsop.stack.fr"],
1137
1239
  ),
1138
- glovekind = dict(
1139
- optional = True,
1140
- default = '[glove::realkind]',
1240
+ glovekind=dict(
1241
+ optional=True,
1242
+ default="[glove::realkind]",
1141
1243
  ),
1142
- )
1244
+ ),
1143
1245
  )
1144
1246
 
1145
1247
  def alternates_netloc(self):
1146
1248
  """For Non-Op users, Op caches may be accessed in read-only mode."""
1147
- todo = ['vsop.stacked-cache-mt.fr', ]
1148
- if self.glovekind != 'opuser':
1249
+ todo = [
1250
+ "vsop.stacked-cache-mt.fr",
1251
+ ]
1252
+ if self.glovekind != "opuser":
1149
1253
  todo.append("vsop.stacked-cache-op2r.fr")
1150
1254
  return todo
1151
1255
 
@@ -1157,24 +1261,30 @@ class VortexStoreLegacy(MultiStore):
1157
1261
  """
1158
1262
 
1159
1263
  _footprint = dict(
1160
- info='VORTEX multi access',
1264
+ info="VORTEX multi access",
1161
1265
  attr=dict(
1162
1266
  scheme=dict(
1163
- values=['vortex'],
1267
+ values=["vortex"],
1164
1268
  ),
1165
1269
  netloc=dict(
1166
- values=['vortex.multi-legacy.fr', 'vortex-free.multi-legacy.fr', 'vsop.multi-legacy.fr'],
1270
+ values=[
1271
+ "vortex.multi-legacy.fr",
1272
+ "vortex-free.multi-legacy.fr",
1273
+ "vsop.multi-legacy.fr",
1274
+ ],
1167
1275
  ),
1168
1276
  refillstore=dict(
1169
1277
  default=True,
1170
- )
1171
- )
1278
+ ),
1279
+ ),
1172
1280
  )
1173
1281
 
1174
1282
  def alternates_netloc(self):
1175
1283
  """Tuple of alternates domains names, e.g. ``cache`` and ``archive``."""
1176
- return [self.netloc.firstname + d for d in ('.cache.fr',
1177
- '.archive-legacy.fr')]
1284
+ return [
1285
+ self.netloc.firstname + d
1286
+ for d in (".cache.fr", ".archive-legacy.fr")
1287
+ ]
1178
1288
 
1179
1289
 
1180
1290
  class VortexStore(MultiStore):
@@ -1184,64 +1294,81 @@ class VortexStore(MultiStore):
1184
1294
  """
1185
1295
 
1186
1296
  _footprint = dict(
1187
- info = 'VORTEX multi access',
1188
- attr = dict(
1189
- scheme = dict(
1190
- values = ['vortex'],
1297
+ info="VORTEX multi access",
1298
+ attr=dict(
1299
+ scheme=dict(
1300
+ values=["vortex"],
1191
1301
  ),
1192
- netloc = dict(
1193
- values = ['vortex.multi.fr', 'vortex-free.multi.fr', 'vsop.multi.fr'],
1302
+ netloc=dict(
1303
+ values=[
1304
+ "vortex.multi.fr",
1305
+ "vortex-free.multi.fr",
1306
+ "vsop.multi.fr",
1307
+ ],
1194
1308
  ),
1195
- refillstore = dict(
1196
- default = False
1197
- )
1198
- )
1309
+ refillstore=dict(default=False),
1310
+ ),
1199
1311
  )
1200
1312
 
1201
1313
  def filtered_readable_openedstores(self, remote):
1202
1314
  """Deals with stacked stores that are not always active."""
1203
- ostores = [self.openedstores[0], ]
1204
- ostores.extend([sto for sto in self.openedstores[1:]
1205
- if not sto.stackedstore or 'stackpath' in remote['query']
1206
- ])
1315
+ ostores = [
1316
+ self.openedstores[0],
1317
+ ]
1318
+ ostores.extend(
1319
+ [
1320
+ sto
1321
+ for sto in self.openedstores[1:]
1322
+ if not sto.stackedstore or "stackpath" in remote["query"]
1323
+ ]
1324
+ )
1207
1325
  return ostores
1208
1326
 
1209
1327
  def alternates_netloc(self):
1210
1328
  """Tuple of alternates domains names, e.g. ``cache`` and ``archive``."""
1211
- return [self.netloc.firstname + d for d in ('.multi-legacy.fr',
1212
- '.stacked-archive-smart.fr',)]
1329
+ return [
1330
+ self.netloc.firstname + d
1331
+ for d in (
1332
+ ".multi-legacy.fr",
1333
+ ".stacked-archive-smart.fr",
1334
+ )
1335
+ ]
1213
1336
 
1214
1337
 
1215
1338
  class PromiseCacheStore(VortexCacheMtStore):
1216
1339
  """Some kind of vortex cache for EXPECTED resources."""
1217
1340
 
1218
1341
  _footprint = dict(
1219
- info = 'EXPECTED cache access',
1220
- attr = dict(
1221
- netloc = dict(
1222
- values = ['promise.cache.fr'],
1342
+ info="EXPECTED cache access",
1343
+ attr=dict(
1344
+ netloc=dict(
1345
+ values=["promise.cache.fr"],
1223
1346
  ),
1224
- headdir = dict(
1225
- default = 'promise',
1226
- outcast = ['xp', 'vortex'],
1347
+ headdir=dict(
1348
+ default="promise",
1349
+ outcast=["xp", "vortex"],
1227
1350
  ),
1228
- )
1351
+ ),
1229
1352
  )
1230
1353
 
1231
1354
  @staticmethod
1232
1355
  def _add_default_options(options):
1233
1356
  options_upd = options.copy()
1234
- options_upd['fmt'] = 'ascii' # Promises are always JSON files
1235
- options_upd['intent'] = 'in' # Promises are always read-only
1357
+ options_upd["fmt"] = "ascii" # Promises are always JSON files
1358
+ options_upd["intent"] = "in" # Promises are always read-only
1236
1359
  return options_upd
1237
1360
 
1238
1361
  def vortexget(self, remote, local, options):
1239
1362
  """Proxy to :meth:`incacheget`."""
1240
- return super().vortexget(remote, local, self._add_default_options(options))
1363
+ return super().vortexget(
1364
+ remote, local, self._add_default_options(options)
1365
+ )
1241
1366
 
1242
1367
  def vortexput(self, local, remote, options):
1243
1368
  """Proxy to :meth:`incacheput`."""
1244
- return super().vortexput(local, remote, self._add_default_options(options))
1369
+ return super().vortexput(
1370
+ local, remote, self._add_default_options(options)
1371
+ )
1245
1372
 
1246
1373
  def vortexdelete(self, remote, options):
1247
1374
  """Proxy to :meth:`incachedelete`."""
@@ -1252,20 +1379,24 @@ class VortexPromiseStore(PromiseStore):
1252
1379
  """Combine a Promise Store for expected resources and any VORTEX Store."""
1253
1380
 
1254
1381
  _footprint = dict(
1255
- info = 'VORTEX promise store',
1256
- attr = dict(
1257
- scheme = dict(
1258
- values = ['xvortex'],
1382
+ info="VORTEX promise store",
1383
+ attr=dict(
1384
+ scheme=dict(
1385
+ values=["xvortex"],
1259
1386
  ),
1260
1387
  netloc=dict(
1261
- outcast = ['vortex-demo.cache.fr', 'vortex-demo.multi.fr',
1262
- 'vortex.testcache.fr', 'vortex.testmulti.fr'],
1388
+ outcast=[
1389
+ "vortex-demo.cache.fr",
1390
+ "vortex-demo.multi.fr",
1391
+ "vortex.testcache.fr",
1392
+ "vortex.testmulti.fr",
1393
+ ],
1263
1394
  ),
1264
- )
1395
+ ),
1265
1396
  )
1266
1397
 
1267
1398
 
1268
1399
  # Activate the footprint's fasttrack on the stores collector
1269
- fcollect = footprints.collectors.get(tag='store')
1270
- fcollect.fasttrack = ('netloc', 'scheme')
1400
+ fcollect = footprints.collectors.get(tag="store")
1401
+ fcollect.fasttrack = ("netloc", "scheme")
1271
1402
  del fcollect