vortex-nwp 2.0.0b1__py3-none-any.whl → 2.0.0b2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- vortex/__init__.py +59 -45
- vortex/algo/__init__.py +3 -2
- vortex/algo/components.py +940 -614
- vortex/algo/mpitools.py +802 -497
- vortex/algo/serversynctools.py +34 -33
- vortex/config.py +19 -22
- vortex/data/__init__.py +9 -3
- vortex/data/abstractstores.py +593 -655
- vortex/data/containers.py +217 -162
- vortex/data/contents.py +65 -39
- vortex/data/executables.py +93 -102
- vortex/data/flow.py +40 -34
- vortex/data/geometries.py +228 -132
- vortex/data/handlers.py +428 -225
- vortex/data/outflow.py +15 -15
- vortex/data/providers.py +185 -163
- vortex/data/resources.py +48 -42
- vortex/data/stores.py +544 -413
- vortex/gloves.py +114 -87
- vortex/layout/__init__.py +1 -8
- vortex/layout/contexts.py +150 -84
- vortex/layout/dataflow.py +353 -202
- vortex/layout/monitor.py +264 -128
- vortex/nwp/__init__.py +5 -2
- vortex/nwp/algo/__init__.py +14 -5
- vortex/nwp/algo/assim.py +205 -151
- vortex/nwp/algo/clim.py +683 -517
- vortex/nwp/algo/coupling.py +447 -225
- vortex/nwp/algo/eda.py +437 -229
- vortex/nwp/algo/eps.py +403 -231
- vortex/nwp/algo/forecasts.py +420 -271
- vortex/nwp/algo/fpserver.py +683 -307
- vortex/nwp/algo/ifsnaming.py +205 -145
- vortex/nwp/algo/ifsroot.py +210 -122
- vortex/nwp/algo/monitoring.py +132 -76
- vortex/nwp/algo/mpitools.py +321 -191
- vortex/nwp/algo/odbtools.py +617 -353
- vortex/nwp/algo/oopsroot.py +449 -273
- vortex/nwp/algo/oopstests.py +90 -56
- vortex/nwp/algo/request.py +287 -206
- vortex/nwp/algo/stdpost.py +878 -522
- vortex/nwp/data/__init__.py +22 -4
- vortex/nwp/data/assim.py +125 -137
- vortex/nwp/data/boundaries.py +121 -68
- vortex/nwp/data/climfiles.py +193 -211
- vortex/nwp/data/configfiles.py +73 -69
- vortex/nwp/data/consts.py +426 -401
- vortex/nwp/data/ctpini.py +59 -43
- vortex/nwp/data/diagnostics.py +94 -66
- vortex/nwp/data/eda.py +50 -51
- vortex/nwp/data/eps.py +195 -146
- vortex/nwp/data/executables.py +440 -434
- vortex/nwp/data/fields.py +63 -48
- vortex/nwp/data/gridfiles.py +183 -111
- vortex/nwp/data/logs.py +250 -217
- vortex/nwp/data/modelstates.py +180 -151
- vortex/nwp/data/monitoring.py +72 -99
- vortex/nwp/data/namelists.py +254 -202
- vortex/nwp/data/obs.py +400 -308
- vortex/nwp/data/oopsexec.py +22 -20
- vortex/nwp/data/providers.py +90 -65
- vortex/nwp/data/query.py +71 -82
- vortex/nwp/data/stores.py +49 -36
- vortex/nwp/data/surfex.py +136 -137
- vortex/nwp/syntax/__init__.py +1 -1
- vortex/nwp/syntax/stdattrs.py +173 -111
- vortex/nwp/tools/__init__.py +2 -2
- vortex/nwp/tools/addons.py +22 -17
- vortex/nwp/tools/agt.py +24 -12
- vortex/nwp/tools/bdap.py +16 -5
- vortex/nwp/tools/bdcp.py +4 -1
- vortex/nwp/tools/bdm.py +3 -0
- vortex/nwp/tools/bdmp.py +14 -9
- vortex/nwp/tools/conftools.py +728 -378
- vortex/nwp/tools/drhook.py +12 -8
- vortex/nwp/tools/grib.py +65 -39
- vortex/nwp/tools/gribdiff.py +22 -17
- vortex/nwp/tools/ifstools.py +82 -42
- vortex/nwp/tools/igastuff.py +167 -143
- vortex/nwp/tools/mars.py +14 -2
- vortex/nwp/tools/odb.py +234 -125
- vortex/nwp/tools/partitioning.py +61 -37
- vortex/nwp/tools/satrad.py +27 -12
- vortex/nwp/util/async.py +83 -55
- vortex/nwp/util/beacon.py +10 -10
- vortex/nwp/util/diffpygram.py +174 -86
- vortex/nwp/util/ens.py +144 -63
- vortex/nwp/util/hooks.py +30 -19
- vortex/nwp/util/taskdeco.py +28 -24
- vortex/nwp/util/usepygram.py +278 -172
- vortex/nwp/util/usetnt.py +31 -17
- vortex/sessions.py +72 -39
- vortex/syntax/__init__.py +1 -1
- vortex/syntax/stdattrs.py +410 -171
- vortex/syntax/stddeco.py +31 -22
- vortex/toolbox.py +327 -192
- vortex/tools/__init__.py +11 -2
- vortex/tools/actions.py +125 -59
- vortex/tools/addons.py +111 -92
- vortex/tools/arm.py +42 -22
- vortex/tools/compression.py +72 -69
- vortex/tools/date.py +11 -4
- vortex/tools/delayedactions.py +242 -132
- vortex/tools/env.py +75 -47
- vortex/tools/folder.py +342 -171
- vortex/tools/grib.py +311 -149
- vortex/tools/lfi.py +423 -216
- vortex/tools/listings.py +109 -40
- vortex/tools/names.py +218 -156
- vortex/tools/net.py +632 -298
- vortex/tools/parallelism.py +93 -61
- vortex/tools/prestaging.py +55 -31
- vortex/tools/schedulers.py +172 -105
- vortex/tools/services.py +402 -333
- vortex/tools/storage.py +293 -358
- vortex/tools/surfex.py +24 -24
- vortex/tools/systems.py +1211 -631
- vortex/tools/targets.py +156 -100
- vortex/util/__init__.py +1 -1
- vortex/util/config.py +377 -327
- vortex/util/empty.py +2 -2
- vortex/util/helpers.py +56 -24
- vortex/util/introspection.py +18 -12
- vortex/util/iosponge.py +8 -4
- vortex/util/roles.py +4 -6
- vortex/util/storefunctions.py +39 -13
- vortex/util/structs.py +3 -3
- vortex/util/worker.py +29 -17
- vortex_nwp-2.0.0b2.dist-info/METADATA +66 -0
- vortex_nwp-2.0.0b2.dist-info/RECORD +142 -0
- {vortex_nwp-2.0.0b1.dist-info → vortex_nwp-2.0.0b2.dist-info}/WHEEL +1 -1
- vortex/layout/appconf.py +0 -109
- vortex/layout/jobs.py +0 -1276
- vortex/layout/nodes.py +0 -1424
- vortex/layout/subjobs.py +0 -464
- vortex_nwp-2.0.0b1.dist-info/METADATA +0 -50
- vortex_nwp-2.0.0b1.dist-info/RECORD +0 -146
- {vortex_nwp-2.0.0b1.dist-info → vortex_nwp-2.0.0b2.dist-info}/LICENSE +0 -0
- {vortex_nwp-2.0.0b1.dist-info → vortex_nwp-2.0.0b2.dist-info}/top_level.txt +0 -0
vortex/data/stores.py
CHANGED
|
@@ -8,6 +8,7 @@ Store objects use the :mod:`footprints` mechanism.
|
|
|
8
8
|
import copy
|
|
9
9
|
import ftplib
|
|
10
10
|
import io
|
|
11
|
+
import os
|
|
11
12
|
import re
|
|
12
13
|
|
|
13
14
|
from bronx.fancies import loggers
|
|
@@ -15,12 +16,15 @@ import footprints
|
|
|
15
16
|
|
|
16
17
|
from vortex import sessions
|
|
17
18
|
from vortex import config
|
|
18
|
-
from vortex.data.abstractstores import
|
|
19
|
+
from vortex.data.abstractstores import (
|
|
20
|
+
Store,
|
|
21
|
+
ArchiveStore,
|
|
22
|
+
CacheStore,
|
|
23
|
+
)
|
|
19
24
|
from vortex.data.abstractstores import MultiStore, PromiseStore
|
|
20
25
|
from vortex.data.abstractstores import ARCHIVE_GET_INTENT_DEFAULT
|
|
21
26
|
from vortex.layout import dataflow
|
|
22
27
|
from vortex.syntax.stdattrs import hashalgo_avail_list
|
|
23
|
-
from vortex.syntax.stdattrs import FreeXPid
|
|
24
28
|
from vortex.syntax.stdattrs import DelayedEnvValue
|
|
25
29
|
from vortex.tools.systems import ExecutionError
|
|
26
30
|
|
|
@@ -30,24 +34,35 @@ __all__ = []
|
|
|
30
34
|
logger = loggers.getLogger(__name__)
|
|
31
35
|
|
|
32
36
|
|
|
37
|
+
def get_cache_location():
|
|
38
|
+
try:
|
|
39
|
+
cacheloc = config.from_config(
|
|
40
|
+
section="data-tree",
|
|
41
|
+
key="rootdir",
|
|
42
|
+
)
|
|
43
|
+
except config.ConfigurationError:
|
|
44
|
+
cacheloc = os.path.join(os.environ["HOME"], ".vortex.d")
|
|
45
|
+
return cacheloc
|
|
46
|
+
|
|
47
|
+
|
|
33
48
|
class MagicPlace(Store):
|
|
34
49
|
"""Somewhere, over the rainbow!"""
|
|
35
50
|
|
|
36
51
|
_footprint = dict(
|
|
37
|
-
info
|
|
38
|
-
attr
|
|
39
|
-
scheme
|
|
40
|
-
values
|
|
52
|
+
info="Evanescent physical store",
|
|
53
|
+
attr=dict(
|
|
54
|
+
scheme=dict(
|
|
55
|
+
values=["magic"],
|
|
41
56
|
),
|
|
42
57
|
),
|
|
43
|
-
priority
|
|
44
|
-
level
|
|
45
|
-
)
|
|
58
|
+
priority=dict(
|
|
59
|
+
level=footprints.priorities.top.DEFAULT # @UndefinedVariable
|
|
60
|
+
),
|
|
46
61
|
)
|
|
47
62
|
|
|
48
63
|
@property
|
|
49
64
|
def realkind(self):
|
|
50
|
-
return
|
|
65
|
+
return "magicstore"
|
|
51
66
|
|
|
52
67
|
def has_fast_check(self):
|
|
53
68
|
"""A void check is very fast !"""
|
|
@@ -59,7 +74,7 @@ class MagicPlace(Store):
|
|
|
59
74
|
|
|
60
75
|
def magiclocate(self, remote, options):
|
|
61
76
|
"""Void - Empty string returned."""
|
|
62
|
-
return
|
|
77
|
+
return ""
|
|
63
78
|
|
|
64
79
|
def magicget(self, remote, local, options):
|
|
65
80
|
"""Void - Always True."""
|
|
@@ -108,23 +123,23 @@ class FunctionStore(Store):
|
|
|
108
123
|
"""
|
|
109
124
|
|
|
110
125
|
_footprint = dict(
|
|
111
|
-
info
|
|
112
|
-
attr
|
|
113
|
-
scheme
|
|
114
|
-
values
|
|
126
|
+
info="Dummy store that calls a function",
|
|
127
|
+
attr=dict(
|
|
128
|
+
scheme=dict(
|
|
129
|
+
values=["function"],
|
|
130
|
+
),
|
|
131
|
+
netloc=dict(
|
|
132
|
+
values=[""],
|
|
115
133
|
),
|
|
116
|
-
netloc = dict(
|
|
117
|
-
values = [''],
|
|
118
|
-
)
|
|
119
134
|
),
|
|
120
|
-
priority
|
|
121
|
-
level
|
|
122
|
-
)
|
|
135
|
+
priority=dict(
|
|
136
|
+
level=footprints.priorities.top.DEFAULT # @UndefinedVariable
|
|
137
|
+
),
|
|
123
138
|
)
|
|
124
139
|
|
|
125
140
|
@property
|
|
126
141
|
def realkind(self):
|
|
127
|
-
return
|
|
142
|
+
return "functionstore"
|
|
128
143
|
|
|
129
144
|
def has_fast_check(self):
|
|
130
145
|
"""A void check is very fast !"""
|
|
@@ -136,20 +151,21 @@ class FunctionStore(Store):
|
|
|
136
151
|
|
|
137
152
|
def functionlocate(self, remote, options):
|
|
138
153
|
"""The name of the function that will be called."""
|
|
139
|
-
cleanname = remote[
|
|
140
|
-
if cleanname.endswith(
|
|
154
|
+
cleanname = remote["path"][1:]
|
|
155
|
+
if cleanname.endswith("/"):
|
|
141
156
|
cleanname = cleanname[:-1]
|
|
142
157
|
return cleanname
|
|
143
158
|
|
|
144
159
|
def functionget(self, remote, local, options):
|
|
145
160
|
"""Calls the appropriate function and writes the result."""
|
|
146
161
|
# Find the appropriate function
|
|
147
|
-
cbfunc = self.system.import_function(
|
|
148
|
-
|
|
162
|
+
cbfunc = self.system.import_function(
|
|
163
|
+
self.functionlocate(remote, options)
|
|
164
|
+
)
|
|
149
165
|
# ... and call it
|
|
150
166
|
opts = dict()
|
|
151
167
|
opts.update(options)
|
|
152
|
-
opts.update(remote[
|
|
168
|
+
opts.update(remote["query"])
|
|
153
169
|
try:
|
|
154
170
|
fres = cbfunc(opts)
|
|
155
171
|
except FunctionStoreCallbackError as e:
|
|
@@ -157,15 +173,15 @@ class FunctionStore(Store):
|
|
|
157
173
|
logger.error("Here is the exception: %s", str(e))
|
|
158
174
|
fres = None
|
|
159
175
|
if fres is not None:
|
|
160
|
-
if
|
|
161
|
-
logger.info(
|
|
176
|
+
if "intent" in options and options["intent"] == dataflow.intent.IN:
|
|
177
|
+
logger.info("Ignore intent <in> for function input.")
|
|
162
178
|
# Handle StringIO objects, by changing them to ByteIOs...
|
|
163
179
|
if isinstance(fres, io.StringIO):
|
|
164
180
|
s_fres = fres
|
|
165
181
|
s_fres.seek(0)
|
|
166
182
|
fres = io.BytesIO()
|
|
167
183
|
for l in s_fres:
|
|
168
|
-
fres.write(l.encode(encoding=
|
|
184
|
+
fres.write(l.encode(encoding="utf-8"))
|
|
169
185
|
fres.seek(0)
|
|
170
186
|
# NB: fres should be a file like object (BytesIO will do the trick)
|
|
171
187
|
return self.system.cp(fres, local)
|
|
@@ -187,30 +203,30 @@ class Finder(Store):
|
|
|
187
203
|
"""The most usual store: your current filesystem!"""
|
|
188
204
|
|
|
189
205
|
_footprint = dict(
|
|
190
|
-
info
|
|
191
|
-
attr
|
|
192
|
-
scheme
|
|
193
|
-
values
|
|
206
|
+
info="Miscellaneous file access",
|
|
207
|
+
attr=dict(
|
|
208
|
+
scheme=dict(
|
|
209
|
+
values=["file", "ftp", "symlink", "rcp", "scp"],
|
|
194
210
|
),
|
|
195
|
-
netloc
|
|
196
|
-
outcast
|
|
211
|
+
netloc=dict(
|
|
212
|
+
outcast=["oper.inline.fr"],
|
|
197
213
|
),
|
|
198
|
-
storehash
|
|
199
|
-
values
|
|
214
|
+
storehash=dict(
|
|
215
|
+
values=hashalgo_avail_list,
|
|
200
216
|
),
|
|
201
217
|
),
|
|
202
|
-
priority
|
|
203
|
-
level
|
|
204
|
-
)
|
|
218
|
+
priority=dict(
|
|
219
|
+
level=footprints.priorities.top.DEFAULT # @UndefinedVariable
|
|
220
|
+
),
|
|
205
221
|
)
|
|
206
222
|
|
|
207
223
|
def __init__(self, *args, **kw):
|
|
208
|
-
logger.debug(
|
|
224
|
+
logger.debug("Finder store init %s", self.__class__)
|
|
209
225
|
super().__init__(*args, **kw)
|
|
210
226
|
|
|
211
227
|
@property
|
|
212
228
|
def realkind(self):
|
|
213
|
-
return
|
|
229
|
+
return "finder"
|
|
214
230
|
|
|
215
231
|
def hostname(self):
|
|
216
232
|
"""Returns the current :attr:`netloc`."""
|
|
@@ -218,21 +234,28 @@ class Finder(Store):
|
|
|
218
234
|
|
|
219
235
|
def fullpath(self, remote):
|
|
220
236
|
"""Return actual path unless explicitly defined as relative path."""
|
|
221
|
-
if remote[
|
|
222
|
-
return remote[
|
|
237
|
+
if remote["query"].get("relative", False):
|
|
238
|
+
return remote["path"].lstrip("/")
|
|
223
239
|
else:
|
|
224
|
-
return remote[
|
|
240
|
+
return remote["path"]
|
|
225
241
|
|
|
226
242
|
def _localtarfix(self, local):
|
|
227
|
-
if (
|
|
228
|
-
|
|
229
|
-
|
|
243
|
+
if (
|
|
244
|
+
isinstance(local, str)
|
|
245
|
+
and self.system.path.isfile(local)
|
|
246
|
+
and self.system.is_tarfile(local)
|
|
247
|
+
):
|
|
248
|
+
destdir = self.system.path.dirname(
|
|
249
|
+
self.system.path.realpath(local)
|
|
250
|
+
)
|
|
230
251
|
try:
|
|
231
252
|
self.system.smartuntar(local, destdir)
|
|
232
253
|
except ExecutionError:
|
|
233
254
|
if not self.system.is_tarname(local):
|
|
234
|
-
logger.warning(
|
|
235
|
-
|
|
255
|
+
logger.warning(
|
|
256
|
+
"An automatic untar was attempted but it failed. "
|
|
257
|
+
+ "Maybe the system's is_tarfile got it wrong ?"
|
|
258
|
+
)
|
|
236
259
|
else:
|
|
237
260
|
raise
|
|
238
261
|
|
|
@@ -251,10 +274,12 @@ class Finder(Store):
|
|
|
251
274
|
def fileget(self, remote, local, options):
|
|
252
275
|
"""Delegates to ``system`` the copy of ``remote`` to ``local``."""
|
|
253
276
|
rpath = self.fullpath(remote)
|
|
254
|
-
logger.info(
|
|
255
|
-
if
|
|
256
|
-
logger.info(
|
|
257
|
-
rc = self.system.cp(
|
|
277
|
+
logger.info("fileget on %s (to: %s)", rpath, local)
|
|
278
|
+
if "intent" in options and options["intent"] == dataflow.intent.IN:
|
|
279
|
+
logger.info("Ignore intent <in> for remote input %s", rpath)
|
|
280
|
+
rc = self.system.cp(
|
|
281
|
+
rpath, local, fmt=options.get("fmt"), intent=dataflow.intent.INOUT
|
|
282
|
+
)
|
|
258
283
|
rc = rc and self._hash_get_check(self.fileget, remote, local, options)
|
|
259
284
|
if rc:
|
|
260
285
|
self._localtarfix(local)
|
|
@@ -263,8 +288,8 @@ class Finder(Store):
|
|
|
263
288
|
def fileput(self, local, remote, options):
|
|
264
289
|
"""Delegates to ``system`` the copy of ``local`` to ``remote``."""
|
|
265
290
|
rpath = self.fullpath(remote)
|
|
266
|
-
logger.info(
|
|
267
|
-
rc = self.system.cp(local, rpath, fmt=options.get(
|
|
291
|
+
logger.info("fileput to %s (from: %s)", rpath, local)
|
|
292
|
+
rc = self.system.cp(local, rpath, fmt=options.get("fmt"))
|
|
268
293
|
return rc and self._hash_put(self.fileput, local, remote, options)
|
|
269
294
|
|
|
270
295
|
def filedelete(self, remote, options):
|
|
@@ -272,10 +297,13 @@ class Finder(Store):
|
|
|
272
297
|
rc = None
|
|
273
298
|
if self.filecheck(remote, options):
|
|
274
299
|
rpath = self.fullpath(remote)
|
|
275
|
-
logger.info(
|
|
276
|
-
rc = self.system.remove(rpath, fmt=options.get(
|
|
300
|
+
logger.info("filedelete on %s", rpath)
|
|
301
|
+
rc = self.system.remove(rpath, fmt=options.get("fmt"))
|
|
277
302
|
else:
|
|
278
|
-
logger.error(
|
|
303
|
+
logger.error(
|
|
304
|
+
"Try to remove a non-existing resource <%s>",
|
|
305
|
+
self.fullpath(remote),
|
|
306
|
+
)
|
|
279
307
|
return rc
|
|
280
308
|
|
|
281
309
|
symlinkcheck = filecheck
|
|
@@ -283,34 +311,40 @@ class Finder(Store):
|
|
|
283
311
|
|
|
284
312
|
def symlinkget(self, remote, local, options):
|
|
285
313
|
rpath = self.fullpath(remote)
|
|
286
|
-
if
|
|
287
|
-
logger.error(
|
|
314
|
+
if "intent" in options and options["intent"] == dataflow.intent.INOUT:
|
|
315
|
+
logger.error(
|
|
316
|
+
"It is unsafe to have a symlink with intent=inout: %s", rpath
|
|
317
|
+
)
|
|
288
318
|
return False
|
|
289
319
|
rc = self.system.remove(local)
|
|
290
320
|
self.system.symlink(rpath, local)
|
|
291
321
|
return rc and self.system.path.exists(local)
|
|
292
322
|
|
|
293
323
|
def symlinkput(self, local, remote, options):
|
|
294
|
-
logger.error(
|
|
324
|
+
logger.error(
|
|
325
|
+
"The Finder store with scheme:symlink is not able to perform Puts."
|
|
326
|
+
)
|
|
295
327
|
return False
|
|
296
328
|
|
|
297
329
|
def symlinkdelete(self, remote, options):
|
|
298
|
-
logger.error(
|
|
330
|
+
logger.error(
|
|
331
|
+
"The Finder store with scheme:symlink is not able to perform Deletes."
|
|
332
|
+
)
|
|
299
333
|
return False
|
|
300
334
|
|
|
301
335
|
def _ftpinfos(self, remote, **kwargs):
|
|
302
336
|
args = kwargs.copy()
|
|
303
|
-
args[
|
|
304
|
-
args[
|
|
337
|
+
args["hostname"] = self.hostname()
|
|
338
|
+
args["logname"] = remote["username"]
|
|
305
339
|
port = self.hostname().netport
|
|
306
340
|
if port is not None:
|
|
307
|
-
args[
|
|
341
|
+
args["port"] = port
|
|
308
342
|
return args
|
|
309
343
|
|
|
310
344
|
def ftpcheck(self, remote, options):
|
|
311
345
|
"""Delegates to ``system.ftp`` a distant check."""
|
|
312
346
|
rc = None
|
|
313
|
-
ftp = self.system.ftp(**
|
|
347
|
+
ftp = self.system.ftp(**self._ftpinfos(remote))
|
|
314
348
|
if ftp:
|
|
315
349
|
try:
|
|
316
350
|
rc = ftp.size(self.fullpath(remote))
|
|
@@ -324,7 +358,7 @@ class Finder(Store):
|
|
|
324
358
|
|
|
325
359
|
def ftplocate(self, remote, options):
|
|
326
360
|
"""Delegates to ``system`` qualified name creation."""
|
|
327
|
-
ftp = self.system.ftp(**
|
|
361
|
+
ftp = self.system.ftp(**self._ftpinfos(remote, delayed=True))
|
|
328
362
|
if ftp:
|
|
329
363
|
rloc = ftp.netpath(self.fullpath(remote))
|
|
330
364
|
ftp.close()
|
|
@@ -335,13 +369,15 @@ class Finder(Store):
|
|
|
335
369
|
def ftpget(self, remote, local, options):
|
|
336
370
|
"""Delegates to ``system`` the file transfer of ``remote`` to ``local``."""
|
|
337
371
|
rpath = self.fullpath(remote)
|
|
338
|
-
logger.info(
|
|
372
|
+
logger.info(
|
|
373
|
+
"ftpget on ftp://%s/%s (to: %s)", self.hostname(), rpath, local
|
|
374
|
+
)
|
|
339
375
|
rc = self.system.smartftget(
|
|
340
376
|
rpath,
|
|
341
377
|
local,
|
|
342
|
-
fmt=options.get(
|
|
378
|
+
fmt=options.get("fmt"),
|
|
343
379
|
# ftp control
|
|
344
|
-
**
|
|
380
|
+
**self._ftpinfos(remote),
|
|
345
381
|
)
|
|
346
382
|
rc = rc and self._hash_get_check(self.ftpget, remote, local, options)
|
|
347
383
|
if rc:
|
|
@@ -352,14 +388,16 @@ class Finder(Store):
|
|
|
352
388
|
"""Delegates to ``system`` the file transfer of ``local`` to ``remote``."""
|
|
353
389
|
rpath = self.fullpath(remote)
|
|
354
390
|
put_opts = dict()
|
|
355
|
-
put_opts[
|
|
356
|
-
put_opts[
|
|
357
|
-
logger.info(
|
|
391
|
+
put_opts["fmt"] = options.get("fmt")
|
|
392
|
+
put_opts["sync"] = options.get("enforcesync", False)
|
|
393
|
+
logger.info(
|
|
394
|
+
"ftpput to ftp://%s/%s (from: %s)", self.hostname(), rpath, local
|
|
395
|
+
)
|
|
358
396
|
rc = self.system.smartftput(
|
|
359
397
|
local,
|
|
360
398
|
rpath,
|
|
361
399
|
# ftp control
|
|
362
|
-
**
|
|
400
|
+
**self._ftpinfos(remote, **put_opts),
|
|
363
401
|
)
|
|
364
402
|
return rc and self._hash_put(self.ftpput, local, remote, options)
|
|
365
403
|
|
|
@@ -368,7 +406,9 @@ class Finder(Store):
|
|
|
368
406
|
rc = None
|
|
369
407
|
actualpath = self.fullpath(remote)
|
|
370
408
|
if self.ftpcheck(remote, options):
|
|
371
|
-
logger.info(
|
|
409
|
+
logger.info(
|
|
410
|
+
"ftpdelete on ftp://%s/%s", self.hostname(), actualpath
|
|
411
|
+
)
|
|
372
412
|
ftp = self.system.ftp(**self._ftpinfos(remote))
|
|
373
413
|
if ftp:
|
|
374
414
|
try:
|
|
@@ -376,14 +416,16 @@ class Finder(Store):
|
|
|
376
416
|
finally:
|
|
377
417
|
ftp.close()
|
|
378
418
|
else:
|
|
379
|
-
logger.error(
|
|
419
|
+
logger.error(
|
|
420
|
+
"Try to remove a non-existing resource <%s>", actualpath
|
|
421
|
+
)
|
|
380
422
|
return rc
|
|
381
423
|
|
|
382
424
|
|
|
383
425
|
class _VortexStackedStorageMixin:
|
|
384
426
|
"""Mixin class that adds utility functions to work with stacked data."""
|
|
385
427
|
|
|
386
|
-
_STACKED_RE = re.compile(
|
|
428
|
+
_STACKED_RE = re.compile("stacked-")
|
|
387
429
|
|
|
388
430
|
@property
|
|
389
431
|
def stackedstore(self):
|
|
@@ -391,25 +433,31 @@ class _VortexStackedStorageMixin:
|
|
|
391
433
|
return self._STACKED_RE.search(self.netloc)
|
|
392
434
|
|
|
393
435
|
def _stacked_remainder(self, remote, stackpath):
|
|
394
|
-
path_remainder = remote[
|
|
395
|
-
for a_spath in stackpath.split(
|
|
436
|
+
path_remainder = remote["path"].strip("/").split("/")
|
|
437
|
+
for a_spath in stackpath.split("/"):
|
|
396
438
|
if path_remainder and path_remainder[0] == a_spath:
|
|
397
439
|
del path_remainder[0]
|
|
398
440
|
else:
|
|
399
441
|
break
|
|
400
|
-
return
|
|
442
|
+
return "/".join(path_remainder)
|
|
401
443
|
|
|
402
444
|
def _stacked_xremote(self, remote):
|
|
403
445
|
"""The path to **remote** with its stack."""
|
|
404
446
|
if self.stackedstore:
|
|
405
447
|
remote = remote.copy()
|
|
406
|
-
remote[
|
|
407
|
-
stackpath = remote[
|
|
408
|
-
stackfmt = remote[
|
|
448
|
+
remote["query"] = remote["query"].copy()
|
|
449
|
+
stackpath = remote["query"].pop("stackpath", (None,))[0]
|
|
450
|
+
stackfmt = remote["query"].pop("stackfmt", (None,))[0]
|
|
409
451
|
if stackpath is None or stackfmt is None:
|
|
410
|
-
raise ValueError(
|
|
452
|
+
raise ValueError(
|
|
453
|
+
'"stackpath" and "stackfmt" are not available in the query.'
|
|
454
|
+
)
|
|
411
455
|
else:
|
|
412
|
-
remote[
|
|
456
|
+
remote["path"] = (
|
|
457
|
+
stackpath
|
|
458
|
+
+ "/"
|
|
459
|
+
+ self._stacked_remainder(remote, stackpath)
|
|
460
|
+
)
|
|
413
461
|
return remote
|
|
414
462
|
|
|
415
463
|
def _stacked_xegglocate(self, remote):
|
|
@@ -423,14 +471,16 @@ class _VortexStackedStorageMixin:
|
|
|
423
471
|
|
|
424
472
|
"""
|
|
425
473
|
remote = remote.copy()
|
|
426
|
-
remote[
|
|
427
|
-
stackpath = remote[
|
|
428
|
-
stackfmt = remote[
|
|
474
|
+
remote["query"] = remote["query"].copy()
|
|
475
|
+
stackpath = remote["query"].pop("stackpath", (None,))[0].strip("/")
|
|
476
|
+
stackfmt = remote["query"].pop("stackfmt", (None,))[0]
|
|
429
477
|
if stackpath is None or stackfmt is None:
|
|
430
|
-
raise ValueError(
|
|
478
|
+
raise ValueError(
|
|
479
|
+
'"stackpath" and "stackfmt" are not available in the query.'
|
|
480
|
+
)
|
|
431
481
|
else:
|
|
432
482
|
resource_remainder = self._stacked_remainder(remote, stackpath)
|
|
433
|
-
remote[
|
|
483
|
+
remote["path"] = "/" + stackpath
|
|
434
484
|
return remote, stackfmt, resource_remainder
|
|
435
485
|
|
|
436
486
|
|
|
@@ -438,11 +488,13 @@ _vortex_readonly_store = footprints.Footprint(
|
|
|
438
488
|
info="Abstract store' readonly=True attribute",
|
|
439
489
|
attr=dict(
|
|
440
490
|
readonly=dict(
|
|
441
|
-
values=[
|
|
491
|
+
values=[
|
|
492
|
+
True,
|
|
493
|
+
],
|
|
442
494
|
optional=True,
|
|
443
|
-
default=True
|
|
495
|
+
default=True,
|
|
444
496
|
)
|
|
445
|
-
)
|
|
497
|
+
),
|
|
446
498
|
)
|
|
447
499
|
|
|
448
500
|
|
|
@@ -451,37 +503,38 @@ class _VortexBaseArchiveStore(ArchiveStore, _VortexStackedStorageMixin):
|
|
|
451
503
|
|
|
452
504
|
_abstract = True
|
|
453
505
|
_footprint = dict(
|
|
454
|
-
info
|
|
455
|
-
attr
|
|
456
|
-
scheme
|
|
457
|
-
values
|
|
458
|
-
),
|
|
459
|
-
netloc = dict(
|
|
506
|
+
info="VORTEX archive access",
|
|
507
|
+
attr=dict(
|
|
508
|
+
scheme=dict(
|
|
509
|
+
values=["vortex"],
|
|
460
510
|
),
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
511
|
+
netloc=dict(),
|
|
512
|
+
storehead=dict(
|
|
513
|
+
optional=True,
|
|
514
|
+
default="vortex",
|
|
515
|
+
outcast=["xp"],
|
|
465
516
|
),
|
|
466
|
-
)
|
|
517
|
+
),
|
|
467
518
|
)
|
|
468
519
|
|
|
469
|
-
_STACKS_AUTOREFILL_CRIT =
|
|
520
|
+
_STACKS_AUTOREFILL_CRIT = "stacked-archive-smart"
|
|
470
521
|
|
|
471
522
|
def __init__(self, *args, **kw):
|
|
472
|
-
logger.debug(
|
|
523
|
+
logger.debug("Vortex archive store init %s", self.__class__)
|
|
473
524
|
super().__init__(*args, **kw)
|
|
474
525
|
|
|
475
526
|
def remap_read(self, remote, options):
|
|
476
527
|
"""Remap actual remote path to distant store path for intrusive actions."""
|
|
477
|
-
|
|
528
|
+
raise NotImplementedError
|
|
478
529
|
|
|
479
530
|
def remap_list(self, remote, options):
|
|
480
531
|
"""Reformulates the remote path to compatible vortex namespace."""
|
|
481
|
-
if len(remote[
|
|
532
|
+
if len(remote["path"].split("/")) >= 4:
|
|
482
533
|
return self.remap_read(remote, options)
|
|
483
534
|
else:
|
|
484
|
-
logger.critical(
|
|
535
|
+
logger.critical(
|
|
536
|
+
"The << %s >> path is not listable.", remote["path"]
|
|
537
|
+
)
|
|
485
538
|
return None
|
|
486
539
|
|
|
487
540
|
remap_write = remap_read
|
|
@@ -490,7 +543,7 @@ class _VortexBaseArchiveStore(ArchiveStore, _VortexStackedStorageMixin):
|
|
|
490
543
|
def stacks_autorefill(self):
|
|
491
544
|
"""Where to refill a stack retrieved from the archive."""
|
|
492
545
|
if self._STACKS_AUTOREFILL_CRIT in self.netloc:
|
|
493
|
-
return self.netloc.replace(self._STACKS_AUTOREFILL_CRIT,
|
|
546
|
+
return self.netloc.replace(self._STACKS_AUTOREFILL_CRIT, "cache")
|
|
494
547
|
else:
|
|
495
548
|
return None
|
|
496
549
|
|
|
@@ -500,24 +553,34 @@ class _VortexBaseArchiveStore(ArchiveStore, _VortexStackedStorageMixin):
|
|
|
500
553
|
rundir = sessions.current().context.rundir
|
|
501
554
|
if not rundir:
|
|
502
555
|
rundir = self.system.pwd()
|
|
503
|
-
rundir = self.system.path.join(rundir,
|
|
504
|
-
target = self.system.path.join(
|
|
556
|
+
rundir = self.system.path.join(rundir, "vortex_stacks_xeggs")
|
|
557
|
+
target = self.system.path.join(
|
|
558
|
+
rundir, *remote["path"].strip("/").split("/")
|
|
559
|
+
)
|
|
505
560
|
targetopts = dict(fmt=remotefmt, intent=dataflow.intent.IN)
|
|
506
561
|
if self.system.path.exists(target):
|
|
507
|
-
logger.info(
|
|
562
|
+
logger.info(
|
|
563
|
+
"Stack previously retrieved (in %s). Using it.", target
|
|
564
|
+
)
|
|
508
565
|
rc = True
|
|
509
566
|
else:
|
|
510
567
|
if result_id:
|
|
511
|
-
rc = self._vortexfinaliseget(
|
|
568
|
+
rc = self._vortexfinaliseget(
|
|
569
|
+
result_id, remote, target, targetopts
|
|
570
|
+
)
|
|
512
571
|
else:
|
|
513
572
|
rc = self._vortexget(remote, target, targetopts)
|
|
514
573
|
if rc and self.stacks_autorefill:
|
|
515
|
-
rstore = footprints.proxy.store(
|
|
574
|
+
rstore = footprints.proxy.store(
|
|
575
|
+
scheme=self.scheme, netloc=self.stacks_autorefill
|
|
576
|
+
)
|
|
516
577
|
logger.info("Refilling the stack egg to [%s]", rstore)
|
|
517
578
|
try:
|
|
518
579
|
rstore.put(target, remote.copy(), targetopts)
|
|
519
580
|
except (ExecutionError, OSError) as e:
|
|
520
|
-
logger.error(
|
|
581
|
+
logger.error(
|
|
582
|
+
"An ExecutionError happened during the refill: %s", str(e)
|
|
583
|
+
)
|
|
521
584
|
logger.error("This error is ignored... but that's ugly !")
|
|
522
585
|
return rc, target, remainder
|
|
523
586
|
|
|
@@ -526,11 +589,15 @@ class _VortexBaseArchiveStore(ArchiveStore, _VortexStackedStorageMixin):
|
|
|
526
589
|
if self.stackedstore:
|
|
527
590
|
s_remote, s_remotefmt, _ = self._stacked_xegglocate(remote)
|
|
528
591
|
options = options.copy()
|
|
529
|
-
options[
|
|
592
|
+
options["fmt"] = s_remotefmt
|
|
530
593
|
rc = self._vortexcheck(s_remote, options)
|
|
531
594
|
if rc:
|
|
532
|
-
rc, target, remainder = self._vortex_stacked_egg_retrieve(
|
|
533
|
-
|
|
595
|
+
rc, target, remainder = self._vortex_stacked_egg_retrieve(
|
|
596
|
+
remote
|
|
597
|
+
)
|
|
598
|
+
rc = rc and self.system.path.exists(
|
|
599
|
+
self.system.path.join(target, remainder)
|
|
600
|
+
)
|
|
534
601
|
return rc
|
|
535
602
|
else:
|
|
536
603
|
return self._vortexcheck(remote, options)
|
|
@@ -545,7 +612,7 @@ class _VortexBaseArchiveStore(ArchiveStore, _VortexStackedStorageMixin):
|
|
|
545
612
|
if self.stackedstore:
|
|
546
613
|
remote, s_remotefmt, _ = self._stacked_xegglocate(remote)
|
|
547
614
|
options = options.copy()
|
|
548
|
-
options[
|
|
615
|
+
options["fmt"] = s_remotefmt
|
|
549
616
|
return self._vortexlocate(remote, options)
|
|
550
617
|
|
|
551
618
|
def _vortexlocate(self, remote, options):
|
|
@@ -573,7 +640,7 @@ class _VortexBaseArchiveStore(ArchiveStore, _VortexStackedStorageMixin):
|
|
|
573
640
|
if self.stackedstore:
|
|
574
641
|
remote, s_remotefmt, _ = self._stacked_xegglocate(remote)
|
|
575
642
|
options = options.copy()
|
|
576
|
-
options[
|
|
643
|
+
options["fmt"] = s_remotefmt
|
|
577
644
|
return self._vortexprestageinfo(remote, options)
|
|
578
645
|
|
|
579
646
|
def _vortexprestageinfo(self, remote, options):
|
|
@@ -585,10 +652,12 @@ class _VortexBaseArchiveStore(ArchiveStore, _VortexStackedStorageMixin):
|
|
|
585
652
|
"""Vortex' archive get sequence."""
|
|
586
653
|
if self.stackedstore:
|
|
587
654
|
rc, target, remainder = self._vortex_stacked_egg_retrieve(remote)
|
|
588
|
-
rc = rc and self.system.cp(
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
655
|
+
rc = rc and self.system.cp(
|
|
656
|
+
self.system.path.join(target, remainder),
|
|
657
|
+
local,
|
|
658
|
+
fmt=options.get("fmt"),
|
|
659
|
+
intent=options.get("intent", ARCHIVE_GET_INTENT_DEFAULT),
|
|
660
|
+
)
|
|
592
661
|
return rc
|
|
593
662
|
else:
|
|
594
663
|
return self._vortexget(remote, local, options)
|
|
@@ -603,7 +672,7 @@ class _VortexBaseArchiveStore(ArchiveStore, _VortexStackedStorageMixin):
|
|
|
603
672
|
if self.stackedstore:
|
|
604
673
|
s_remote, s_remotefmt, _ = self._stacked_xegglocate(remote)
|
|
605
674
|
targetopts = dict(fmt=s_remotefmt, intent=dataflow.intent.IN)
|
|
606
|
-
return self._vortexearlyget(s_remote,
|
|
675
|
+
return self._vortexearlyget(s_remote, "somelocalfile", targetopts)
|
|
607
676
|
else:
|
|
608
677
|
return self._vortexearlyget(remote, local, options)
|
|
609
678
|
|
|
@@ -615,11 +684,15 @@ class _VortexBaseArchiveStore(ArchiveStore, _VortexStackedStorageMixin):
|
|
|
615
684
|
def vortexfinaliseget(self, result_id, remote, local, options):
|
|
616
685
|
"""Vortex' archive finaliseget sequence."""
|
|
617
686
|
if self.stackedstore:
|
|
618
|
-
rc, target, remainder = self._vortex_stacked_egg_retrieve(
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
687
|
+
rc, target, remainder = self._vortex_stacked_egg_retrieve(
|
|
688
|
+
remote, result_id=result_id
|
|
689
|
+
)
|
|
690
|
+
rc = rc and self.system.cp(
|
|
691
|
+
self.system.path.join(target, remainder),
|
|
692
|
+
local,
|
|
693
|
+
fmt=options.get("fmt"),
|
|
694
|
+
intent=options.get("intent", ARCHIVE_GET_INTENT_DEFAULT),
|
|
695
|
+
)
|
|
623
696
|
return rc
|
|
624
697
|
else:
|
|
625
698
|
return self._vortexfinaliseget(result_id, remote, local, options)
|
|
@@ -655,30 +728,31 @@ class VortexStdBaseArchiveStore(_VortexBaseArchiveStore):
|
|
|
655
728
|
"""
|
|
656
729
|
|
|
657
730
|
_footprint = dict(
|
|
658
|
-
info
|
|
659
|
-
attr
|
|
660
|
-
netloc
|
|
661
|
-
values
|
|
731
|
+
info="VORTEX archive access for casual experiments",
|
|
732
|
+
attr=dict(
|
|
733
|
+
netloc=dict(
|
|
734
|
+
values=["vortex.archive-legacy.fr"],
|
|
662
735
|
),
|
|
663
|
-
)
|
|
736
|
+
),
|
|
664
737
|
)
|
|
665
738
|
|
|
666
|
-
@property
|
|
667
|
-
def _actual_mappingroot(self):
|
|
668
|
-
"""Read the get entry point form configuration."""
|
|
669
|
-
return config.from_config(
|
|
670
|
-
section="storage", key="rootdir",
|
|
671
|
-
)
|
|
672
|
-
|
|
673
739
|
def remap_read(self, remote, options):
|
|
674
740
|
"""Reformulates the remote path to compatible vortex namespace."""
|
|
675
741
|
remote = copy.copy(remote)
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
742
|
+
try:
|
|
743
|
+
remote["root"] = config.from_config(
|
|
744
|
+
section="storage",
|
|
745
|
+
key="rootdir",
|
|
746
|
+
)
|
|
747
|
+
except config.ConfigurationError as e:
|
|
748
|
+
msg = (
|
|
749
|
+
"Trying to write to archive but location is not configured.\n"
|
|
750
|
+
'Make sure key "rootdir" is defined in storage section of '
|
|
751
|
+
"the configuration.\n"
|
|
752
|
+
"See https://vortex-nwp.readthedocs.io/en/latest/user-guide/configuration.html#storage"
|
|
753
|
+
)
|
|
754
|
+
logger.error(msg)
|
|
755
|
+
raise e
|
|
682
756
|
return remote
|
|
683
757
|
|
|
684
758
|
|
|
@@ -693,70 +767,18 @@ class VortexStdStackedArchiveStore(VortexStdBaseArchiveStore):
|
|
|
693
767
|
_footprint = [
|
|
694
768
|
_vortex_readonly_store,
|
|
695
769
|
dict(
|
|
696
|
-
attr
|
|
697
|
-
netloc
|
|
698
|
-
values
|
|
699
|
-
|
|
770
|
+
attr=dict(
|
|
771
|
+
netloc=dict(
|
|
772
|
+
values=[
|
|
773
|
+
"vortex.stacked-archive-legacy.fr",
|
|
774
|
+
"vortex.stacked-archive-smart.fr",
|
|
775
|
+
],
|
|
700
776
|
),
|
|
701
777
|
)
|
|
702
|
-
)
|
|
778
|
+
),
|
|
703
779
|
]
|
|
704
780
|
|
|
705
781
|
|
|
706
|
-
class VortexFreeStdBaseArchiveStore(_VortexBaseArchiveStore, ConfigurableArchiveStore):
|
|
707
|
-
"""Archive for casual VORTEX experiments: Support for Free XPIDs.
|
|
708
|
-
|
|
709
|
-
This 'archive-legacy' store looks into the resource 'main' location not
|
|
710
|
-
into a potential stack.
|
|
711
|
-
"""
|
|
712
|
-
|
|
713
|
-
#: Path to the vortex-free Store configuration file
|
|
714
|
-
_store_global_config = '@store-vortex-free.ini'
|
|
715
|
-
_datastore_id = 'store-vortex-free-conf'
|
|
716
|
-
|
|
717
|
-
_footprint = dict(
|
|
718
|
-
info = 'VORTEX archive access for casual experiments',
|
|
719
|
-
attr = dict(
|
|
720
|
-
netloc = dict(
|
|
721
|
-
values = ['vortex-free.archive-legacy.fr'],
|
|
722
|
-
),
|
|
723
|
-
)
|
|
724
|
-
)
|
|
725
|
-
|
|
726
|
-
def remap_read(self, remote, options):
|
|
727
|
-
"""Reformulates the remote path to compatible vortex namespace."""
|
|
728
|
-
remote = copy.copy(remote)
|
|
729
|
-
xpath = remote['path'].strip('/').split('/')
|
|
730
|
-
f_xpid = FreeXPid(xpath[2])
|
|
731
|
-
xpath[2] = f_xpid.id
|
|
732
|
-
if 'root' not in remote:
|
|
733
|
-
remote['root'] = self._actual_storeroot(f_xpid)
|
|
734
|
-
remote['path'] = self.system.path.join(*xpath)
|
|
735
|
-
return remote
|
|
736
|
-
|
|
737
|
-
remap_write = remap_read
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
class VortexFreeStdStackedArchiveStore(VortexFreeStdBaseArchiveStore):
|
|
741
|
-
"""Archive for casual VORTEX experiments: Support for Free XPIDs.
|
|
742
|
-
|
|
743
|
-
This 'stacked-archive-legacy' or 'stacked-archive-smart' store looks into
|
|
744
|
-
the stack associated to the resource. The '-smart' variant, has the ability
|
|
745
|
-
to refill the whole stack into local cache (to be faster in the future).
|
|
746
|
-
"""
|
|
747
|
-
|
|
748
|
-
_footprint = [
|
|
749
|
-
_vortex_readonly_store,
|
|
750
|
-
dict(
|
|
751
|
-
attr = dict(
|
|
752
|
-
netloc = dict(
|
|
753
|
-
values = ['vortex-free.stacked-archive-legacy.fr',
|
|
754
|
-
'vortex-free.stacked-archive-smart.fr'],
|
|
755
|
-
),
|
|
756
|
-
)
|
|
757
|
-
)]
|
|
758
|
-
|
|
759
|
-
|
|
760
782
|
class VortexOpBaseArchiveStore(_VortexBaseArchiveStore):
|
|
761
783
|
"""Archive for op VORTEX experiments.
|
|
762
784
|
|
|
@@ -765,39 +787,43 @@ class VortexOpBaseArchiveStore(_VortexBaseArchiveStore):
|
|
|
765
787
|
"""
|
|
766
788
|
|
|
767
789
|
_footprint = dict(
|
|
768
|
-
info
|
|
769
|
-
attr
|
|
770
|
-
netloc
|
|
771
|
-
values
|
|
790
|
+
info="VORTEX archive access for op experiments",
|
|
791
|
+
attr=dict(
|
|
792
|
+
netloc=dict(
|
|
793
|
+
values=["vsop.archive-legacy.fr"],
|
|
772
794
|
),
|
|
773
|
-
storetrue
|
|
774
|
-
default
|
|
795
|
+
storetrue=dict(
|
|
796
|
+
default=DelayedEnvValue("op_archive", True),
|
|
775
797
|
),
|
|
776
|
-
)
|
|
798
|
+
),
|
|
777
799
|
)
|
|
778
800
|
|
|
779
|
-
@property
|
|
780
|
-
def _actual_storeroot(self):
|
|
781
|
-
return (
|
|
782
|
-
self.storeroot or
|
|
783
|
-
config.from_config(
|
|
784
|
-
section="storage", key="op_rootdir",
|
|
785
|
-
)
|
|
786
|
-
)
|
|
787
|
-
|
|
788
801
|
def remap_read(self, remote, options):
|
|
789
802
|
"""Reformulates the remote path to compatible vortex namespace."""
|
|
790
803
|
remote = copy.copy(remote)
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
804
|
+
try:
|
|
805
|
+
remote["root"] = config.from_config(
|
|
806
|
+
section="storage",
|
|
807
|
+
key="op_rootdir",
|
|
808
|
+
)
|
|
809
|
+
except config.ConfigurationError as e:
|
|
810
|
+
msg = (
|
|
811
|
+
"Trying to write to operational data archive but location"
|
|
812
|
+
'is not configured.\nMake sure key "rootdir" is defined in '
|
|
813
|
+
"the storage section of the configuration.\n"
|
|
814
|
+
"See https://vortex-nwp.readthedocs.io/en/latest/user-guide/configuration.html#storage"
|
|
815
|
+
)
|
|
816
|
+
logger.error(msg)
|
|
817
|
+
raise e
|
|
818
|
+
xpath = remote["path"].split("/")
|
|
819
|
+
if len(xpath) >= 5 and re.match(r"^\d{8}T\d{2,4}", xpath[4]):
|
|
794
820
|
# If a date is detected
|
|
795
821
|
vxdate = list(xpath[4])
|
|
796
|
-
vxdate.insert(4,
|
|
797
|
-
vxdate.insert(7,
|
|
798
|
-
vxdate.insert(10,
|
|
799
|
-
xpath[4] =
|
|
800
|
-
remote[
|
|
822
|
+
vxdate.insert(4, "/")
|
|
823
|
+
vxdate.insert(7, "/")
|
|
824
|
+
vxdate.insert(10, "/")
|
|
825
|
+
xpath[4] = "".join(vxdate)
|
|
826
|
+
remote["path"] = self.system.path.join(*xpath)
|
|
801
827
|
return remote
|
|
802
828
|
|
|
803
829
|
remap_write = remap_read
|
|
@@ -814,13 +840,16 @@ class VortexOpStackedArchiveStore(VortexOpBaseArchiveStore):
|
|
|
814
840
|
_footprint = [
|
|
815
841
|
_vortex_readonly_store,
|
|
816
842
|
dict(
|
|
817
|
-
attr
|
|
818
|
-
netloc
|
|
819
|
-
values
|
|
820
|
-
|
|
843
|
+
attr=dict(
|
|
844
|
+
netloc=dict(
|
|
845
|
+
values=[
|
|
846
|
+
"vsop.stacked-archive-legacy.fr",
|
|
847
|
+
"vsop.stacked-archive-smart.fr",
|
|
848
|
+
],
|
|
821
849
|
),
|
|
822
850
|
)
|
|
823
|
-
)
|
|
851
|
+
),
|
|
852
|
+
]
|
|
824
853
|
|
|
825
854
|
|
|
826
855
|
class VortexArchiveStore(MultiStore):
|
|
@@ -835,42 +864,58 @@ class VortexArchiveStore(MultiStore):
|
|
|
835
864
|
"""
|
|
836
865
|
|
|
837
866
|
_footprint = dict(
|
|
838
|
-
info
|
|
839
|
-
attr
|
|
840
|
-
scheme
|
|
841
|
-
values
|
|
867
|
+
info="VORTEX archive access",
|
|
868
|
+
attr=dict(
|
|
869
|
+
scheme=dict(
|
|
870
|
+
values=["vortex"],
|
|
842
871
|
),
|
|
843
|
-
netloc
|
|
844
|
-
values
|
|
872
|
+
netloc=dict(
|
|
873
|
+
values=[
|
|
874
|
+
"vortex.archive.fr",
|
|
875
|
+
"vortex-free.archive.fr",
|
|
876
|
+
"vsop.archive.fr",
|
|
877
|
+
],
|
|
845
878
|
),
|
|
846
|
-
refillstore
|
|
847
|
-
default
|
|
879
|
+
refillstore=dict(
|
|
880
|
+
default=False,
|
|
848
881
|
),
|
|
849
|
-
storehead
|
|
850
|
-
optional
|
|
882
|
+
storehead=dict(
|
|
883
|
+
optional=True,
|
|
851
884
|
),
|
|
852
|
-
storesync
|
|
853
|
-
alias
|
|
854
|
-
type
|
|
855
|
-
optional
|
|
885
|
+
storesync=dict(
|
|
886
|
+
alias=("archsync", "synchro"),
|
|
887
|
+
type=bool,
|
|
888
|
+
optional=True,
|
|
856
889
|
),
|
|
857
|
-
)
|
|
890
|
+
),
|
|
858
891
|
)
|
|
859
892
|
|
|
860
893
|
def filtered_readable_openedstores(self, remote):
|
|
861
894
|
"""Only use the stacked store if sensible."""
|
|
862
|
-
ostores = [
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
895
|
+
ostores = [
|
|
896
|
+
self.openedstores[0],
|
|
897
|
+
]
|
|
898
|
+
ostores.extend(
|
|
899
|
+
[
|
|
900
|
+
sto
|
|
901
|
+
for sto in self.openedstores[1:]
|
|
902
|
+
if not sto.stackedstore or "stackpath" in remote["query"]
|
|
903
|
+
]
|
|
904
|
+
)
|
|
866
905
|
return ostores
|
|
867
906
|
|
|
868
907
|
def alternates_netloc(self):
|
|
869
908
|
"""Return netlocs describing both base and stacked archives."""
|
|
870
|
-
netloc_m = re.match(
|
|
909
|
+
netloc_m = re.match(
|
|
910
|
+
r"(?P<base>v.*)\.archive\.(?P<country>\w+)", self.netloc
|
|
911
|
+
)
|
|
871
912
|
return [
|
|
872
|
-
|
|
873
|
-
|
|
913
|
+
"{base:s}.archive-legacy.{country:s}".format(
|
|
914
|
+
**netloc_m.groupdict()
|
|
915
|
+
),
|
|
916
|
+
"{base:s}.stacked-archive-legacy.{country:s}".format(
|
|
917
|
+
**netloc_m.groupdict()
|
|
918
|
+
),
|
|
874
919
|
]
|
|
875
920
|
|
|
876
921
|
def alternates_fpextras(self):
|
|
@@ -883,26 +928,28 @@ class _VortexCacheBaseStore(CacheStore, _VortexStackedStorageMixin):
|
|
|
883
928
|
|
|
884
929
|
_abstract = True
|
|
885
930
|
_footprint = dict(
|
|
886
|
-
info
|
|
887
|
-
attr
|
|
888
|
-
scheme
|
|
889
|
-
values
|
|
931
|
+
info="VORTEX cache access",
|
|
932
|
+
attr=dict(
|
|
933
|
+
scheme=dict(
|
|
934
|
+
values=["vortex"],
|
|
890
935
|
),
|
|
891
|
-
headdir
|
|
892
|
-
default
|
|
893
|
-
outcast
|
|
936
|
+
headdir=dict(
|
|
937
|
+
default="",
|
|
938
|
+
outcast=[
|
|
939
|
+
"xp",
|
|
940
|
+
],
|
|
894
941
|
),
|
|
895
|
-
rtouch
|
|
896
|
-
default
|
|
942
|
+
rtouch=dict(
|
|
943
|
+
default=True,
|
|
897
944
|
),
|
|
898
|
-
rtouchskip
|
|
899
|
-
default
|
|
945
|
+
rtouchskip=dict(
|
|
946
|
+
default=3,
|
|
900
947
|
),
|
|
901
|
-
)
|
|
948
|
+
),
|
|
902
949
|
)
|
|
903
950
|
|
|
904
951
|
def __init__(self, *args, **kw):
|
|
905
|
-
logger.debug(
|
|
952
|
+
logger.debug("Vortex cache store init %s", self.__class__)
|
|
906
953
|
del self.cache
|
|
907
954
|
super().__init__(*args, **kw)
|
|
908
955
|
|
|
@@ -939,45 +986,56 @@ class VortexCacheMtStore(_VortexCacheBaseStore):
|
|
|
939
986
|
"""Some kind of MTOOL cache for VORTEX experiments."""
|
|
940
987
|
|
|
941
988
|
_footprint = dict(
|
|
942
|
-
info
|
|
943
|
-
attr
|
|
944
|
-
netloc
|
|
945
|
-
values
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
989
|
+
info="VORTEX MTOOL like Cache access",
|
|
990
|
+
attr=dict(
|
|
991
|
+
netloc=dict(
|
|
992
|
+
values=[
|
|
993
|
+
"{:s}.{:s}cache-mt.fr".format(v, s)
|
|
994
|
+
for v in ("vortex", "vortex-free", "vsop")
|
|
995
|
+
for s in ("", "stacked-")
|
|
996
|
+
]
|
|
950
997
|
),
|
|
951
|
-
)
|
|
998
|
+
),
|
|
952
999
|
)
|
|
953
1000
|
|
|
1001
|
+
def __init__(self, *args, **kw):
|
|
1002
|
+
super().__init__(*args, **kw)
|
|
1003
|
+
self.location = get_cache_location()
|
|
1004
|
+
|
|
954
1005
|
|
|
955
|
-
# TODO Not sure this class is needed anymore
|
|
956
1006
|
class VortexCacheOp2ResearchStore(_VortexCacheBaseStore):
|
|
957
1007
|
"""The DSI/OP VORTEX cache where researchers can get the freshest data."""
|
|
958
1008
|
|
|
959
1009
|
_footprint = dict(
|
|
960
|
-
info
|
|
961
|
-
attr
|
|
962
|
-
netloc
|
|
963
|
-
values
|
|
964
|
-
|
|
965
|
-
for s in (
|
|
1010
|
+
info="VORTEX Mtool cache access",
|
|
1011
|
+
attr=dict(
|
|
1012
|
+
netloc=dict(
|
|
1013
|
+
values=[
|
|
1014
|
+
"vsop.{:s}cache-op2r.fr".format(s)
|
|
1015
|
+
for s in ("", "stacked-")
|
|
966
1016
|
],
|
|
967
1017
|
),
|
|
968
|
-
|
|
969
|
-
default
|
|
1018
|
+
readonly=dict(
|
|
1019
|
+
default=True,
|
|
970
1020
|
),
|
|
971
|
-
|
|
972
|
-
default = True,
|
|
973
|
-
)
|
|
974
|
-
)
|
|
1021
|
+
),
|
|
975
1022
|
)
|
|
976
1023
|
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
1024
|
+
def __init__(self, *args, **kw):
|
|
1025
|
+
super().__init__(*args, **kw)
|
|
1026
|
+
try:
|
|
1027
|
+
cachepath = config.from_config(
|
|
1028
|
+
section="data-tree",
|
|
1029
|
+
key="op_rootdir",
|
|
1030
|
+
)
|
|
1031
|
+
except config.ConfigurationError as e:
|
|
1032
|
+
logger.error(
|
|
1033
|
+
"Cannot use special experiment cache without providing",
|
|
1034
|
+
"cache location",
|
|
1035
|
+
)
|
|
1036
|
+
raise e
|
|
1037
|
+
|
|
1038
|
+
self.location = os.path.join(cachepath, "vortex")
|
|
981
1039
|
|
|
982
1040
|
|
|
983
1041
|
class _AbstractVortexCacheMultiStore(MultiStore):
|
|
@@ -985,33 +1043,49 @@ class _AbstractVortexCacheMultiStore(MultiStore):
|
|
|
985
1043
|
|
|
986
1044
|
_abstract = True
|
|
987
1045
|
_footprint = dict(
|
|
988
|
-
info
|
|
989
|
-
attr
|
|
990
|
-
scheme
|
|
991
|
-
values
|
|
1046
|
+
info="VORTEX cache access",
|
|
1047
|
+
attr=dict(
|
|
1048
|
+
scheme=dict(
|
|
1049
|
+
values=["vortex"],
|
|
992
1050
|
),
|
|
993
|
-
refillstore
|
|
994
|
-
default
|
|
995
|
-
)
|
|
996
|
-
)
|
|
1051
|
+
refillstore=dict(
|
|
1052
|
+
default=False,
|
|
1053
|
+
),
|
|
1054
|
+
),
|
|
997
1055
|
)
|
|
998
1056
|
|
|
999
1057
|
def filtered_readable_openedstores(self, remote):
|
|
1000
1058
|
"""Deals with stacked stores that are not always active."""
|
|
1001
|
-
ostores = [
|
|
1059
|
+
ostores = [
|
|
1060
|
+
self.openedstores[0],
|
|
1061
|
+
]
|
|
1002
1062
|
# TODO is the call to cache.allow_reads still required without
|
|
1003
1063
|
# marketplace stores?
|
|
1004
|
-
ostores.extend(
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1064
|
+
ostores.extend(
|
|
1065
|
+
[
|
|
1066
|
+
sto
|
|
1067
|
+
for sto in self.openedstores[1:]
|
|
1068
|
+
if (
|
|
1069
|
+
(not sto.stackedstore or "stackpath" in remote["query"])
|
|
1070
|
+
and sto.cache.allow_reads(remote["path"])
|
|
1071
|
+
)
|
|
1072
|
+
]
|
|
1073
|
+
)
|
|
1008
1074
|
return ostores
|
|
1009
1075
|
|
|
1010
1076
|
def filtered_writeable_openedstores(self, remote):
|
|
1011
1077
|
"""never writes into stack stores."""
|
|
1012
|
-
ostores = [
|
|
1013
|
-
|
|
1014
|
-
|
|
1078
|
+
ostores = [
|
|
1079
|
+
self.openedstores[0],
|
|
1080
|
+
]
|
|
1081
|
+
ostores.extend(
|
|
1082
|
+
[
|
|
1083
|
+
sto
|
|
1084
|
+
for sto in self.openedstores[1:]
|
|
1085
|
+
if not sto.stackedstore
|
|
1086
|
+
and sto.cache.allow_writes(remote["path"])
|
|
1087
|
+
]
|
|
1088
|
+
)
|
|
1015
1089
|
return ostores
|
|
1016
1090
|
|
|
1017
1091
|
|
|
@@ -1019,18 +1093,27 @@ class VortexCacheStore(_AbstractVortexCacheMultiStore):
|
|
|
1019
1093
|
"""The go to store for data cached by VORTEX R&D experiments."""
|
|
1020
1094
|
|
|
1021
1095
|
_footprint = dict(
|
|
1022
|
-
attr
|
|
1023
|
-
netloc
|
|
1024
|
-
values
|
|
1096
|
+
attr=dict(
|
|
1097
|
+
netloc=dict(
|
|
1098
|
+
values=[
|
|
1099
|
+
"vortex.cache.fr",
|
|
1100
|
+
"vortex-free.cache.fr",
|
|
1101
|
+
],
|
|
1025
1102
|
),
|
|
1026
1103
|
)
|
|
1027
1104
|
)
|
|
1028
1105
|
|
|
1029
1106
|
def alternates_netloc(self):
|
|
1030
1107
|
"""For Non-Op users, Op caches may be accessed in read-only mode."""
|
|
1031
|
-
netloc_m = re.match(
|
|
1032
|
-
|
|
1033
|
-
|
|
1108
|
+
netloc_m = re.match(
|
|
1109
|
+
r"(?P<base>vortex.*)\.cache\.(?P<country>\w+)", self.netloc
|
|
1110
|
+
)
|
|
1111
|
+
mt_netloc = "{base:s}.cache-mt.{country:s}".format(
|
|
1112
|
+
**netloc_m.groupdict()
|
|
1113
|
+
)
|
|
1114
|
+
s_mt_netloc = "{base:s}.stacked-cache-mt.{country:s}".format(
|
|
1115
|
+
**netloc_m.groupdict()
|
|
1116
|
+
)
|
|
1034
1117
|
return [mt_netloc, s_mt_netloc]
|
|
1035
1118
|
|
|
1036
1119
|
|
|
@@ -1042,35 +1125,36 @@ class VortexVsopCacheStore(_AbstractVortexCacheMultiStore):
|
|
|
1042
1125
|
"""
|
|
1043
1126
|
|
|
1044
1127
|
_footprint = dict(
|
|
1045
|
-
info
|
|
1046
|
-
attr
|
|
1047
|
-
netloc
|
|
1048
|
-
values
|
|
1128
|
+
info="VORTEX vsop magic cache access",
|
|
1129
|
+
attr=dict(
|
|
1130
|
+
netloc=dict(
|
|
1131
|
+
values=[
|
|
1132
|
+
"vsop.cache.fr",
|
|
1133
|
+
],
|
|
1049
1134
|
),
|
|
1050
|
-
glovekind
|
|
1051
|
-
optional
|
|
1052
|
-
default
|
|
1135
|
+
glovekind=dict(
|
|
1136
|
+
optional=True,
|
|
1137
|
+
default="[glove::realkind]",
|
|
1053
1138
|
),
|
|
1054
|
-
)
|
|
1139
|
+
),
|
|
1055
1140
|
)
|
|
1056
1141
|
|
|
1057
1142
|
def alternates_netloc(self):
|
|
1058
1143
|
"""For Non-Op users, Op caches may be accessed in read-only mode."""
|
|
1059
1144
|
todo = [
|
|
1060
|
-
|
|
1061
|
-
|
|
1145
|
+
"vsop.cache-mt.fr",
|
|
1146
|
+
"vsop.stacked-cache-mt.fr",
|
|
1062
1147
|
]
|
|
1063
1148
|
|
|
1064
1149
|
# Only set up op2r cache if the associated filepath
|
|
1065
1150
|
# is configured
|
|
1066
|
-
if (
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
section="data-tree", key="op_rootdir",
|
|
1070
|
-
)
|
|
1151
|
+
if (self.glovekind != "opuser") and config.is_defined(
|
|
1152
|
+
section="data-tree",
|
|
1153
|
+
key="op_rootdir",
|
|
1071
1154
|
):
|
|
1072
1155
|
todo += [
|
|
1073
|
-
|
|
1156
|
+
"vsop.cache-op2r.fr",
|
|
1157
|
+
"vsop.stacked-cache-op2r.fr",
|
|
1074
1158
|
]
|
|
1075
1159
|
return todo
|
|
1076
1160
|
|
|
@@ -1080,30 +1164,44 @@ class _AbstractVortexStackMultiStore(MultiStore):
|
|
|
1080
1164
|
|
|
1081
1165
|
_abstract = True
|
|
1082
1166
|
_footprint = dict(
|
|
1083
|
-
info
|
|
1084
|
-
attr
|
|
1085
|
-
scheme
|
|
1086
|
-
values
|
|
1167
|
+
info="VORTEX stack access",
|
|
1168
|
+
attr=dict(
|
|
1169
|
+
scheme=dict(
|
|
1170
|
+
values=["vortex"],
|
|
1087
1171
|
),
|
|
1088
|
-
refillstore
|
|
1089
|
-
default
|
|
1090
|
-
)
|
|
1091
|
-
)
|
|
1172
|
+
refillstore=dict(
|
|
1173
|
+
default=False,
|
|
1174
|
+
),
|
|
1175
|
+
),
|
|
1092
1176
|
)
|
|
1093
1177
|
|
|
1094
1178
|
# TODO is this still needed without marketplace stores?
|
|
1095
1179
|
def filtered_readable_openedstores(self, remote):
|
|
1096
1180
|
"""Deals with marketplace stores that are not always active."""
|
|
1097
|
-
ostores = [
|
|
1098
|
-
|
|
1099
|
-
|
|
1181
|
+
ostores = [
|
|
1182
|
+
self.openedstores[0],
|
|
1183
|
+
]
|
|
1184
|
+
ostores.extend(
|
|
1185
|
+
[
|
|
1186
|
+
sto
|
|
1187
|
+
for sto in self.openedstores[1:]
|
|
1188
|
+
if sto.cache.allow_reads(remote["path"])
|
|
1189
|
+
]
|
|
1190
|
+
)
|
|
1100
1191
|
return ostores
|
|
1101
1192
|
|
|
1102
1193
|
def filtered_writeable_openedstores(self, remote):
|
|
1103
1194
|
"""Deals with marketplace stores that are not always active."""
|
|
1104
|
-
ostores = [
|
|
1105
|
-
|
|
1106
|
-
|
|
1195
|
+
ostores = [
|
|
1196
|
+
self.openedstores[0],
|
|
1197
|
+
]
|
|
1198
|
+
ostores.extend(
|
|
1199
|
+
[
|
|
1200
|
+
sto
|
|
1201
|
+
for sto in self.openedstores[1:]
|
|
1202
|
+
if sto.cache.allow_writes(remote["path"])
|
|
1203
|
+
]
|
|
1204
|
+
)
|
|
1107
1205
|
return ostores
|
|
1108
1206
|
|
|
1109
1207
|
|
|
@@ -1111,18 +1209,22 @@ class VortexStackStore(_AbstractVortexStackMultiStore):
|
|
|
1111
1209
|
"""Store intended to read and write data into VORTEX R&D stacks."""
|
|
1112
1210
|
|
|
1113
1211
|
_footprint = dict(
|
|
1114
|
-
info
|
|
1115
|
-
attr
|
|
1116
|
-
netloc
|
|
1117
|
-
values
|
|
1212
|
+
info="VORTEX stack access",
|
|
1213
|
+
attr=dict(
|
|
1214
|
+
netloc=dict(
|
|
1215
|
+
values=["vortex.stack.fr", "vortex-free.stack.fr"],
|
|
1118
1216
|
),
|
|
1119
|
-
)
|
|
1217
|
+
),
|
|
1120
1218
|
)
|
|
1121
1219
|
|
|
1122
1220
|
def alternates_netloc(self):
|
|
1123
1221
|
"""Go through the various stacked stores."""
|
|
1124
|
-
netloc_m = re.match(
|
|
1125
|
-
|
|
1222
|
+
netloc_m = re.match(
|
|
1223
|
+
r"(?P<base>vortex.*)\.stack\.(?P<country>\w+)", self.netloc
|
|
1224
|
+
)
|
|
1225
|
+
s_mt_netloc = "{base:s}.stacked-cache-mt.{country:s}".format(
|
|
1226
|
+
**netloc_m.groupdict()
|
|
1227
|
+
)
|
|
1126
1228
|
return [s_mt_netloc]
|
|
1127
1229
|
|
|
1128
1230
|
|
|
@@ -1130,22 +1232,24 @@ class VortexVsopStackStore(_AbstractVortexStackMultiStore):
|
|
|
1130
1232
|
"""Store intended to read and write data into VORTEX R&D stacks."""
|
|
1131
1233
|
|
|
1132
1234
|
_footprint = dict(
|
|
1133
|
-
info
|
|
1134
|
-
attr
|
|
1135
|
-
netloc
|
|
1136
|
-
values
|
|
1235
|
+
info="VORTEX stack access",
|
|
1236
|
+
attr=dict(
|
|
1237
|
+
netloc=dict(
|
|
1238
|
+
values=["vsop.stack.fr"],
|
|
1137
1239
|
),
|
|
1138
|
-
glovekind
|
|
1139
|
-
optional
|
|
1140
|
-
default
|
|
1240
|
+
glovekind=dict(
|
|
1241
|
+
optional=True,
|
|
1242
|
+
default="[glove::realkind]",
|
|
1141
1243
|
),
|
|
1142
|
-
)
|
|
1244
|
+
),
|
|
1143
1245
|
)
|
|
1144
1246
|
|
|
1145
1247
|
def alternates_netloc(self):
|
|
1146
1248
|
"""For Non-Op users, Op caches may be accessed in read-only mode."""
|
|
1147
|
-
todo = [
|
|
1148
|
-
|
|
1249
|
+
todo = [
|
|
1250
|
+
"vsop.stacked-cache-mt.fr",
|
|
1251
|
+
]
|
|
1252
|
+
if self.glovekind != "opuser":
|
|
1149
1253
|
todo.append("vsop.stacked-cache-op2r.fr")
|
|
1150
1254
|
return todo
|
|
1151
1255
|
|
|
@@ -1157,24 +1261,30 @@ class VortexStoreLegacy(MultiStore):
|
|
|
1157
1261
|
"""
|
|
1158
1262
|
|
|
1159
1263
|
_footprint = dict(
|
|
1160
|
-
info=
|
|
1264
|
+
info="VORTEX multi access",
|
|
1161
1265
|
attr=dict(
|
|
1162
1266
|
scheme=dict(
|
|
1163
|
-
values=[
|
|
1267
|
+
values=["vortex"],
|
|
1164
1268
|
),
|
|
1165
1269
|
netloc=dict(
|
|
1166
|
-
values=[
|
|
1270
|
+
values=[
|
|
1271
|
+
"vortex.multi-legacy.fr",
|
|
1272
|
+
"vortex-free.multi-legacy.fr",
|
|
1273
|
+
"vsop.multi-legacy.fr",
|
|
1274
|
+
],
|
|
1167
1275
|
),
|
|
1168
1276
|
refillstore=dict(
|
|
1169
1277
|
default=True,
|
|
1170
|
-
)
|
|
1171
|
-
)
|
|
1278
|
+
),
|
|
1279
|
+
),
|
|
1172
1280
|
)
|
|
1173
1281
|
|
|
1174
1282
|
def alternates_netloc(self):
|
|
1175
1283
|
"""Tuple of alternates domains names, e.g. ``cache`` and ``archive``."""
|
|
1176
|
-
return [
|
|
1177
|
-
|
|
1284
|
+
return [
|
|
1285
|
+
self.netloc.firstname + d
|
|
1286
|
+
for d in (".cache.fr", ".archive-legacy.fr")
|
|
1287
|
+
]
|
|
1178
1288
|
|
|
1179
1289
|
|
|
1180
1290
|
class VortexStore(MultiStore):
|
|
@@ -1184,64 +1294,81 @@ class VortexStore(MultiStore):
|
|
|
1184
1294
|
"""
|
|
1185
1295
|
|
|
1186
1296
|
_footprint = dict(
|
|
1187
|
-
info
|
|
1188
|
-
attr
|
|
1189
|
-
scheme
|
|
1190
|
-
values
|
|
1297
|
+
info="VORTEX multi access",
|
|
1298
|
+
attr=dict(
|
|
1299
|
+
scheme=dict(
|
|
1300
|
+
values=["vortex"],
|
|
1191
1301
|
),
|
|
1192
|
-
netloc
|
|
1193
|
-
values
|
|
1302
|
+
netloc=dict(
|
|
1303
|
+
values=[
|
|
1304
|
+
"vortex.multi.fr",
|
|
1305
|
+
"vortex-free.multi.fr",
|
|
1306
|
+
"vsop.multi.fr",
|
|
1307
|
+
],
|
|
1194
1308
|
),
|
|
1195
|
-
refillstore
|
|
1196
|
-
|
|
1197
|
-
)
|
|
1198
|
-
)
|
|
1309
|
+
refillstore=dict(default=False),
|
|
1310
|
+
),
|
|
1199
1311
|
)
|
|
1200
1312
|
|
|
1201
1313
|
def filtered_readable_openedstores(self, remote):
|
|
1202
1314
|
"""Deals with stacked stores that are not always active."""
|
|
1203
|
-
ostores = [
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1315
|
+
ostores = [
|
|
1316
|
+
self.openedstores[0],
|
|
1317
|
+
]
|
|
1318
|
+
ostores.extend(
|
|
1319
|
+
[
|
|
1320
|
+
sto
|
|
1321
|
+
for sto in self.openedstores[1:]
|
|
1322
|
+
if not sto.stackedstore or "stackpath" in remote["query"]
|
|
1323
|
+
]
|
|
1324
|
+
)
|
|
1207
1325
|
return ostores
|
|
1208
1326
|
|
|
1209
1327
|
def alternates_netloc(self):
|
|
1210
1328
|
"""Tuple of alternates domains names, e.g. ``cache`` and ``archive``."""
|
|
1211
|
-
return [
|
|
1212
|
-
|
|
1329
|
+
return [
|
|
1330
|
+
self.netloc.firstname + d
|
|
1331
|
+
for d in (
|
|
1332
|
+
".multi-legacy.fr",
|
|
1333
|
+
".stacked-archive-smart.fr",
|
|
1334
|
+
)
|
|
1335
|
+
]
|
|
1213
1336
|
|
|
1214
1337
|
|
|
1215
1338
|
class PromiseCacheStore(VortexCacheMtStore):
|
|
1216
1339
|
"""Some kind of vortex cache for EXPECTED resources."""
|
|
1217
1340
|
|
|
1218
1341
|
_footprint = dict(
|
|
1219
|
-
info
|
|
1220
|
-
attr
|
|
1221
|
-
netloc
|
|
1222
|
-
values
|
|
1342
|
+
info="EXPECTED cache access",
|
|
1343
|
+
attr=dict(
|
|
1344
|
+
netloc=dict(
|
|
1345
|
+
values=["promise.cache.fr"],
|
|
1223
1346
|
),
|
|
1224
|
-
headdir
|
|
1225
|
-
default
|
|
1226
|
-
outcast
|
|
1347
|
+
headdir=dict(
|
|
1348
|
+
default="promise",
|
|
1349
|
+
outcast=["xp", "vortex"],
|
|
1227
1350
|
),
|
|
1228
|
-
)
|
|
1351
|
+
),
|
|
1229
1352
|
)
|
|
1230
1353
|
|
|
1231
1354
|
@staticmethod
|
|
1232
1355
|
def _add_default_options(options):
|
|
1233
1356
|
options_upd = options.copy()
|
|
1234
|
-
options_upd[
|
|
1235
|
-
options_upd[
|
|
1357
|
+
options_upd["fmt"] = "ascii" # Promises are always JSON files
|
|
1358
|
+
options_upd["intent"] = "in" # Promises are always read-only
|
|
1236
1359
|
return options_upd
|
|
1237
1360
|
|
|
1238
1361
|
def vortexget(self, remote, local, options):
|
|
1239
1362
|
"""Proxy to :meth:`incacheget`."""
|
|
1240
|
-
return super().vortexget(
|
|
1363
|
+
return super().vortexget(
|
|
1364
|
+
remote, local, self._add_default_options(options)
|
|
1365
|
+
)
|
|
1241
1366
|
|
|
1242
1367
|
def vortexput(self, local, remote, options):
|
|
1243
1368
|
"""Proxy to :meth:`incacheput`."""
|
|
1244
|
-
return super().vortexput(
|
|
1369
|
+
return super().vortexput(
|
|
1370
|
+
local, remote, self._add_default_options(options)
|
|
1371
|
+
)
|
|
1245
1372
|
|
|
1246
1373
|
def vortexdelete(self, remote, options):
|
|
1247
1374
|
"""Proxy to :meth:`incachedelete`."""
|
|
@@ -1252,20 +1379,24 @@ class VortexPromiseStore(PromiseStore):
|
|
|
1252
1379
|
"""Combine a Promise Store for expected resources and any VORTEX Store."""
|
|
1253
1380
|
|
|
1254
1381
|
_footprint = dict(
|
|
1255
|
-
info
|
|
1256
|
-
attr
|
|
1257
|
-
scheme
|
|
1258
|
-
values
|
|
1382
|
+
info="VORTEX promise store",
|
|
1383
|
+
attr=dict(
|
|
1384
|
+
scheme=dict(
|
|
1385
|
+
values=["xvortex"],
|
|
1259
1386
|
),
|
|
1260
1387
|
netloc=dict(
|
|
1261
|
-
outcast
|
|
1262
|
-
|
|
1388
|
+
outcast=[
|
|
1389
|
+
"vortex-demo.cache.fr",
|
|
1390
|
+
"vortex-demo.multi.fr",
|
|
1391
|
+
"vortex.testcache.fr",
|
|
1392
|
+
"vortex.testmulti.fr",
|
|
1393
|
+
],
|
|
1263
1394
|
),
|
|
1264
|
-
)
|
|
1395
|
+
),
|
|
1265
1396
|
)
|
|
1266
1397
|
|
|
1267
1398
|
|
|
1268
1399
|
# Activate the footprint's fasttrack on the stores collector
|
|
1269
|
-
fcollect = footprints.collectors.get(tag=
|
|
1270
|
-
fcollect.fasttrack = (
|
|
1400
|
+
fcollect = footprints.collectors.get(tag="store")
|
|
1401
|
+
fcollect.fasttrack = ("netloc", "scheme")
|
|
1271
1402
|
del fcollect
|