vortex-nwp 2.0.0b1__py3-none-any.whl → 2.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- vortex/__init__.py +75 -47
- vortex/algo/__init__.py +3 -2
- vortex/algo/components.py +944 -618
- vortex/algo/mpitools.py +802 -497
- vortex/algo/mpitools_templates/__init__.py +1 -0
- vortex/algo/serversynctools.py +34 -33
- vortex/config.py +19 -22
- vortex/data/__init__.py +9 -3
- vortex/data/abstractstores.py +593 -655
- vortex/data/containers.py +217 -162
- vortex/data/contents.py +65 -39
- vortex/data/executables.py +93 -102
- vortex/data/flow.py +40 -34
- vortex/data/geometries.py +228 -132
- vortex/data/handlers.py +436 -227
- vortex/data/outflow.py +15 -15
- vortex/data/providers.py +185 -163
- vortex/data/resources.py +48 -42
- vortex/data/stores.py +540 -417
- vortex/data/sync_templates/__init__.py +0 -0
- vortex/gloves.py +114 -87
- vortex/layout/__init__.py +1 -8
- vortex/layout/contexts.py +150 -84
- vortex/layout/dataflow.py +353 -202
- vortex/layout/monitor.py +264 -128
- vortex/nwp/__init__.py +5 -2
- vortex/nwp/algo/__init__.py +14 -5
- vortex/nwp/algo/assim.py +205 -151
- vortex/nwp/algo/clim.py +683 -517
- vortex/nwp/algo/coupling.py +447 -225
- vortex/nwp/algo/eda.py +437 -229
- vortex/nwp/algo/eps.py +403 -231
- vortex/nwp/algo/forecasts.py +416 -275
- vortex/nwp/algo/fpserver.py +683 -307
- vortex/nwp/algo/ifsnaming.py +205 -145
- vortex/nwp/algo/ifsroot.py +215 -122
- vortex/nwp/algo/monitoring.py +137 -76
- vortex/nwp/algo/mpitools.py +330 -190
- vortex/nwp/algo/odbtools.py +637 -353
- vortex/nwp/algo/oopsroot.py +454 -273
- vortex/nwp/algo/oopstests.py +90 -56
- vortex/nwp/algo/request.py +287 -206
- vortex/nwp/algo/stdpost.py +878 -522
- vortex/nwp/data/__init__.py +22 -4
- vortex/nwp/data/assim.py +125 -137
- vortex/nwp/data/boundaries.py +121 -68
- vortex/nwp/data/climfiles.py +193 -211
- vortex/nwp/data/configfiles.py +73 -69
- vortex/nwp/data/consts.py +426 -401
- vortex/nwp/data/ctpini.py +59 -43
- vortex/nwp/data/diagnostics.py +94 -66
- vortex/nwp/data/eda.py +50 -51
- vortex/nwp/data/eps.py +195 -146
- vortex/nwp/data/executables.py +440 -434
- vortex/nwp/data/fields.py +63 -48
- vortex/nwp/data/gridfiles.py +183 -111
- vortex/nwp/data/logs.py +250 -217
- vortex/nwp/data/modelstates.py +180 -151
- vortex/nwp/data/monitoring.py +72 -99
- vortex/nwp/data/namelists.py +254 -202
- vortex/nwp/data/obs.py +400 -308
- vortex/nwp/data/oopsexec.py +22 -20
- vortex/nwp/data/providers.py +90 -65
- vortex/nwp/data/query.py +71 -82
- vortex/nwp/data/stores.py +49 -36
- vortex/nwp/data/surfex.py +136 -137
- vortex/nwp/syntax/__init__.py +1 -1
- vortex/nwp/syntax/stdattrs.py +173 -111
- vortex/nwp/tools/__init__.py +2 -2
- vortex/nwp/tools/addons.py +22 -17
- vortex/nwp/tools/agt.py +24 -12
- vortex/nwp/tools/bdap.py +16 -5
- vortex/nwp/tools/bdcp.py +4 -1
- vortex/nwp/tools/bdm.py +3 -0
- vortex/nwp/tools/bdmp.py +14 -9
- vortex/nwp/tools/conftools.py +728 -378
- vortex/nwp/tools/drhook.py +12 -8
- vortex/nwp/tools/grib.py +65 -39
- vortex/nwp/tools/gribdiff.py +22 -17
- vortex/nwp/tools/ifstools.py +82 -42
- vortex/nwp/tools/igastuff.py +167 -143
- vortex/nwp/tools/mars.py +14 -2
- vortex/nwp/tools/odb.py +234 -125
- vortex/nwp/tools/partitioning.py +61 -37
- vortex/nwp/tools/satrad.py +27 -12
- vortex/nwp/util/async.py +83 -55
- vortex/nwp/util/beacon.py +10 -10
- vortex/nwp/util/diffpygram.py +174 -86
- vortex/nwp/util/ens.py +144 -63
- vortex/nwp/util/hooks.py +30 -19
- vortex/nwp/util/taskdeco.py +28 -24
- vortex/nwp/util/usepygram.py +278 -172
- vortex/nwp/util/usetnt.py +31 -17
- vortex/sessions.py +72 -39
- vortex/syntax/__init__.py +1 -1
- vortex/syntax/stdattrs.py +410 -171
- vortex/syntax/stddeco.py +31 -22
- vortex/toolbox.py +327 -192
- vortex/tools/__init__.py +11 -2
- vortex/tools/actions.py +110 -121
- vortex/tools/addons.py +111 -92
- vortex/tools/arm.py +42 -22
- vortex/tools/compression.py +72 -69
- vortex/tools/date.py +11 -4
- vortex/tools/delayedactions.py +242 -132
- vortex/tools/env.py +75 -47
- vortex/tools/folder.py +342 -171
- vortex/tools/grib.py +341 -162
- vortex/tools/lfi.py +423 -216
- vortex/tools/listings.py +109 -40
- vortex/tools/names.py +218 -156
- vortex/tools/net.py +655 -299
- vortex/tools/parallelism.py +93 -61
- vortex/tools/prestaging.py +55 -31
- vortex/tools/schedulers.py +172 -105
- vortex/tools/services.py +403 -334
- vortex/tools/storage.py +293 -358
- vortex/tools/surfex.py +24 -24
- vortex/tools/systems.py +1234 -643
- vortex/tools/targets.py +156 -100
- vortex/util/__init__.py +1 -1
- vortex/util/config.py +378 -327
- vortex/util/empty.py +2 -2
- vortex/util/helpers.py +56 -24
- vortex/util/introspection.py +18 -12
- vortex/util/iosponge.py +8 -4
- vortex/util/roles.py +4 -6
- vortex/util/storefunctions.py +39 -13
- vortex/util/structs.py +3 -3
- vortex/util/worker.py +29 -17
- vortex_nwp-2.1.0.dist-info/METADATA +67 -0
- vortex_nwp-2.1.0.dist-info/RECORD +144 -0
- {vortex_nwp-2.0.0b1.dist-info → vortex_nwp-2.1.0.dist-info}/WHEEL +1 -1
- vortex/layout/appconf.py +0 -109
- vortex/layout/jobs.py +0 -1276
- vortex/layout/nodes.py +0 -1424
- vortex/layout/subjobs.py +0 -464
- vortex_nwp-2.0.0b1.dist-info/METADATA +0 -50
- vortex_nwp-2.0.0b1.dist-info/RECORD +0 -146
- {vortex_nwp-2.0.0b1.dist-info → vortex_nwp-2.1.0.dist-info/licenses}/LICENSE +0 -0
- {vortex_nwp-2.0.0b1.dist-info → vortex_nwp-2.1.0.dist-info}/top_level.txt +0 -0
vortex/tools/lfi.py
CHANGED
|
@@ -29,7 +29,7 @@ logger = loggers.getLogger(__name__)
|
|
|
29
29
|
|
|
30
30
|
def use_in_shell(sh, **kw):
|
|
31
31
|
"""Extend current shell with the LFI interface defined by optional arguments."""
|
|
32
|
-
kw[
|
|
32
|
+
kw["shell"] = sh
|
|
33
33
|
return footprints.proxy.addon(**kw)
|
|
34
34
|
|
|
35
35
|
|
|
@@ -49,7 +49,9 @@ class LFI_Status:
|
|
|
49
49
|
self._result = result or list()
|
|
50
50
|
|
|
51
51
|
def __str__(self):
|
|
52
|
-
return
|
|
52
|
+
return "{:s} | rc={:d} result={:d}>".format(
|
|
53
|
+
repr(self).rstrip(">"), self.rc, len(self.result)
|
|
54
|
+
)
|
|
53
55
|
|
|
54
56
|
@property
|
|
55
57
|
def ok(self):
|
|
@@ -107,42 +109,44 @@ class LFI_Tool_Raw(addons.FtrawEnableAddon):
|
|
|
107
109
|
Interface to LFI commands through Perl wrappers.
|
|
108
110
|
"""
|
|
109
111
|
|
|
110
|
-
LFI_HNDL_SPEC =
|
|
112
|
+
LFI_HNDL_SPEC = ":1"
|
|
111
113
|
DR_HOOK_SILENT = 1
|
|
112
114
|
DR_HOOK_NOT_MPI = 1
|
|
113
115
|
DR_HOOK_ASSERT_MPI_INITIALIZED = 0
|
|
114
|
-
OMP_STACKSIZE =
|
|
115
|
-
KMP_STACKSIZE =
|
|
116
|
-
KMP_MONITOR_STACKSIZE =
|
|
116
|
+
OMP_STACKSIZE = "32M"
|
|
117
|
+
KMP_STACKSIZE = "32M"
|
|
118
|
+
KMP_MONITOR_STACKSIZE = "32M"
|
|
117
119
|
|
|
118
120
|
_footprint = dict(
|
|
119
|
-
info
|
|
120
|
-
attr
|
|
121
|
-
kind
|
|
122
|
-
values
|
|
121
|
+
info="Default LFI system interface (Perl)",
|
|
122
|
+
attr=dict(
|
|
123
|
+
kind=dict(
|
|
124
|
+
values=["lfi"],
|
|
123
125
|
),
|
|
124
|
-
cmd
|
|
125
|
-
alias
|
|
126
|
-
default
|
|
126
|
+
cmd=dict(
|
|
127
|
+
alias=("lficmd",),
|
|
128
|
+
default="lfitools",
|
|
127
129
|
),
|
|
128
|
-
path
|
|
129
|
-
alias
|
|
130
|
+
path=dict(
|
|
131
|
+
alias=("lfipath",),
|
|
130
132
|
),
|
|
131
|
-
warnpack
|
|
132
|
-
type
|
|
133
|
-
optional
|
|
134
|
-
default
|
|
133
|
+
warnpack=dict(
|
|
134
|
+
type=bool,
|
|
135
|
+
optional=True,
|
|
136
|
+
default=False,
|
|
135
137
|
),
|
|
136
|
-
wraplanguage
|
|
137
|
-
values
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
138
|
+
wraplanguage=dict(
|
|
139
|
+
values=[
|
|
140
|
+
"perl",
|
|
141
|
+
],
|
|
142
|
+
default="perl",
|
|
143
|
+
optional=True,
|
|
144
|
+
doc_visibility=footprints.doc.visibility.ADVANCED,
|
|
141
145
|
),
|
|
142
|
-
toolkind
|
|
143
|
-
default
|
|
146
|
+
toolkind=dict(
|
|
147
|
+
default="lfitools",
|
|
144
148
|
),
|
|
145
|
-
)
|
|
149
|
+
),
|
|
146
150
|
)
|
|
147
151
|
|
|
148
152
|
def __init__(self, *args, **kw):
|
|
@@ -154,7 +158,9 @@ class LFI_Tool_Raw(addons.FtrawEnableAddon):
|
|
|
154
158
|
def lfitools_path(self):
|
|
155
159
|
ctxtag = contexts.Context.tag_focus()
|
|
156
160
|
if ctxtag not in self._lfitools_path:
|
|
157
|
-
self._lfitools_path[ctxtag] = self.sh.path.join(
|
|
161
|
+
self._lfitools_path[ctxtag] = self.sh.path.join(
|
|
162
|
+
self.actual_path, self.actual_cmd
|
|
163
|
+
)
|
|
158
164
|
self.sh.xperm(self._lfitools_path[ctxtag], force=True)
|
|
159
165
|
return self._lfitools_path[ctxtag]
|
|
160
166
|
|
|
@@ -166,14 +172,20 @@ class LFI_Tool_Raw(addons.FtrawEnableAddon):
|
|
|
166
172
|
def _spawn_wrap(self, func, cmd, **kw):
|
|
167
173
|
"""Tube to set LFITOOLS env variable."""
|
|
168
174
|
self.env.LFITOOLS = self.lfitools_path
|
|
169
|
-
return super()._spawn_wrap(
|
|
175
|
+
return super()._spawn_wrap(
|
|
176
|
+
[
|
|
177
|
+
"lfi_" + func,
|
|
178
|
+
]
|
|
179
|
+
+ cmd,
|
|
180
|
+
**kw,
|
|
181
|
+
)
|
|
170
182
|
|
|
171
183
|
def is_xlfi(self, source):
|
|
172
184
|
"""Check if the given ``source`` is a multipart-lfi file."""
|
|
173
185
|
rc = False
|
|
174
186
|
if source and isinstance(source, str) and self.sh.path.exists(source):
|
|
175
|
-
with open(source,
|
|
176
|
-
rc = fd.read(8) == b
|
|
187
|
+
with open(source, "rb") as fd:
|
|
188
|
+
rc = fd.read(8) == b"LFI_ALTM"
|
|
177
189
|
return rc
|
|
178
190
|
|
|
179
191
|
def _std_table(self, lfifile, **kw):
|
|
@@ -184,13 +196,13 @@ class LFI_Tool_Raw(addons.FtrawEnableAddon):
|
|
|
184
196
|
* lfifile : lfi file name
|
|
185
197
|
|
|
186
198
|
"""
|
|
187
|
-
cmd = [
|
|
188
|
-
kw[
|
|
199
|
+
cmd = ["lfilist", lfifile]
|
|
200
|
+
kw["output"] = True
|
|
189
201
|
rawout = self._spawn(cmd, **kw)
|
|
190
202
|
return LFI_Status(
|
|
191
203
|
rc=0,
|
|
192
204
|
stdout=rawout,
|
|
193
|
-
result=[tuple(eval(x)[0]) for x in rawout if x.startswith(
|
|
205
|
+
result=[tuple(eval(x)[0]) for x in rawout if x.startswith("[")],
|
|
194
206
|
)
|
|
195
207
|
|
|
196
208
|
fa_table = lfi_table = _std_table
|
|
@@ -208,39 +220,39 @@ class LFI_Tool_Raw(addons.FtrawEnableAddon):
|
|
|
208
220
|
* skipfields : LFI fields not to be compared
|
|
209
221
|
* skiplength : Offset at which the comparison starts for each LFI fields
|
|
210
222
|
"""
|
|
211
|
-
cmd = [
|
|
223
|
+
cmd = ["lfidiff", "--lfi-file-1", lfi1, "--lfi-file-2", lfi2]
|
|
212
224
|
|
|
213
|
-
maxprint = kw.pop(
|
|
225
|
+
maxprint = kw.pop("maxprint", 2)
|
|
214
226
|
if maxprint:
|
|
215
|
-
cmd.extend([
|
|
227
|
+
cmd.extend(["--max-print-diff", str(maxprint)])
|
|
216
228
|
|
|
217
|
-
skipfields = kw.pop(
|
|
229
|
+
skipfields = kw.pop("skipfields", 0)
|
|
218
230
|
if skipfields:
|
|
219
|
-
cmd.extend([
|
|
231
|
+
cmd.extend(["--lfi-skip-fields", str(skipfields)])
|
|
220
232
|
|
|
221
|
-
skiplength = kw.pop(
|
|
233
|
+
skiplength = kw.pop("skiplength", 0)
|
|
222
234
|
if skiplength:
|
|
223
|
-
cmd.extend([
|
|
235
|
+
cmd.extend(["--lfi-skip-length", str(skiplength)])
|
|
224
236
|
|
|
225
|
-
kw[
|
|
237
|
+
kw["output"] = True
|
|
226
238
|
|
|
227
239
|
rawout = self._spawn(cmd, **kw)
|
|
228
|
-
fields = [
|
|
240
|
+
fields = [
|
|
241
|
+
tuple(x.split(" ", 2)[-2:])
|
|
242
|
+
for x in rawout
|
|
243
|
+
if re.match(r" (?:\!=|\+\+|\-\-)", x)
|
|
244
|
+
]
|
|
229
245
|
|
|
230
246
|
trfields = Tracker(
|
|
231
|
-
deleted=[x[1] for x in fields if x[0] ==
|
|
232
|
-
created=[x[1] for x in fields if x[0] ==
|
|
233
|
-
updated=[x[1] for x in fields if x[0] ==
|
|
247
|
+
deleted=[x[1] for x in fields if x[0] == "--"],
|
|
248
|
+
created=[x[1] for x in fields if x[0] == "++"],
|
|
249
|
+
updated=[x[1] for x in fields if x[0] == "!="],
|
|
234
250
|
)
|
|
235
251
|
|
|
236
252
|
stlist = self.lfi_table(lfi1, output=True)
|
|
237
253
|
trfields.unchanged = {x[0] for x in stlist.result} - set(trfields)
|
|
238
254
|
|
|
239
|
-
return LFI_Status(
|
|
240
|
-
rc=int(bool(fields)),
|
|
241
|
-
stdout=rawout,
|
|
242
|
-
result=trfields
|
|
243
|
-
)
|
|
255
|
+
return LFI_Status(rc=int(bool(fields)), stdout=rawout, result=trfields)
|
|
244
256
|
|
|
245
257
|
fa_diff = lfi_diff = _std_diff
|
|
246
258
|
|
|
@@ -253,17 +265,29 @@ class LFI_Tool_Raw(addons.FtrawEnableAddon):
|
|
|
253
265
|
* fa2 : The new empty file
|
|
254
266
|
|
|
255
267
|
"""
|
|
256
|
-
cmd = [
|
|
268
|
+
cmd = ["faempty", fa1, fa2]
|
|
257
269
|
self._spawn(cmd, **kw)
|
|
258
270
|
|
|
259
271
|
def _pack_stream(self, source):
|
|
260
|
-
return self._spawn_wrap(
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
272
|
+
return self._spawn_wrap(
|
|
273
|
+
"pack",
|
|
274
|
+
[
|
|
275
|
+
source,
|
|
276
|
+
],
|
|
277
|
+
output=False,
|
|
278
|
+
inpipe=True,
|
|
279
|
+
bufsize=8192,
|
|
280
|
+
)
|
|
264
281
|
|
|
265
282
|
def _packed_size(self, source):
|
|
266
|
-
out = self._spawn_wrap(
|
|
283
|
+
out = self._spawn_wrap(
|
|
284
|
+
"size",
|
|
285
|
+
[
|
|
286
|
+
source,
|
|
287
|
+
],
|
|
288
|
+
output=True,
|
|
289
|
+
inpipe=False,
|
|
290
|
+
)
|
|
267
291
|
try:
|
|
268
292
|
return int(out[0])
|
|
269
293
|
except ValueError:
|
|
@@ -273,14 +297,19 @@ class LFI_Tool_Raw(addons.FtrawEnableAddon):
|
|
|
273
297
|
def _std_forcepack(self, source, destination=None):
|
|
274
298
|
"""Returned a path to a packed data."""
|
|
275
299
|
if self.is_xlfi(source):
|
|
276
|
-
destination = (
|
|
277
|
-
|
|
300
|
+
destination = (
|
|
301
|
+
destination
|
|
302
|
+
if destination
|
|
303
|
+
else self.sh.safe_fileaddsuffix(source)
|
|
304
|
+
)
|
|
278
305
|
if not self.sh.path.exists(destination):
|
|
279
|
-
st = self._std_copy(
|
|
306
|
+
st = self._std_copy(
|
|
307
|
+
source=source, destination=destination, pack=True
|
|
308
|
+
)
|
|
280
309
|
if st:
|
|
281
310
|
return destination
|
|
282
311
|
else:
|
|
283
|
-
raise OSError(
|
|
312
|
+
raise OSError("XLFI packing failed")
|
|
284
313
|
else:
|
|
285
314
|
return destination
|
|
286
315
|
else:
|
|
@@ -288,8 +317,16 @@ class LFI_Tool_Raw(addons.FtrawEnableAddon):
|
|
|
288
317
|
|
|
289
318
|
fa_forcepack = lfi_forcepack = _std_forcepack
|
|
290
319
|
|
|
291
|
-
def _std_ftput(
|
|
292
|
-
|
|
320
|
+
def _std_ftput(
|
|
321
|
+
self,
|
|
322
|
+
source,
|
|
323
|
+
destination,
|
|
324
|
+
hostname=None,
|
|
325
|
+
logname=None,
|
|
326
|
+
port=DEFAULT_FTP_PORT,
|
|
327
|
+
cpipeline=None,
|
|
328
|
+
sync=False,
|
|
329
|
+
):
|
|
293
330
|
"""On the fly packing and ftp."""
|
|
294
331
|
if self.is_xlfi(source):
|
|
295
332
|
if cpipeline is not None:
|
|
@@ -301,64 +338,100 @@ class LFI_Tool_Raw(addons.FtrawEnableAddon):
|
|
|
301
338
|
if ftp:
|
|
302
339
|
packed_size = self._packed_size(source)
|
|
303
340
|
p = self._pack_stream(source)
|
|
304
|
-
st.rc = ftp.put(
|
|
341
|
+
st.rc = ftp.put(
|
|
342
|
+
p.stdout, destination, size=packed_size, exact=True
|
|
343
|
+
)
|
|
305
344
|
self.sh.pclose(p)
|
|
306
345
|
st.result = [destination]
|
|
307
346
|
st.stdout = [
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
347
|
+
"Connection time : {:f}".format(ftp.length),
|
|
348
|
+
"Packed source size: {:d}".format(packed_size),
|
|
349
|
+
"Actual target size: {:d}".format(ftp.size(destination)),
|
|
311
350
|
]
|
|
312
351
|
ftp.close()
|
|
313
352
|
else:
|
|
314
353
|
st.rc = 1
|
|
315
|
-
st.result = [
|
|
354
|
+
st.result = [
|
|
355
|
+
"Could not connect to " + hostname + " as user " + logname
|
|
356
|
+
]
|
|
316
357
|
return st
|
|
317
358
|
else:
|
|
318
|
-
return self.sh.ftput(
|
|
319
|
-
|
|
320
|
-
|
|
359
|
+
return self.sh.ftput(
|
|
360
|
+
source,
|
|
361
|
+
destination,
|
|
362
|
+
hostname=hostname,
|
|
363
|
+
logname=logname,
|
|
364
|
+
cpipeline=cpipeline,
|
|
365
|
+
port=port,
|
|
366
|
+
sync=sync,
|
|
367
|
+
)
|
|
321
368
|
|
|
322
|
-
def _std_rawftput(
|
|
323
|
-
|
|
369
|
+
def _std_rawftput(
|
|
370
|
+
self,
|
|
371
|
+
source,
|
|
372
|
+
destination,
|
|
373
|
+
hostname=None,
|
|
374
|
+
logname=None,
|
|
375
|
+
port=None,
|
|
376
|
+
cpipeline=None,
|
|
377
|
+
sync=False,
|
|
378
|
+
):
|
|
324
379
|
"""Use ftserv as much as possible."""
|
|
325
380
|
if self.is_xlfi(source):
|
|
326
381
|
if cpipeline is not None:
|
|
327
382
|
raise OSError("It's not allowed to compress xlfi files.")
|
|
328
383
|
if self.sh.ftraw and self.rawftshell is not None:
|
|
329
|
-
newsource = self.sh.copy2ftspool(source, fmt=
|
|
330
|
-
rc = self.sh.ftserv_put(
|
|
331
|
-
|
|
332
|
-
|
|
384
|
+
newsource = self.sh.copy2ftspool(source, fmt="lfi")
|
|
385
|
+
rc = self.sh.ftserv_put(
|
|
386
|
+
newsource,
|
|
387
|
+
destination,
|
|
388
|
+
hostname=hostname,
|
|
389
|
+
logname=logname,
|
|
390
|
+
port=port,
|
|
391
|
+
specialshell=self.rawftshell,
|
|
392
|
+
sync=sync,
|
|
393
|
+
)
|
|
333
394
|
self.sh.rm(newsource) # Delete the request file
|
|
334
395
|
return rc
|
|
335
396
|
else:
|
|
336
397
|
if port is None:
|
|
337
398
|
port = DEFAULT_FTP_PORT
|
|
338
|
-
return self._std_ftput(
|
|
339
|
-
|
|
399
|
+
return self._std_ftput(
|
|
400
|
+
source,
|
|
401
|
+
destination,
|
|
402
|
+
hostname,
|
|
403
|
+
logname,
|
|
404
|
+
port=port,
|
|
405
|
+
sync=sync,
|
|
406
|
+
)
|
|
340
407
|
else:
|
|
341
|
-
return self.sh.rawftput(
|
|
342
|
-
|
|
343
|
-
|
|
408
|
+
return self.sh.rawftput(
|
|
409
|
+
source,
|
|
410
|
+
destination,
|
|
411
|
+
hostname=hostname,
|
|
412
|
+
logname=logname,
|
|
413
|
+
port=port,
|
|
414
|
+
cpipeline=cpipeline,
|
|
415
|
+
sync=sync,
|
|
416
|
+
)
|
|
344
417
|
|
|
345
418
|
fa_ftput = lfi_ftput = _std_ftput
|
|
346
419
|
fa_rawftput = lfi_rawftput = _std_rawftput
|
|
347
420
|
|
|
348
|
-
def _std_prepare(self, source, destination, intent=
|
|
421
|
+
def _std_prepare(self, source, destination, intent="in"):
|
|
349
422
|
"""Check for the source and prepare the destination."""
|
|
350
|
-
if intent not in (
|
|
351
|
-
raise ValueError(
|
|
423
|
+
if intent not in ("in", "inout"):
|
|
424
|
+
raise ValueError("Incorrect value for intent ({})".format(intent))
|
|
352
425
|
st = LFI_Status()
|
|
353
426
|
if not self.sh.path.exists(source):
|
|
354
|
-
logger.error(
|
|
427
|
+
logger.error("Missing source %s", source)
|
|
355
428
|
st.rc = 2
|
|
356
|
-
st.stderr =
|
|
429
|
+
st.stderr = "No such source file or directory : [" + source + "]"
|
|
357
430
|
return st
|
|
358
431
|
if not self.sh.filecocoon(destination):
|
|
359
|
-
raise OSError(
|
|
432
|
+
raise OSError("Could not cocoon [" + destination + "]")
|
|
360
433
|
if not self.lfi_rm(destination):
|
|
361
|
-
raise OSError(
|
|
434
|
+
raise OSError("Could not clean destination [" + destination + "]")
|
|
362
435
|
return st
|
|
363
436
|
|
|
364
437
|
def _std_remove(self, *args):
|
|
@@ -366,16 +439,30 @@ class LFI_Tool_Raw(addons.FtrawEnableAddon):
|
|
|
366
439
|
st = LFI_Status(result=list())
|
|
367
440
|
for pname in args:
|
|
368
441
|
for objpath in self.sh.glob(pname):
|
|
369
|
-
rc = self._spawn_wrap(
|
|
370
|
-
|
|
371
|
-
|
|
442
|
+
rc = self._spawn_wrap(
|
|
443
|
+
"remove",
|
|
444
|
+
[
|
|
445
|
+
objpath,
|
|
446
|
+
],
|
|
447
|
+
output=False,
|
|
448
|
+
)
|
|
449
|
+
st.result.append(
|
|
450
|
+
dict(path=objpath, multi=self.is_xlfi(objpath), rc=rc)
|
|
451
|
+
)
|
|
372
452
|
st.rc = rc
|
|
373
453
|
return st
|
|
374
454
|
|
|
375
455
|
lfi_rm = lfi_remove = fa_rm = fa_remove = _std_remove
|
|
376
456
|
|
|
377
|
-
def _std_copy(
|
|
378
|
-
|
|
457
|
+
def _std_copy(
|
|
458
|
+
self,
|
|
459
|
+
source,
|
|
460
|
+
destination,
|
|
461
|
+
smartcp_threshold=0,
|
|
462
|
+
intent="in",
|
|
463
|
+
pack=False,
|
|
464
|
+
silent=False,
|
|
465
|
+
):
|
|
379
466
|
"""Extended copy for (possibly) multi lfi file."""
|
|
380
467
|
st = self._std_prepare(source, destination, intent)
|
|
381
468
|
if st.rc == 0:
|
|
@@ -385,23 +472,48 @@ class LFI_Tool_Raw(addons.FtrawEnableAddon):
|
|
|
385
472
|
actual_intent = intent
|
|
386
473
|
if ln_cross_users and not self.sh.allow_cross_users_links:
|
|
387
474
|
actual_pack = True
|
|
388
|
-
actual_intent =
|
|
475
|
+
actual_intent = "inout"
|
|
389
476
|
try:
|
|
390
|
-
st.rc = self._spawn_wrap(
|
|
391
|
-
|
|
392
|
-
|
|
477
|
+
st.rc = self._spawn_wrap(
|
|
478
|
+
"copy",
|
|
479
|
+
(
|
|
480
|
+
[
|
|
481
|
+
"-pack",
|
|
482
|
+
]
|
|
483
|
+
if actual_pack
|
|
484
|
+
else []
|
|
485
|
+
)
|
|
486
|
+
+ [
|
|
487
|
+
"-intent={}".format(actual_intent),
|
|
488
|
+
source,
|
|
489
|
+
destination,
|
|
490
|
+
],
|
|
491
|
+
output=False,
|
|
492
|
+
)
|
|
393
493
|
except systems.ExecutionError:
|
|
394
494
|
if self.sh.allow_cross_users_links and ln_cross_users:
|
|
395
495
|
# This is expected to fail if the fs.protected_hardlinks
|
|
396
496
|
# Linux kernel setting is 1.
|
|
397
|
-
logger.info(
|
|
497
|
+
logger.info(
|
|
498
|
+
"Force System's allow_cross_users_links to False"
|
|
499
|
+
)
|
|
398
500
|
self.sh.allow_cross_users_links = False
|
|
399
501
|
logger.info("Re-running the cp command on this LFI file")
|
|
400
|
-
st = self._std_copy(
|
|
502
|
+
st = self._std_copy(
|
|
503
|
+
source,
|
|
504
|
+
destination,
|
|
505
|
+
intent=intent,
|
|
506
|
+
pack=pack,
|
|
507
|
+
silent=silent,
|
|
508
|
+
)
|
|
401
509
|
else:
|
|
402
510
|
raise
|
|
403
511
|
else:
|
|
404
|
-
if
|
|
512
|
+
if (
|
|
513
|
+
ln_cross_users
|
|
514
|
+
and not self.sh.allow_cross_users_links
|
|
515
|
+
and intent == "in"
|
|
516
|
+
):
|
|
405
517
|
self.sh.readonly(destination)
|
|
406
518
|
return st
|
|
407
519
|
|
|
@@ -411,12 +523,23 @@ class LFI_Tool_Raw(addons.FtrawEnableAddon):
|
|
|
411
523
|
"""Extended mv for (possibly) multi lfi file."""
|
|
412
524
|
if self.is_xlfi(source):
|
|
413
525
|
if intent is None:
|
|
414
|
-
intent =
|
|
526
|
+
intent = (
|
|
527
|
+
"inout" if self.sh.access(source, self.sh.W_OK) else "in"
|
|
528
|
+
)
|
|
415
529
|
st = self._std_prepare(source, destination, intent)
|
|
416
530
|
if st.rc == 0:
|
|
417
|
-
st.rc = self._spawn_wrap(
|
|
418
|
-
|
|
419
|
-
|
|
531
|
+
st.rc = self._spawn_wrap(
|
|
532
|
+
"move",
|
|
533
|
+
(
|
|
534
|
+
[
|
|
535
|
+
"-pack",
|
|
536
|
+
]
|
|
537
|
+
if pack
|
|
538
|
+
else []
|
|
539
|
+
)
|
|
540
|
+
+ ["-intent={}".format(intent), source, destination],
|
|
541
|
+
output=False,
|
|
542
|
+
)
|
|
420
543
|
else:
|
|
421
544
|
st = LFI_Status()
|
|
422
545
|
st.rc = self.sh.mv(source, destination)
|
|
@@ -424,27 +547,35 @@ class LFI_Tool_Raw(addons.FtrawEnableAddon):
|
|
|
424
547
|
|
|
425
548
|
lfi_mv = lfi_move = fa_mv = fa_move = _std_move
|
|
426
549
|
|
|
427
|
-
def _std_scpput(
|
|
550
|
+
def _std_scpput(
|
|
551
|
+
self, source, destination, hostname, logname=None, cpipeline=None
|
|
552
|
+
):
|
|
428
553
|
"""On the fly packing and scp."""
|
|
429
554
|
if not self.is_xlfi(source):
|
|
430
|
-
rc = self.sh.scpput(
|
|
555
|
+
rc = self.sh.scpput(
|
|
556
|
+
source, destination, hostname, logname, cpipeline
|
|
557
|
+
)
|
|
431
558
|
else:
|
|
432
559
|
if cpipeline is not None:
|
|
433
560
|
raise OSError("It's not allowed to compress xlfi files.")
|
|
434
|
-
logname = self.sh.fix_ftuser(
|
|
561
|
+
logname = self.sh.fix_ftuser(
|
|
562
|
+
hostname, logname, fatal=False, defaults_to_user=False
|
|
563
|
+
)
|
|
435
564
|
ssh = self.sh.ssh(hostname, logname)
|
|
436
565
|
permissions = ssh.get_permissions(source)
|
|
437
566
|
# remove the .d companion directory (scp_stream removes the destination)
|
|
438
567
|
# go on on failure : the .d lingers on, but the lfi will be self-contained
|
|
439
|
-
ssh.remove(destination +
|
|
568
|
+
ssh.remove(destination + ".d")
|
|
440
569
|
p = self._pack_stream(source)
|
|
441
|
-
rc = ssh.scpput_stream(
|
|
570
|
+
rc = ssh.scpput_stream(
|
|
571
|
+
p.stdout, destination, permissions=permissions
|
|
572
|
+
)
|
|
442
573
|
self.sh.pclose(p)
|
|
443
574
|
return rc
|
|
444
575
|
|
|
445
576
|
fa_scpput = lfi_scpput = _std_scpput
|
|
446
577
|
|
|
447
|
-
@addons.require_external_addon(
|
|
578
|
+
@addons.require_external_addon("ecfs")
|
|
448
579
|
def _std_ecfsput(self, source, target, cpipeline=None, options=None):
|
|
449
580
|
"""
|
|
450
581
|
:param source: source file
|
|
@@ -459,29 +590,38 @@ class LFI_Tool_Raw(addons.FtrawEnableAddon):
|
|
|
459
590
|
psource = self.sh.safe_fileaddsuffix(source)
|
|
460
591
|
rc = LFI_Status()
|
|
461
592
|
try:
|
|
462
|
-
st = self._std_copy(
|
|
463
|
-
|
|
464
|
-
|
|
593
|
+
st = self._std_copy(
|
|
594
|
+
source=source, destination=psource, pack=True
|
|
595
|
+
)
|
|
465
596
|
rc = rc and st.rc
|
|
466
597
|
dict_args = dict()
|
|
467
598
|
if rc:
|
|
468
|
-
rc, dict_args = self.sh.ecfsput(
|
|
469
|
-
|
|
470
|
-
|
|
599
|
+
rc, dict_args = self.sh.ecfsput(
|
|
600
|
+
source=psource, target=target, options=options
|
|
601
|
+
)
|
|
471
602
|
finally:
|
|
472
603
|
self.sh.rm(psource)
|
|
473
604
|
return rc, dict_args
|
|
474
605
|
else:
|
|
475
|
-
return self.sh.ecfsput(
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
606
|
+
return self.sh.ecfsput(
|
|
607
|
+
source=source,
|
|
608
|
+
target=target,
|
|
609
|
+
options=options,
|
|
610
|
+
cpipeline=cpipeline,
|
|
611
|
+
)
|
|
479
612
|
|
|
480
613
|
fa_ecfsput = lfi_ecfsput = _std_ecfsput
|
|
481
614
|
|
|
482
|
-
@addons.require_external_addon(
|
|
483
|
-
def _std_ectransput(
|
|
484
|
-
|
|
615
|
+
@addons.require_external_addon("ectrans")
|
|
616
|
+
def _std_ectransput(
|
|
617
|
+
self,
|
|
618
|
+
source,
|
|
619
|
+
target,
|
|
620
|
+
gateway=None,
|
|
621
|
+
remote=None,
|
|
622
|
+
cpipeline=None,
|
|
623
|
+
sync=False,
|
|
624
|
+
):
|
|
485
625
|
"""
|
|
486
626
|
:param source: source file
|
|
487
627
|
:param target: target file
|
|
@@ -497,27 +637,31 @@ class LFI_Tool_Raw(addons.FtrawEnableAddon):
|
|
|
497
637
|
psource = self.sh.safe_fileaddsuffix(source)
|
|
498
638
|
rc = LFI_Status()
|
|
499
639
|
try:
|
|
500
|
-
st = self._std_copy(
|
|
501
|
-
|
|
502
|
-
|
|
640
|
+
st = self._std_copy(
|
|
641
|
+
source=source, destination=psource, pack=True
|
|
642
|
+
)
|
|
503
643
|
rc = rc and st.rc
|
|
504
644
|
dict_args = dict()
|
|
505
645
|
if rc:
|
|
506
|
-
rc, dict_args = self.sh.raw_ectransput(
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
646
|
+
rc, dict_args = self.sh.raw_ectransput(
|
|
647
|
+
source=psource,
|
|
648
|
+
target=target,
|
|
649
|
+
gateway=gateway,
|
|
650
|
+
remote=remote,
|
|
651
|
+
sync=sync,
|
|
652
|
+
)
|
|
511
653
|
finally:
|
|
512
654
|
self.sh.rm(psource)
|
|
513
655
|
return rc, dict_args
|
|
514
656
|
else:
|
|
515
|
-
return self.sh.ectransput(
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
657
|
+
return self.sh.ectransput(
|
|
658
|
+
source=source,
|
|
659
|
+
target=target,
|
|
660
|
+
gateway=gateway,
|
|
661
|
+
remote=remote,
|
|
662
|
+
cpipeline=cpipeline,
|
|
663
|
+
sync=sync,
|
|
664
|
+
)
|
|
521
665
|
|
|
522
666
|
fa_ectransput = lfi_ectransput = _std_ectransput
|
|
523
667
|
|
|
@@ -530,20 +674,23 @@ class LFI_Tool_Py(LFI_Tool_Raw):
|
|
|
530
674
|
"""
|
|
531
675
|
|
|
532
676
|
_footprint = dict(
|
|
533
|
-
info
|
|
534
|
-
attr
|
|
535
|
-
wraplanguage
|
|
536
|
-
values
|
|
677
|
+
info="Default LFI system interface (Python)",
|
|
678
|
+
attr=dict(
|
|
679
|
+
wraplanguage=dict(
|
|
680
|
+
values=[
|
|
681
|
+
"python",
|
|
682
|
+
],
|
|
537
683
|
),
|
|
538
|
-
)
|
|
684
|
+
),
|
|
539
685
|
)
|
|
540
686
|
|
|
541
687
|
def _pack_stream(self, source):
|
|
542
|
-
return self._spawn(
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
688
|
+
return self._spawn(
|
|
689
|
+
["lfi_alt_pack", "--lfi-file-in", source, "--lfi-file-out", "-"],
|
|
690
|
+
output=False,
|
|
691
|
+
inpipe=True,
|
|
692
|
+
bufsize=8192,
|
|
693
|
+
)
|
|
547
694
|
|
|
548
695
|
def _std_remove(self, *args):
|
|
549
696
|
"""Remove (possibly) multi lfi files."""
|
|
@@ -552,12 +699,14 @@ class LFI_Tool_Py(LFI_Tool_Raw):
|
|
|
552
699
|
for objpath in self.sh.glob(pname):
|
|
553
700
|
xlfi = self.is_xlfi(objpath)
|
|
554
701
|
if xlfi:
|
|
555
|
-
rc = self._spawn(
|
|
702
|
+
rc = self._spawn(
|
|
703
|
+
["lfi_alt_remv", "--lfi-file", objpath], output=False
|
|
704
|
+
)
|
|
556
705
|
else:
|
|
557
706
|
rc = self.sh.remove(objpath)
|
|
558
707
|
st.result.append(dict(path=objpath, multi=xlfi, rc=rc))
|
|
559
708
|
st.rc = rc
|
|
560
|
-
for dirpath in self.sh.glob(pname +
|
|
709
|
+
for dirpath in self.sh.glob(pname + ".d"):
|
|
561
710
|
if self.sh.path.exists(dirpath):
|
|
562
711
|
rc = self.sh.remove(dirpath)
|
|
563
712
|
st.result.append(dict(path=dirpath, multi=True, rc=rc))
|
|
@@ -568,29 +717,61 @@ class LFI_Tool_Py(LFI_Tool_Raw):
|
|
|
568
717
|
|
|
569
718
|
def _cp_pack_read(self, source, destination):
|
|
570
719
|
if self.warnpack:
|
|
571
|
-
logger.warning(
|
|
572
|
-
rc = self._spawn(
|
|
573
|
-
|
|
720
|
+
logger.warning("Suspicious packing <%s>", source)
|
|
721
|
+
rc = self._spawn(
|
|
722
|
+
[
|
|
723
|
+
"lfi_alt_pack",
|
|
724
|
+
"--lfi-file-in",
|
|
725
|
+
source,
|
|
726
|
+
"--lfi-file-out",
|
|
727
|
+
destination,
|
|
728
|
+
],
|
|
729
|
+
output=False,
|
|
730
|
+
)
|
|
574
731
|
self.sh.chmod(destination, 0o444)
|
|
575
732
|
return rc
|
|
576
733
|
|
|
577
734
|
def _cp_pack_write(self, source, destination):
|
|
578
735
|
if self.warnpack:
|
|
579
|
-
logger.warning(
|
|
580
|
-
rc = self._spawn(
|
|
581
|
-
|
|
736
|
+
logger.warning("Suspicious packing <%s>", source)
|
|
737
|
+
rc = self._spawn(
|
|
738
|
+
[
|
|
739
|
+
"lfi_alt_pack",
|
|
740
|
+
"--lfi-file-in",
|
|
741
|
+
source,
|
|
742
|
+
"--lfi-file-out",
|
|
743
|
+
destination,
|
|
744
|
+
],
|
|
745
|
+
output=False,
|
|
746
|
+
)
|
|
582
747
|
self.sh.chmod(destination, 0o644)
|
|
583
748
|
return rc
|
|
584
749
|
|
|
585
750
|
def _cp_copy_read(self, source, destination):
|
|
586
|
-
rc = self._spawn(
|
|
587
|
-
|
|
751
|
+
rc = self._spawn(
|
|
752
|
+
[
|
|
753
|
+
"lfi_alt_copy",
|
|
754
|
+
"--lfi-file-in",
|
|
755
|
+
source,
|
|
756
|
+
"--lfi-file-out",
|
|
757
|
+
destination,
|
|
758
|
+
],
|
|
759
|
+
output=False,
|
|
760
|
+
)
|
|
588
761
|
self.sh.chmod(destination, 0o444)
|
|
589
762
|
return rc
|
|
590
763
|
|
|
591
764
|
def _cp_copy_write(self, source, destination):
|
|
592
|
-
rc = self._spawn(
|
|
593
|
-
|
|
765
|
+
rc = self._spawn(
|
|
766
|
+
[
|
|
767
|
+
"lfi_alt_copy",
|
|
768
|
+
"--lfi-file-in",
|
|
769
|
+
source,
|
|
770
|
+
"--lfi-file-out",
|
|
771
|
+
destination,
|
|
772
|
+
],
|
|
773
|
+
output=False,
|
|
774
|
+
)
|
|
594
775
|
self.sh.chmod(destination, 0o644)
|
|
595
776
|
return rc
|
|
596
777
|
|
|
@@ -604,38 +785,55 @@ class LFI_Tool_Py(LFI_Tool_Raw):
|
|
|
604
785
|
_cp_nopack_fsko_read = _cp_pack_read
|
|
605
786
|
_cp_nopack_fsko_write = _cp_pack_write
|
|
606
787
|
|
|
607
|
-
def _multicpmethod(self, pack=False, intent=
|
|
608
|
-
return
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
788
|
+
def _multicpmethod(self, pack=False, intent="in", samefs=False):
|
|
789
|
+
return "_cp_{:s}_{:s}_{:s}".format(
|
|
790
|
+
"aspack" if pack else "nopack",
|
|
791
|
+
"fsok" if samefs else "fsko",
|
|
792
|
+
"read" if intent == "in" else "write",
|
|
612
793
|
)
|
|
613
794
|
|
|
614
|
-
def _std_copy(
|
|
615
|
-
|
|
795
|
+
def _std_copy(
|
|
796
|
+
self,
|
|
797
|
+
source,
|
|
798
|
+
destination,
|
|
799
|
+
smartcp_threshold=0,
|
|
800
|
+
intent="in",
|
|
801
|
+
pack=False,
|
|
802
|
+
silent=False,
|
|
803
|
+
):
|
|
616
804
|
"""Extended copy for (possibly) multi lfi file."""
|
|
617
805
|
st = LFI_Status()
|
|
618
806
|
if not self.sh.path.exists(source):
|
|
619
|
-
logger.error(
|
|
807
|
+
logger.error("Missing source %s", source)
|
|
620
808
|
st.rc = 2
|
|
621
|
-
st.stderr =
|
|
809
|
+
st.stderr = "No such source file or directory : [" + source + "]"
|
|
622
810
|
return st
|
|
623
811
|
if self.is_xlfi(source):
|
|
624
812
|
if not self.sh.filecocoon(destination):
|
|
625
|
-
raise OSError(
|
|
813
|
+
raise OSError("Could not cocoon [" + destination + "]")
|
|
626
814
|
if not self.lfi_rm(destination):
|
|
627
|
-
raise OSError(
|
|
628
|
-
|
|
815
|
+
raise OSError(
|
|
816
|
+
"Could not clean destination [" + destination + "]"
|
|
817
|
+
)
|
|
818
|
+
xcp = self._multicpmethod(
|
|
819
|
+
pack=pack,
|
|
820
|
+
intent=intent,
|
|
821
|
+
samefs=self.sh.is_samefs(source, destination),
|
|
822
|
+
)
|
|
629
823
|
actualcp = getattr(self, xcp, None)
|
|
630
824
|
if actualcp is None:
|
|
631
|
-
raise AttributeError(
|
|
825
|
+
raise AttributeError("No actual LFI cp command " + xcp)
|
|
632
826
|
else:
|
|
633
827
|
st.rc = actualcp(source, self.sh.path.realpath(destination))
|
|
634
828
|
else:
|
|
635
|
-
if intent ==
|
|
636
|
-
st.rc = self.sh.smartcp(
|
|
829
|
+
if intent == "in":
|
|
830
|
+
st.rc = self.sh.smartcp(
|
|
831
|
+
source, destination, smartcp_threshold=smartcp_threshold
|
|
832
|
+
)
|
|
637
833
|
else:
|
|
638
|
-
st.rc = self.sh.cp(
|
|
834
|
+
st.rc = self.sh.cp(
|
|
835
|
+
source, destination, smartcp_threshold=smartcp_threshold
|
|
836
|
+
)
|
|
639
837
|
return st
|
|
640
838
|
|
|
641
839
|
lfi_cp = lfi_copy = fa_cp = fa_copy = _std_copy
|
|
@@ -644,7 +842,9 @@ class LFI_Tool_Py(LFI_Tool_Raw):
|
|
|
644
842
|
"""Extended mv for (possibly) multi lfi file."""
|
|
645
843
|
if self.is_xlfi(source):
|
|
646
844
|
if intent is None:
|
|
647
|
-
intent =
|
|
845
|
+
intent = (
|
|
846
|
+
"inout" if self.sh.access(source, self.sh.W_OK) else "in"
|
|
847
|
+
)
|
|
648
848
|
st = self.lfi_cp(source, destination, intent=intent, pack=pack)
|
|
649
849
|
if st:
|
|
650
850
|
st = self.lfi_rm(source)
|
|
@@ -662,72 +862,79 @@ class IO_Poll(addons.Addon):
|
|
|
662
862
|
This addon is in charge of multi-file reshaping after IFS-ARPEGE execution.
|
|
663
863
|
"""
|
|
664
864
|
|
|
665
|
-
LFI_HNDL_SPEC =
|
|
865
|
+
LFI_HNDL_SPEC = ":1"
|
|
666
866
|
DR_HOOK_SILENT = 1
|
|
667
867
|
DR_HOOK_NOT_MPI = 1
|
|
668
868
|
DR_HOOK_ASSERT_MPI_INITIALIZED = 0
|
|
669
|
-
OMP_STACKSIZE =
|
|
670
|
-
KMP_STACKSIZE =
|
|
671
|
-
KMP_MONITOR_STACKSIZE =
|
|
869
|
+
OMP_STACKSIZE = "32M"
|
|
870
|
+
KMP_STACKSIZE = "32M"
|
|
871
|
+
KMP_MONITOR_STACKSIZE = "32M"
|
|
672
872
|
|
|
673
873
|
_footprint = dict(
|
|
674
|
-
info
|
|
675
|
-
attr
|
|
676
|
-
kind
|
|
677
|
-
values
|
|
678
|
-
remap
|
|
679
|
-
io_poll
|
|
680
|
-
)
|
|
874
|
+
info="Default io_poll system interface",
|
|
875
|
+
attr=dict(
|
|
876
|
+
kind=dict(
|
|
877
|
+
values=["iopoll", "io_poll"],
|
|
878
|
+
remap=dict(
|
|
879
|
+
io_poll="iopoll",
|
|
880
|
+
),
|
|
681
881
|
),
|
|
682
|
-
cfginfo
|
|
683
|
-
default
|
|
882
|
+
cfginfo=dict(
|
|
883
|
+
default="lfi",
|
|
684
884
|
),
|
|
685
|
-
cmd
|
|
686
|
-
alias
|
|
687
|
-
default
|
|
885
|
+
cmd=dict(
|
|
886
|
+
alias=("iopollcmd", "io_pollcmd", "io_poll_cmd"),
|
|
887
|
+
default="io_poll",
|
|
688
888
|
),
|
|
689
|
-
path
|
|
690
|
-
alias
|
|
889
|
+
path=dict(
|
|
890
|
+
alias=("iopollpath", "io_pollpath", "io_poll_path", "iopath"),
|
|
691
891
|
),
|
|
692
|
-
interpreter
|
|
693
|
-
values
|
|
694
|
-
remap
|
|
695
|
-
|
|
696
|
-
|
|
892
|
+
interpreter=dict(
|
|
893
|
+
values=["perl", "none", "None"],
|
|
894
|
+
remap=dict(
|
|
895
|
+
{
|
|
896
|
+
"None": "none",
|
|
897
|
+
}
|
|
898
|
+
),
|
|
899
|
+
default="perl",
|
|
900
|
+
optional=True,
|
|
697
901
|
),
|
|
698
|
-
toolkind
|
|
699
|
-
|
|
700
|
-
)
|
|
701
|
-
)
|
|
902
|
+
toolkind=dict(default="iopoll"),
|
|
903
|
+
),
|
|
702
904
|
)
|
|
703
905
|
|
|
704
906
|
def __init__(self, *args, **kw):
|
|
705
907
|
"""Abstract Addon initialisation."""
|
|
706
|
-
logger.debug(
|
|
908
|
+
logger.debug("IO_Poll init %s", self.__class__)
|
|
707
909
|
super().__init__(*args, **kw)
|
|
708
910
|
self._polled = set()
|
|
709
911
|
|
|
710
912
|
def _spawn(self, cmd, **kw):
|
|
711
913
|
"""Tube to set LFITOOLS env variable."""
|
|
712
|
-
if
|
|
713
|
-
active_lfi =
|
|
714
|
-
|
|
914
|
+
if "LFITOOLS" not in self.env:
|
|
915
|
+
active_lfi = LFI_Tool_Raw.in_shell(
|
|
916
|
+
self.sh
|
|
917
|
+
) or LFI_Tool_Py.in_shell(self.sh)
|
|
715
918
|
if active_lfi is None:
|
|
716
|
-
raise RuntimeError(
|
|
717
|
-
self.env.LFITOOLS =
|
|
919
|
+
raise RuntimeError("Could not find any active LFI Tool")
|
|
920
|
+
self.env.LFITOOLS = (
|
|
921
|
+
active_lfi.actual_path + "/" + active_lfi.actual_cmd
|
|
922
|
+
)
|
|
718
923
|
# Is there a need for an interpreter ?
|
|
719
|
-
if self.interpreter !=
|
|
720
|
-
kw[
|
|
924
|
+
if self.interpreter != "none":
|
|
925
|
+
kw["interpreter"] = self.interpreter
|
|
721
926
|
return super()._spawn(cmd, **kw)
|
|
722
927
|
|
|
723
928
|
def io_poll(self, prefix, nproc_io=None):
|
|
724
929
|
"""Do the actual job of polling files prefixed by ``prefix``."""
|
|
725
|
-
cmd = [
|
|
930
|
+
cmd = ["--prefix", prefix]
|
|
726
931
|
if nproc_io is None:
|
|
727
|
-
if not self.sh.path.exists(
|
|
728
|
-
raise OSError(
|
|
932
|
+
if not self.sh.path.exists("fort.4"):
|
|
933
|
+
raise OSError(
|
|
934
|
+
"The proc_io option or a fort.4 file should be provided."
|
|
935
|
+
)
|
|
729
936
|
else:
|
|
730
|
-
cmd.extend([
|
|
937
|
+
cmd.extend(["--nproc_io", str(nproc_io)])
|
|
731
938
|
|
|
732
939
|
# Catch the file processed
|
|
733
940
|
rawout = self._spawn(cmd)
|