DIRAC 9.0.0a63__py3-none-any.whl → 9.0.0a66__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- DIRAC/ConfigurationSystem/Client/LocalConfiguration.py +11 -8
- DIRAC/Core/Utilities/DErrno.py +5 -309
- DIRAC/Core/Utilities/ReturnValues.py +7 -252
- DIRAC/Core/scripts/dirac_agent.py +1 -1
- DIRAC/Interfaces/API/Dirac.py +8 -0
- DIRAC/Interfaces/API/Job.py +1 -0
- DIRAC/TransformationSystem/Agent/TransformationCleaningAgent.py +1 -1
- DIRAC/WorkloadManagementSystem/Agent/JobCleaningAgent.py +1 -1
- DIRAC/WorkloadManagementSystem/Agent/StalledJobAgent.py +1 -1
- DIRAC/WorkloadManagementSystem/Client/SandboxStoreClient.py +1 -1
- DIRAC/WorkloadManagementSystem/DB/StatusUtils.py +125 -0
- DIRAC/WorkloadManagementSystem/DB/tests/Test_StatusUtils.py +28 -0
- DIRAC/WorkloadManagementSystem/Service/JobManagerHandler.py +1 -1
- DIRAC/WorkloadManagementSystem/Utilities/jobAdministration.py +0 -123
- DIRAC/__init__.py +55 -54
- {dirac-9.0.0a63.dist-info → dirac-9.0.0a66.dist-info}/METADATA +2 -1
- {dirac-9.0.0a63.dist-info → dirac-9.0.0a66.dist-info}/RECORD +21 -20
- DIRAC/WorkloadManagementSystem/Utilities/test/Test_JobAdministration.py +0 -28
- {dirac-9.0.0a63.dist-info → dirac-9.0.0a66.dist-info}/WHEEL +0 -0
- {dirac-9.0.0a63.dist-info → dirac-9.0.0a66.dist-info}/entry_points.txt +0 -0
- {dirac-9.0.0a63.dist-info → dirac-9.0.0a66.dist-info}/licenses/LICENSE +0 -0
- {dirac-9.0.0a63.dist-info → dirac-9.0.0a66.dist-info}/top_level.txt +0 -0
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
"""
|
|
2
|
-
|
|
1
|
+
"""This is the guy that parses and interprets the local configuration options."""
|
|
2
|
+
|
|
3
3
|
import re
|
|
4
4
|
import os
|
|
5
5
|
import sys
|
|
@@ -286,16 +286,18 @@ class LocalConfiguration:
|
|
|
286
286
|
gLogger.exception()
|
|
287
287
|
return S_ERROR(str(e))
|
|
288
288
|
|
|
289
|
-
def initialize(self, *, returnErrors=False):
|
|
289
|
+
def initialize(self, *, returnErrors=False, requireSuccessfulSync=False):
|
|
290
290
|
"""Entrypoint used by :py:class:`DIRAC.initialize`
|
|
291
291
|
|
|
292
|
+
:param requireSuccessfulSync: fails if syncing with the remote did not work
|
|
293
|
+
|
|
292
294
|
TODO: This is currently a hack that returns a list of errors for so it
|
|
293
295
|
can be used by ``__addUserDataToConfiguration``. This entire module
|
|
294
296
|
should be refactored and simplified with ``Script.parseCommandLine``.
|
|
295
297
|
"""
|
|
296
298
|
errorsList = self.__loadCFGFiles()
|
|
297
299
|
if gConfigurationData.getServers():
|
|
298
|
-
retVal = self.syncRemoteConfiguration()
|
|
300
|
+
retVal = self.syncRemoteConfiguration(strict=requireSuccessfulSync)
|
|
299
301
|
if not retVal["OK"]:
|
|
300
302
|
return retVal
|
|
301
303
|
else:
|
|
@@ -321,7 +323,7 @@ class LocalConfiguration:
|
|
|
321
323
|
gLogger.showHeaders(True)
|
|
322
324
|
gLogger.enableLogsFromExternalLibs()
|
|
323
325
|
|
|
324
|
-
def loadUserData(self):
|
|
326
|
+
def loadUserData(self, requireSuccessfulSync=False):
|
|
325
327
|
"""
|
|
326
328
|
This is the magic method that reads the command line and processes it
|
|
327
329
|
It is used by the Script Base class and the dirac-service and dirac-agent scripts
|
|
@@ -329,6 +331,7 @@ class LocalConfiguration:
|
|
|
329
331
|
- any additional switches to be processed
|
|
330
332
|
- mandatory and default configuration configuration options must be defined.
|
|
331
333
|
|
|
334
|
+
:param requireSuccessfulSync: if True, will fail if the sync with remote server failed
|
|
332
335
|
"""
|
|
333
336
|
if self.initialized:
|
|
334
337
|
return S_OK()
|
|
@@ -336,7 +339,7 @@ class LocalConfiguration:
|
|
|
336
339
|
try:
|
|
337
340
|
if not self.isParsed:
|
|
338
341
|
self.__parseCommandLine() # Parse command line
|
|
339
|
-
retVal = self.__addUserDataToConfiguration()
|
|
342
|
+
retVal = self.__addUserDataToConfiguration(requireSuccessfulSync=requireSuccessfulSync)
|
|
340
343
|
|
|
341
344
|
for optionTuple in self.optionalEntryList:
|
|
342
345
|
optionPath = self.__getAbsolutePath(optionTuple[0])
|
|
@@ -496,8 +499,8 @@ class LocalConfiguration:
|
|
|
496
499
|
|
|
497
500
|
return errorsList
|
|
498
501
|
|
|
499
|
-
def __addUserDataToConfiguration(self):
|
|
500
|
-
retVal = self.initialize(returnErrors=True)
|
|
502
|
+
def __addUserDataToConfiguration(self, requireSuccessfulSync=False):
|
|
503
|
+
retVal = self.initialize(returnErrors=True, requireSuccessfulSync=requireSuccessfulSync)
|
|
501
504
|
if not retVal["OK"]:
|
|
502
505
|
return retVal
|
|
503
506
|
errorsList = retVal["Value"]
|
DIRAC/Core/Utilities/DErrno.py
CHANGED
|
@@ -37,320 +37,16 @@
|
|
|
37
37
|
DErrno.ERRX : ['An error message for ERRX that is specific to LHCb']}
|
|
38
38
|
|
|
39
39
|
"""
|
|
40
|
-
import os
|
|
41
40
|
import importlib
|
|
42
41
|
import sys
|
|
43
42
|
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
# To avoid conflict, the error numbers should be greater than 1000
|
|
47
|
-
# We decided to group the by range of 100 per system
|
|
48
|
-
|
|
49
|
-
# 1000: Generic
|
|
50
|
-
# 1100: Core
|
|
51
|
-
# 1200: Framework
|
|
52
|
-
# 1300: Interfaces
|
|
53
|
-
# 1400: Config
|
|
54
|
-
# 1500: WMS + Workflow
|
|
55
|
-
# 1600: DMS + StorageManagement
|
|
56
|
-
# 1700: RMS
|
|
57
|
-
# 1800: Accounting + Monitoring
|
|
58
|
-
# 1900: TS + Production
|
|
59
|
-
# 2000: Resources + RSS
|
|
60
|
-
|
|
61
|
-
# ## Generic (10XX)
|
|
62
|
-
# Python related: 0X
|
|
63
|
-
ETYPE = 1000
|
|
64
|
-
EIMPERR = 1001
|
|
65
|
-
ENOMETH = 1002
|
|
66
|
-
ECONF = 1003
|
|
67
|
-
EVALUE = 1004
|
|
68
|
-
EEEXCEPTION = 1005
|
|
69
|
-
# Files manipulation: 1X
|
|
70
|
-
ECTMPF = 1010
|
|
71
|
-
EOF = 1011
|
|
72
|
-
ERF = 1012
|
|
73
|
-
EWF = 1013
|
|
74
|
-
ESPF = 1014
|
|
75
|
-
|
|
76
|
-
# ## Core (11XX)
|
|
77
|
-
# Certificates and Proxy: 0X
|
|
78
|
-
EX509 = 1100
|
|
79
|
-
EPROXYFIND = 1101
|
|
80
|
-
EPROXYREAD = 1102
|
|
81
|
-
ECERTFIND = 1103
|
|
82
|
-
ECERTREAD = 1104
|
|
83
|
-
ENOCERT = 1105
|
|
84
|
-
ENOCHAIN = 1106
|
|
85
|
-
ENOPKEY = 1107
|
|
86
|
-
ENOGROUP = 1108
|
|
87
|
-
# DISET: 1X
|
|
88
|
-
EDISET = 1110
|
|
89
|
-
ENOAUTH = 1111
|
|
90
|
-
# 3rd party security: 2X
|
|
91
|
-
E3RDPARTY = 1120
|
|
92
|
-
EVOMS = 1121
|
|
93
|
-
# Databases : 3X
|
|
94
|
-
EDB = 1130
|
|
95
|
-
EMYSQL = 1131
|
|
96
|
-
ESQLA = 1132
|
|
97
|
-
# Message Queues: 4X
|
|
98
|
-
EMQUKN = 1140
|
|
99
|
-
EMQNOM = 1141
|
|
100
|
-
EMQCONN = 1142
|
|
101
|
-
# OpenSearch
|
|
102
|
-
EELNOFOUND = 1146
|
|
103
|
-
# Tokens
|
|
104
|
-
EATOKENFIND = 1150
|
|
105
|
-
EATOKENREAD = 1151
|
|
106
|
-
ETOKENTYPE = 1152
|
|
107
|
-
|
|
108
|
-
# config
|
|
109
|
-
ESECTION = 1400
|
|
110
|
-
|
|
111
|
-
# processes
|
|
112
|
-
EEZOMBIE = 1147
|
|
113
|
-
EENOPID = 1148
|
|
114
|
-
|
|
115
|
-
# ## WMS/Workflow
|
|
116
|
-
EWMSUKN = 1500
|
|
117
|
-
EWMSJDL = 1501
|
|
118
|
-
EWMSRESC = 1502
|
|
119
|
-
EWMSSUBM = 1503
|
|
120
|
-
EWMSJMAN = 1504
|
|
121
|
-
EWMSSTATUS = 1505
|
|
122
|
-
EWMSNOMATCH = 1510
|
|
123
|
-
EWMSPLTVER = 1511
|
|
124
|
-
EWMSNOPILOT = 1550
|
|
125
|
-
|
|
126
|
-
# ## DMS/StorageManagement (16XX)
|
|
127
|
-
EFILESIZE = 1601
|
|
128
|
-
EGFAL = 1602
|
|
129
|
-
EBADCKS = 1603
|
|
130
|
-
EFCERR = 1604
|
|
131
|
-
|
|
132
|
-
# ## RMS (17XX)
|
|
133
|
-
ERMSUKN = 1700
|
|
134
|
-
|
|
135
|
-
# ## TS (19XX)
|
|
136
|
-
ETSUKN = 1900
|
|
137
|
-
ETSDATA = 1901
|
|
138
|
-
|
|
139
|
-
# ## Resources and RSS (20XX)
|
|
140
|
-
ERESGEN = 2000
|
|
141
|
-
ERESUNA = 2001
|
|
142
|
-
ERESUNK = 2002
|
|
143
|
-
|
|
144
|
-
# This translates the integer number into the name of the variable
|
|
145
|
-
dErrorCode = {
|
|
146
|
-
# ## Generic (10XX)
|
|
147
|
-
# 100X: Python related
|
|
148
|
-
1000: "ETYPE",
|
|
149
|
-
1001: "EIMPERR",
|
|
150
|
-
1002: "ENOMETH",
|
|
151
|
-
1003: "ECONF",
|
|
152
|
-
1004: "EVALUE",
|
|
153
|
-
1005: "EEEXCEPTION",
|
|
154
|
-
# 101X: Files manipulation
|
|
155
|
-
1010: "ECTMPF",
|
|
156
|
-
1011: "EOF",
|
|
157
|
-
1012: "ERF",
|
|
158
|
-
1013: "EWF",
|
|
159
|
-
1014: "ESPF",
|
|
160
|
-
# ## Core
|
|
161
|
-
# 110X: Certificates and Proxy
|
|
162
|
-
1100: "EX509",
|
|
163
|
-
1101: "EPROXYFIND",
|
|
164
|
-
1102: "EPROXYREAD",
|
|
165
|
-
1103: "ECERTFIND",
|
|
166
|
-
1104: "ECERTREAD",
|
|
167
|
-
1105: "ENOCERT",
|
|
168
|
-
1106: "ENOCHAIN",
|
|
169
|
-
1107: "ENOPKEY",
|
|
170
|
-
1108: "ENOGROUP",
|
|
171
|
-
# 111X: DISET
|
|
172
|
-
1110: "EDISET",
|
|
173
|
-
1111: "ENOAUTH",
|
|
174
|
-
# 112X: 3rd party security
|
|
175
|
-
1120: "E3RDPARTY",
|
|
176
|
-
1121: "EVOMS",
|
|
177
|
-
# 113X: Databases
|
|
178
|
-
1130: "EDB",
|
|
179
|
-
1131: "EMYSQL",
|
|
180
|
-
1132: "ESQLA",
|
|
181
|
-
# 114X: Message Queues
|
|
182
|
-
1140: "EMQUKN",
|
|
183
|
-
1141: "EMQNOM",
|
|
184
|
-
1142: "EMQCONN",
|
|
185
|
-
# OpenSearch
|
|
186
|
-
1146: "EELNOFOUND",
|
|
187
|
-
# 115X: Tokens
|
|
188
|
-
1150: "EATOKENFIND",
|
|
189
|
-
1151: "EATOKENREAD",
|
|
190
|
-
1152: "ETOKENTYPE",
|
|
191
|
-
# Config
|
|
192
|
-
1400: "ESECTION",
|
|
193
|
-
# Processes
|
|
194
|
-
1147: "EEZOMBIE",
|
|
195
|
-
1148: "EENOPID",
|
|
196
|
-
# WMS/Workflow
|
|
197
|
-
1500: "EWMSUKN",
|
|
198
|
-
1501: "EWMSJDL",
|
|
199
|
-
1502: "EWMSRESC",
|
|
200
|
-
1503: "EWMSSUBM",
|
|
201
|
-
1504: "EWMSJMAN",
|
|
202
|
-
1505: "EWMSSTATUS",
|
|
203
|
-
1510: "EWMSNOMATCH",
|
|
204
|
-
1511: "EWMSPLTVER",
|
|
205
|
-
1550: "EWMSNOPILOT",
|
|
206
|
-
# DMS/StorageManagement
|
|
207
|
-
1601: "EFILESIZE",
|
|
208
|
-
1602: "EGFAL",
|
|
209
|
-
1603: "EBADCKS",
|
|
210
|
-
1604: "EFCERR",
|
|
211
|
-
# RMS
|
|
212
|
-
1700: "ERMSUKN",
|
|
213
|
-
# Resources and RSS
|
|
214
|
-
2000: "ERESGEN",
|
|
215
|
-
2001: "ERESUNA",
|
|
216
|
-
2002: "ERESUNK",
|
|
217
|
-
# TS
|
|
218
|
-
1900: "ETSUKN",
|
|
219
|
-
1901: "ETSDATA",
|
|
220
|
-
}
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
dStrError = { # Generic (10XX)
|
|
224
|
-
# 100X: Python related
|
|
225
|
-
ETYPE: "Object Type Error",
|
|
226
|
-
EIMPERR: "Failed to import library",
|
|
227
|
-
ENOMETH: "No such method or function",
|
|
228
|
-
ECONF: "Configuration error",
|
|
229
|
-
EVALUE: "Wrong value passed",
|
|
230
|
-
EEEXCEPTION: "runtime general exception",
|
|
231
|
-
# 101X: Files manipulation
|
|
232
|
-
ECTMPF: "Failed to create temporary file",
|
|
233
|
-
EOF: "Cannot open file",
|
|
234
|
-
ERF: "Cannot read from file",
|
|
235
|
-
EWF: "Cannot write to file",
|
|
236
|
-
ESPF: "Cannot set permissions to file",
|
|
237
|
-
# ## Core
|
|
238
|
-
# 110X: Certificates and Proxy
|
|
239
|
-
EX509: "Generic Error with X509",
|
|
240
|
-
EPROXYFIND: "Can't find proxy",
|
|
241
|
-
EPROXYREAD: "Can't read proxy",
|
|
242
|
-
ECERTFIND: "Can't find certificate",
|
|
243
|
-
ECERTREAD: "Can't read certificate",
|
|
244
|
-
ENOCERT: "No certificate loaded",
|
|
245
|
-
ENOCHAIN: "No chain loaded",
|
|
246
|
-
ENOPKEY: "No private key loaded",
|
|
247
|
-
ENOGROUP: "No DIRAC group",
|
|
248
|
-
# 111X: DISET
|
|
249
|
-
EDISET: "DISET Error",
|
|
250
|
-
ENOAUTH: "Unauthorized query",
|
|
251
|
-
# 112X: 3rd party security
|
|
252
|
-
E3RDPARTY: "3rd party security service error",
|
|
253
|
-
EVOMS: "VOMS Error",
|
|
254
|
-
# 113X: Databases
|
|
255
|
-
EDB: "Database Error",
|
|
256
|
-
EMYSQL: "MySQL Error",
|
|
257
|
-
ESQLA: "SQLAlchemy Error",
|
|
258
|
-
# 114X: Message Queues
|
|
259
|
-
EMQUKN: "Unknown MQ Error",
|
|
260
|
-
EMQNOM: "No messages",
|
|
261
|
-
EMQCONN: "MQ connection failure",
|
|
262
|
-
# 114X OpenSearch
|
|
263
|
-
EELNOFOUND: "Index not found",
|
|
264
|
-
# 115X: Tokens
|
|
265
|
-
EATOKENFIND: "Can't find a bearer access token.",
|
|
266
|
-
EATOKENREAD: "Can't read a bearer access token.",
|
|
267
|
-
ETOKENTYPE: "Unsupported access token type.",
|
|
268
|
-
# Config
|
|
269
|
-
ESECTION: "Section is not found",
|
|
270
|
-
# processes
|
|
271
|
-
EEZOMBIE: "Zombie process",
|
|
272
|
-
EENOPID: "No PID of process",
|
|
273
|
-
# WMS/Workflow
|
|
274
|
-
EWMSUKN: "Unknown WMS error",
|
|
275
|
-
EWMSJDL: "Invalid job description",
|
|
276
|
-
EWMSRESC: "Job to reschedule",
|
|
277
|
-
EWMSSUBM: "Job submission error",
|
|
278
|
-
EWMSJMAN: "Job management error",
|
|
279
|
-
EWMSSTATUS: "Job status error",
|
|
280
|
-
EWMSNOPILOT: "No pilots found",
|
|
281
|
-
EWMSPLTVER: "Pilot version does not match",
|
|
282
|
-
EWMSNOMATCH: "No match found",
|
|
283
|
-
# DMS/StorageManagement
|
|
284
|
-
EFILESIZE: "Bad file size",
|
|
285
|
-
EGFAL: "Error with the gfal call",
|
|
286
|
-
EBADCKS: "Bad checksum",
|
|
287
|
-
EFCERR: "FileCatalog error",
|
|
288
|
-
# RMS
|
|
289
|
-
ERMSUKN: "Unknown RMS error",
|
|
290
|
-
# Resources and RSS
|
|
291
|
-
ERESGEN: "Unknown Resource Failure",
|
|
292
|
-
ERESUNA: "Resource not available",
|
|
293
|
-
ERESUNK: "Unknown Resource",
|
|
294
|
-
# TS
|
|
295
|
-
ETSUKN: "Unknown Transformation System Error",
|
|
296
|
-
ETSDATA: "Invalid Input Data definition",
|
|
297
|
-
}
|
|
43
|
+
# Import all the stateless parts from DIRACCommon
|
|
44
|
+
from DIRACCommon.Utils.DErrno import * # noqa: F401, F403
|
|
298
45
|
|
|
46
|
+
from DIRAC.Core.Utilities.Extensions import extensionsByPriority
|
|
299
47
|
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
It completes it with the DIRAC specific errors.
|
|
303
|
-
"""
|
|
304
|
-
|
|
305
|
-
if code == 0:
|
|
306
|
-
return "Undefined error"
|
|
307
|
-
|
|
308
|
-
errMsg = f"Unknown error {code}"
|
|
309
|
-
|
|
310
|
-
try:
|
|
311
|
-
errMsg = dStrError[code]
|
|
312
|
-
except KeyError:
|
|
313
|
-
# It is not a DIRAC specific error, try the os one
|
|
314
|
-
try:
|
|
315
|
-
errMsg = os.strerror(code)
|
|
316
|
-
# On some system, os.strerror raises an exception with unknown code,
|
|
317
|
-
# on others, it returns a message...
|
|
318
|
-
except ValueError:
|
|
319
|
-
pass
|
|
320
|
-
|
|
321
|
-
return errMsg
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
def cmpError(inErr, candidate):
|
|
325
|
-
"""This function compares an error (in its old form (a string or dictionary) or in its int form
|
|
326
|
-
with a candidate error code.
|
|
327
|
-
|
|
328
|
-
:param inErr: a string, an integer, a S_ERROR dictionary
|
|
329
|
-
:type inErr: str or int or S_ERROR
|
|
330
|
-
:param int candidate: error code to compare with
|
|
331
|
-
|
|
332
|
-
:return: True or False
|
|
333
|
-
|
|
334
|
-
If an S_ERROR instance is passed, we compare the code with S_ERROR['Errno']
|
|
335
|
-
If it is a Integer, we do a direct comparison
|
|
336
|
-
If it is a String, we use strerror to check the error string
|
|
337
|
-
"""
|
|
338
|
-
|
|
339
|
-
if isinstance(inErr, str): # old style
|
|
340
|
-
# Compare error message strings
|
|
341
|
-
errMsg = strerror(candidate)
|
|
342
|
-
return errMsg in inErr
|
|
343
|
-
elif isinstance(inErr, dict): # if the S_ERROR structure is given
|
|
344
|
-
# Check if Errno defined in the dict
|
|
345
|
-
errorNumber = inErr.get("Errno")
|
|
346
|
-
if errorNumber:
|
|
347
|
-
return errorNumber == candidate
|
|
348
|
-
errMsg = strerror(candidate)
|
|
349
|
-
return errMsg in inErr.get("Message", "")
|
|
350
|
-
elif isinstance(inErr, int):
|
|
351
|
-
return inErr == candidate
|
|
352
|
-
else:
|
|
353
|
-
raise TypeError(f"Unknown input error type {type(inErr)}")
|
|
48
|
+
# compatErrorString is used by the extension mechanism but not in DIRACCommon
|
|
49
|
+
compatErrorString = {}
|
|
354
50
|
|
|
355
51
|
|
|
356
52
|
def includeExtensionErrors():
|
|
@@ -1,255 +1,10 @@
|
|
|
1
|
-
"""
|
|
2
|
-
DIRAC return dictionary
|
|
1
|
+
"""Backward compatibility wrapper - moved to DIRACCommon
|
|
3
2
|
|
|
4
|
-
|
|
3
|
+
This module has been moved to DIRACCommon.Utils.ReturnValues to avoid
|
|
4
|
+
circular dependencies and allow DiracX to use these utilities without
|
|
5
|
+
triggering DIRAC's global state initialization.
|
|
5
6
|
|
|
6
|
-
|
|
7
|
+
All exports are maintained for backward compatibility.
|
|
7
8
|
"""
|
|
8
|
-
from
|
|
9
|
-
|
|
10
|
-
import functools
|
|
11
|
-
import sys
|
|
12
|
-
import traceback
|
|
13
|
-
from types import TracebackType
|
|
14
|
-
from typing import Any, Callable, cast, Generic, Literal, overload, Type, TypeVar, Union
|
|
15
|
-
from typing_extensions import TypedDict, ParamSpec, NotRequired
|
|
16
|
-
|
|
17
|
-
from DIRAC.Core.Utilities.DErrno import strerror
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
T = TypeVar("T")
|
|
21
|
-
P = ParamSpec("P")
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
class DOKReturnType(TypedDict, Generic[T]):
|
|
25
|
-
"""used for typing the DIRAC return structure"""
|
|
26
|
-
|
|
27
|
-
OK: Literal[True]
|
|
28
|
-
Value: T
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
class DErrorReturnType(TypedDict):
|
|
32
|
-
"""used for typing the DIRAC return structure"""
|
|
33
|
-
|
|
34
|
-
OK: Literal[False]
|
|
35
|
-
Message: str
|
|
36
|
-
Errno: int
|
|
37
|
-
ExecInfo: NotRequired[tuple[type[BaseException], BaseException, TracebackType]]
|
|
38
|
-
CallStack: NotRequired[list[str]]
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
DReturnType = Union[DOKReturnType[T], DErrorReturnType]
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
def S_ERROR(*args: Any, **kwargs: Any) -> DErrorReturnType:
|
|
45
|
-
"""return value on error condition
|
|
46
|
-
|
|
47
|
-
Arguments are either Errno and ErrorMessage or just ErrorMessage fro backward compatibility
|
|
48
|
-
|
|
49
|
-
:param int errno: Error number
|
|
50
|
-
:param string message: Error message
|
|
51
|
-
:param list callStack: Manually override the CallStack attribute better performance
|
|
52
|
-
"""
|
|
53
|
-
callStack = kwargs.pop("callStack", None)
|
|
54
|
-
|
|
55
|
-
result: DErrorReturnType = {"OK": False, "Errno": 0, "Message": ""}
|
|
56
|
-
|
|
57
|
-
message = ""
|
|
58
|
-
if args:
|
|
59
|
-
if isinstance(args[0], int):
|
|
60
|
-
result["Errno"] = args[0]
|
|
61
|
-
if len(args) > 1:
|
|
62
|
-
message = args[1]
|
|
63
|
-
else:
|
|
64
|
-
message = args[0]
|
|
65
|
-
|
|
66
|
-
if result["Errno"]:
|
|
67
|
-
message = f"{strerror(result['Errno'])} ( {result['Errno']} : {message})"
|
|
68
|
-
result["Message"] = message
|
|
69
|
-
|
|
70
|
-
if callStack is None:
|
|
71
|
-
try:
|
|
72
|
-
callStack = traceback.format_stack()
|
|
73
|
-
callStack.pop()
|
|
74
|
-
except Exception:
|
|
75
|
-
callStack = []
|
|
76
|
-
|
|
77
|
-
result["CallStack"] = callStack
|
|
78
|
-
|
|
79
|
-
return result
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
# mypy doesn't understand default parameter values with generics so use overloads (python/mypy#3737)
|
|
83
|
-
@overload
|
|
84
|
-
def S_OK() -> DOKReturnType[None]:
|
|
85
|
-
...
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
@overload
|
|
89
|
-
def S_OK(value: T) -> DOKReturnType[T]:
|
|
90
|
-
...
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
def S_OK(value=None): # type: ignore
|
|
94
|
-
"""return value on success
|
|
95
|
-
|
|
96
|
-
:param value: value of the 'Value'
|
|
97
|
-
:return: dictionary { 'OK' : True, 'Value' : value }
|
|
98
|
-
"""
|
|
99
|
-
return {"OK": True, "Value": value}
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
def isReturnStructure(unk: Any) -> bool:
|
|
103
|
-
"""Check if value is an `S_OK`/`S_ERROR` object"""
|
|
104
|
-
if not isinstance(unk, dict):
|
|
105
|
-
return False
|
|
106
|
-
if "OK" not in unk:
|
|
107
|
-
return False
|
|
108
|
-
if unk["OK"]:
|
|
109
|
-
return "Value" in unk
|
|
110
|
-
else:
|
|
111
|
-
return "Message" in unk
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
def isSError(value: Any) -> bool:
|
|
115
|
-
"""Check if value is an `S_ERROR` object"""
|
|
116
|
-
if not isinstance(value, dict):
|
|
117
|
-
return False
|
|
118
|
-
if "OK" not in value:
|
|
119
|
-
return False
|
|
120
|
-
return "Message" in value
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
def reprReturnErrorStructure(struct: DErrorReturnType, full: bool = False) -> str:
|
|
124
|
-
errorNumber = struct.get("Errno", 0)
|
|
125
|
-
message = struct.get("Message", "")
|
|
126
|
-
if errorNumber:
|
|
127
|
-
reprStr = f"{strerror(errorNumber)} ( {errorNumber} : {message})"
|
|
128
|
-
else:
|
|
129
|
-
reprStr = message
|
|
130
|
-
|
|
131
|
-
if full:
|
|
132
|
-
callStack = struct.get("CallStack")
|
|
133
|
-
if callStack:
|
|
134
|
-
reprStr += "\n" + "".join(callStack)
|
|
135
|
-
|
|
136
|
-
return reprStr
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
def returnSingleResult(dictRes: DReturnType[Any]) -> DReturnType[Any]:
|
|
140
|
-
"""Transform the S_OK{Successful/Failed} dictionary convention into
|
|
141
|
-
an S_OK/S_ERROR return. To be used when a single returned entity
|
|
142
|
-
is expected from a generally bulk call.
|
|
143
|
-
|
|
144
|
-
:param dictRes: S_ERROR or S_OK( "Failed" : {}, "Successful" : {})
|
|
145
|
-
:returns: S_ERROR or S_OK(value)
|
|
146
|
-
|
|
147
|
-
The following rules are applied:
|
|
148
|
-
|
|
149
|
-
- if dictRes is an S_ERROR: returns it as is
|
|
150
|
-
- we start by looking at the Failed directory
|
|
151
|
-
- if there are several items in a dictionary, we return the first one
|
|
152
|
-
- if both dictionaries are empty, we return S_ERROR
|
|
153
|
-
- For an item in Failed, we return S_ERROR
|
|
154
|
-
- Far an item in Successful we return S_OK
|
|
155
|
-
|
|
156
|
-
Behavior examples (would be perfect unit test :-) )::
|
|
157
|
-
|
|
158
|
-
{'Message': 'Kaput', 'OK': False} -> {'Message': 'Kaput', 'OK': False}
|
|
159
|
-
{'OK': True, 'Value': {'Successful': {}, 'Failed': {'a': 1}}} -> {'Message': '1', 'OK': False}
|
|
160
|
-
{'OK': True, 'Value': {'Successful': {'b': 2}, 'Failed': {}}} -> {'OK': True, 'Value': 2}
|
|
161
|
-
{'OK': True, 'Value': {'Successful': {'b': 2}, 'Failed': {'a': 1}}} -> {'Message': '1', 'OK': False}
|
|
162
|
-
{'OK': True, 'Value': {'Successful': {'b': 2}, 'Failed': {'a': 1, 'c': 3}}} -> {'Message': '1', 'OK': False}
|
|
163
|
-
{'OK': True, 'Value': {'Successful': {'b': 2, 'd': 4}, 'Failed': {}}} -> {'OK': True, 'Value': 2}
|
|
164
|
-
{'OK': True, 'Value': {'Successful': {}, 'Failed': {}}} ->
|
|
165
|
-
{'Message': 'returnSingleResult: Failed and Successful dictionaries are empty', 'OK': False}
|
|
166
|
-
"""
|
|
167
|
-
# if S_ERROR was returned, we return it as well
|
|
168
|
-
if not dictRes["OK"]:
|
|
169
|
-
return dictRes
|
|
170
|
-
# if there is a Failed, we return the first one in an S_ERROR
|
|
171
|
-
if "Failed" in dictRes["Value"] and len(dictRes["Value"]["Failed"]):
|
|
172
|
-
errorMessage = list(dictRes["Value"]["Failed"].values())[0]
|
|
173
|
-
if isinstance(errorMessage, dict):
|
|
174
|
-
if isReturnStructure(errorMessage):
|
|
175
|
-
return cast(DErrorReturnType, errorMessage)
|
|
176
|
-
else:
|
|
177
|
-
return S_ERROR(str(errorMessage))
|
|
178
|
-
return S_ERROR(errorMessage)
|
|
179
|
-
# if there is a Successful, we return the first one in an S_OK
|
|
180
|
-
elif "Successful" in dictRes["Value"] and len(dictRes["Value"]["Successful"]):
|
|
181
|
-
return S_OK(list(dictRes["Value"]["Successful"].values())[0])
|
|
182
|
-
else:
|
|
183
|
-
return S_ERROR("returnSingleResult: Failed and Successful dictionaries are empty")
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
class SErrorException(Exception):
|
|
187
|
-
"""Exception class for use with `convertToReturnValue`"""
|
|
188
|
-
|
|
189
|
-
def __init__(self, result: DErrorReturnType | str, errCode: int = 0):
|
|
190
|
-
"""Create a new exception return value
|
|
191
|
-
|
|
192
|
-
If `result` is a `S_ERROR` return it directly else convert it to an
|
|
193
|
-
appropriate value using `S_ERROR(errCode, result)`.
|
|
194
|
-
|
|
195
|
-
:param result: The error to propagate
|
|
196
|
-
:param errCode: the error code to propagate
|
|
197
|
-
"""
|
|
198
|
-
if not isSError(result):
|
|
199
|
-
result = S_ERROR(errCode, result)
|
|
200
|
-
self.result = cast(DErrorReturnType, result)
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
def returnValueOrRaise(result: DReturnType[T], *, errorCode: int = 0) -> T:
|
|
204
|
-
"""Unwrap an S_OK/S_ERROR response into a value or Exception
|
|
205
|
-
|
|
206
|
-
This method assists with using exceptions in DIRAC code by raising
|
|
207
|
-
:exc:`SErrorException` if `result` is an error. This can then by propagated
|
|
208
|
-
automatically as an `S_ERROR` by wrapping public facing functions with
|
|
209
|
-
`@convertToReturnValue`.
|
|
210
|
-
|
|
211
|
-
:param result: Result of a DIRAC function which returns `S_OK`/`S_ERROR`
|
|
212
|
-
:returns: The value associated with the `S_OK` object
|
|
213
|
-
:raises: If `result["OK"]` is falsey the original exception is re-raised.
|
|
214
|
-
If no exception is known an :exc:`SErrorException` is raised.
|
|
215
|
-
"""
|
|
216
|
-
if not result["OK"]:
|
|
217
|
-
if "ExecInfo" in result:
|
|
218
|
-
raise result["ExecInfo"][0]
|
|
219
|
-
else:
|
|
220
|
-
raise SErrorException(result, errorCode)
|
|
221
|
-
return result["Value"]
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
def convertToReturnValue(func: Callable[P, T]) -> Callable[P, DReturnType[T]]:
|
|
225
|
-
"""Decorate a function to convert return values to `S_OK`/`S_ERROR`
|
|
226
|
-
|
|
227
|
-
If `func` returns, wrap the return value in `S_OK`.
|
|
228
|
-
If `func` raises :exc:`SErrorException`, return the associated `S_ERROR`
|
|
229
|
-
If `func` raises any other exception type, convert it to an `S_ERROR` object
|
|
230
|
-
|
|
231
|
-
:param result: The bare result of a function call
|
|
232
|
-
:returns: `S_OK`/`S_ERROR`
|
|
233
|
-
"""
|
|
234
|
-
|
|
235
|
-
@functools.wraps(func)
|
|
236
|
-
def wrapped(*args: P.args, **kwargs: P.kwargs) -> DReturnType[T]:
|
|
237
|
-
try:
|
|
238
|
-
value = func(*args, **kwargs)
|
|
239
|
-
except SErrorException as e:
|
|
240
|
-
return e.result
|
|
241
|
-
except Exception as e:
|
|
242
|
-
retval = S_ERROR(f"{repr(e)}: {e}")
|
|
243
|
-
# Replace CallStack with the one from the exception
|
|
244
|
-
# Use cast as mypy doesn't understand that sys.exc_info can't return None in an exception block
|
|
245
|
-
retval["ExecInfo"] = cast(tuple[type[BaseException], BaseException, TracebackType], sys.exc_info())
|
|
246
|
-
exc_type, exc_value, exc_tb = retval["ExecInfo"]
|
|
247
|
-
retval["CallStack"] = traceback.format_tb(exc_tb)
|
|
248
|
-
return retval
|
|
249
|
-
else:
|
|
250
|
-
return S_OK(value)
|
|
251
|
-
|
|
252
|
-
# functools will copy the annotations. Since we change the return type
|
|
253
|
-
# we have to update it
|
|
254
|
-
wrapped.__annotations__["return"] = DReturnType
|
|
255
|
-
return wrapped
|
|
9
|
+
# Re-export everything from DIRACCommon for backward compatibility
|
|
10
|
+
from DIRACCommon.Utils.ReturnValues import * # noqa: F401, F403
|
|
@@ -25,7 +25,7 @@ def main():
|
|
|
25
25
|
localCfg.setConfigurationForAgent(agentName)
|
|
26
26
|
localCfg.addDefaultEntry("/DIRAC/Security/UseServerCertificate", "yes")
|
|
27
27
|
localCfg.addDefaultEntry("LogColor", True)
|
|
28
|
-
resultDict = localCfg.loadUserData()
|
|
28
|
+
resultDict = localCfg.loadUserData(requireSuccessfulSync=True)
|
|
29
29
|
if not resultDict["OK"]:
|
|
30
30
|
gLogger.error("There were errors when loading configuration", resultDict["Message"])
|
|
31
31
|
sys.exit(1)
|
DIRAC/Interfaces/API/Dirac.py
CHANGED
|
@@ -422,6 +422,14 @@ class Dirac(API):
|
|
|
422
422
|
sandbox = [isFile.strip() for isFile in sandbox.split(",")]
|
|
423
423
|
for isFile in sandbox:
|
|
424
424
|
self.log.debug(f"Resolving Input Sandbox {isFile}")
|
|
425
|
+
# If the sandbox is already in the sandbox store, download it
|
|
426
|
+
if isFile.startswith("SB:"):
|
|
427
|
+
result = SandboxStoreClient(useCertificates=self.useCertificates).downloadSandbox(
|
|
428
|
+
isFile, destinationDir=os.getcwd()
|
|
429
|
+
)
|
|
430
|
+
if not result["OK"]:
|
|
431
|
+
return S_ERROR(f"Cannot download Input sandbox {isFile}: {result['Message']}")
|
|
432
|
+
continue
|
|
425
433
|
if isFile.lower().startswith("lfn:"): # isFile is an LFN
|
|
426
434
|
isFile = isFile[4:]
|
|
427
435
|
# Attempt to copy into job working directory, unless it is already there
|
DIRAC/Interfaces/API/Job.py
CHANGED
|
@@ -1105,6 +1105,7 @@ class Job(API):
|
|
|
1105
1105
|
uniqueInputSandbox = uniqueElements(finalInputSandbox.split(";"))
|
|
1106
1106
|
paramsDict["InputSandbox"]["value"] = ";".join(uniqueInputSandbox)
|
|
1107
1107
|
self.log.verbose(f"Final unique Input Sandbox {';'.join(uniqueInputSandbox)}")
|
|
1108
|
+
paramsDict["InputSandbox"]["type"] = "JDL"
|
|
1108
1109
|
else:
|
|
1109
1110
|
paramsDict["InputSandbox"] = {}
|
|
1110
1111
|
paramsDict["InputSandbox"]["value"] = extraFiles
|
|
@@ -37,7 +37,7 @@ from DIRAC.WorkloadManagementSystem.Service.JobPolicy import (
|
|
|
37
37
|
RIGHT_DELETE,
|
|
38
38
|
RIGHT_KILL,
|
|
39
39
|
)
|
|
40
|
-
from DIRAC.WorkloadManagementSystem.
|
|
40
|
+
from DIRAC.WorkloadManagementSystem.DB.StatusUtils import kill_delete_jobs
|
|
41
41
|
|
|
42
42
|
# # agent's name
|
|
43
43
|
AGENT_NAME = "Transformation/TransformationCleaningAgent"
|