DIRAC 9.0.0a54__py3-none-any.whl → 9.0.7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- DIRAC/AccountingSystem/Client/AccountingCLI.py +0 -140
- DIRAC/AccountingSystem/Client/DataStoreClient.py +0 -13
- DIRAC/AccountingSystem/Client/Types/BaseAccountingType.py +0 -7
- DIRAC/AccountingSystem/ConfigTemplate.cfg +0 -5
- DIRAC/AccountingSystem/Service/DataStoreHandler.py +0 -72
- DIRAC/ConfigurationSystem/Client/Helpers/CSGlobals.py +0 -9
- DIRAC/ConfigurationSystem/Client/Helpers/Registry.py +34 -32
- DIRAC/ConfigurationSystem/Client/Helpers/Resources.py +11 -43
- DIRAC/ConfigurationSystem/Client/Helpers/test/Test_Helpers.py +0 -16
- DIRAC/ConfigurationSystem/Client/LocalConfiguration.py +14 -8
- DIRAC/ConfigurationSystem/Client/PathFinder.py +47 -8
- DIRAC/ConfigurationSystem/Client/SyncPlugins/CERNLDAPSyncPlugin.py +4 -1
- DIRAC/ConfigurationSystem/Client/VOMS2CSSynchronizer.py +9 -2
- DIRAC/ConfigurationSystem/Client/test/Test_PathFinder.py +41 -1
- DIRAC/ConfigurationSystem/private/RefresherBase.py +4 -2
- DIRAC/Core/DISET/ServiceReactor.py +11 -3
- DIRAC/Core/DISET/private/BaseClient.py +1 -2
- DIRAC/Core/DISET/private/Transports/M2SSLTransport.py +9 -7
- DIRAC/Core/Security/DiracX.py +12 -7
- DIRAC/Core/Security/IAMService.py +4 -3
- DIRAC/Core/Security/ProxyInfo.py +9 -5
- DIRAC/Core/Security/test/test_diracx_token_from_pem.py +161 -0
- DIRAC/Core/Tornado/Client/ClientSelector.py +4 -1
- DIRAC/Core/Tornado/Server/TornadoService.py +1 -1
- DIRAC/Core/Utilities/ClassAd/ClassAdLight.py +4 -290
- DIRAC/Core/Utilities/DErrno.py +5 -309
- DIRAC/Core/Utilities/Extensions.py +10 -1
- DIRAC/Core/Utilities/Graphs/GraphData.py +1 -1
- DIRAC/Core/Utilities/JDL.py +1 -195
- DIRAC/Core/Utilities/List.py +1 -124
- DIRAC/Core/Utilities/MySQL.py +101 -97
- DIRAC/Core/Utilities/Os.py +32 -1
- DIRAC/Core/Utilities/Platform.py +2 -107
- DIRAC/Core/Utilities/ReturnValues.py +7 -252
- DIRAC/Core/Utilities/StateMachine.py +12 -178
- DIRAC/Core/Utilities/TimeUtilities.py +10 -253
- DIRAC/Core/Utilities/test/Test_JDL.py +0 -3
- DIRAC/Core/Utilities/test/Test_Profiler.py +20 -20
- DIRAC/Core/scripts/dirac_agent.py +1 -1
- DIRAC/Core/scripts/dirac_apptainer_exec.py +16 -7
- DIRAC/Core/scripts/dirac_platform.py +1 -92
- DIRAC/DataManagementSystem/Agent/FTS3Agent.py +8 -7
- DIRAC/DataManagementSystem/Agent/RequestOperations/RemoveFile.py +7 -6
- DIRAC/DataManagementSystem/Client/FTS3Job.py +71 -34
- DIRAC/DataManagementSystem/DB/FTS3DB.py +3 -0
- DIRAC/DataManagementSystem/DB/FileCatalogComponents/DatasetManager/DatasetManager.py +1 -1
- DIRAC/DataManagementSystem/Utilities/DMSHelpers.py +6 -2
- DIRAC/DataManagementSystem/scripts/dirac_dms_create_moving_request.py +2 -0
- DIRAC/DataManagementSystem/scripts/dirac_dms_protocol_matrix.py +0 -1
- DIRAC/FrameworkSystem/Client/ComponentInstaller.py +4 -2
- DIRAC/FrameworkSystem/DB/ProxyDB.py +9 -5
- DIRAC/FrameworkSystem/Utilities/TokenManagementUtilities.py +3 -2
- DIRAC/FrameworkSystem/Utilities/diracx.py +2 -74
- DIRAC/FrameworkSystem/private/authorization/AuthServer.py +2 -2
- DIRAC/FrameworkSystem/scripts/dirac_login.py +2 -2
- DIRAC/FrameworkSystem/scripts/dirac_proxy_init.py +1 -1
- DIRAC/Interfaces/API/Dirac.py +27 -13
- DIRAC/Interfaces/API/DiracAdmin.py +42 -7
- DIRAC/Interfaces/API/Job.py +1 -0
- DIRAC/Interfaces/scripts/dirac_admin_allow_site.py +7 -1
- DIRAC/Interfaces/scripts/dirac_admin_ban_site.py +7 -1
- DIRAC/Interfaces/scripts/dirac_wms_job_parameters.py +0 -1
- DIRAC/MonitoringSystem/Client/Types/WMSHistory.py +4 -0
- DIRAC/MonitoringSystem/Client/WebAppClient.py +26 -0
- DIRAC/MonitoringSystem/ConfigTemplate.cfg +9 -0
- DIRAC/MonitoringSystem/DB/MonitoringDB.py +6 -25
- DIRAC/MonitoringSystem/Service/MonitoringHandler.py +0 -33
- DIRAC/MonitoringSystem/Service/WebAppHandler.py +599 -0
- DIRAC/MonitoringSystem/private/MainReporter.py +0 -3
- DIRAC/ProductionSystem/scripts/dirac_prod_get_trans.py +2 -3
- DIRAC/RequestManagementSystem/Agent/RequestExecutingAgent.py +8 -6
- DIRAC/RequestManagementSystem/ConfigTemplate.cfg +6 -6
- DIRAC/RequestManagementSystem/DB/test/RMSTestScenari.py +2 -0
- DIRAC/ResourceStatusSystem/Client/SiteStatus.py +4 -2
- DIRAC/ResourceStatusSystem/Command/FreeDiskSpaceCommand.py +3 -1
- DIRAC/ResourceStatusSystem/Utilities/CSHelpers.py +2 -31
- DIRAC/ResourceStatusSystem/scripts/dirac_rss_set_status.py +18 -4
- DIRAC/Resources/Catalog/RucioFileCatalogClient.py +1 -1
- DIRAC/Resources/Computing/AREXComputingElement.py +19 -3
- DIRAC/Resources/Computing/BatchSystems/Condor.py +126 -108
- DIRAC/Resources/Computing/BatchSystems/SLURM.py +5 -1
- DIRAC/Resources/Computing/BatchSystems/test/Test_SLURM.py +46 -0
- DIRAC/Resources/Computing/HTCondorCEComputingElement.py +37 -43
- DIRAC/Resources/Computing/SingularityComputingElement.py +6 -1
- DIRAC/Resources/Computing/test/Test_HTCondorCEComputingElement.py +67 -49
- DIRAC/Resources/Computing/test/Test_PoolComputingElement.py +2 -1
- DIRAC/Resources/IdProvider/CheckInIdProvider.py +13 -0
- DIRAC/Resources/IdProvider/IdProviderFactory.py +11 -3
- DIRAC/Resources/Storage/StorageBase.py +4 -2
- DIRAC/Resources/Storage/StorageElement.py +4 -4
- DIRAC/TransformationSystem/Agent/TaskManagerAgentBase.py +10 -16
- DIRAC/TransformationSystem/Agent/TransformationAgent.py +22 -1
- DIRAC/TransformationSystem/Agent/TransformationCleaningAgent.py +15 -15
- DIRAC/TransformationSystem/Client/Transformation.py +2 -1
- DIRAC/TransformationSystem/Client/TransformationClient.py +0 -7
- DIRAC/TransformationSystem/Client/Utilities.py +9 -0
- DIRAC/TransformationSystem/Service/TransformationManagerHandler.py +0 -336
- DIRAC/TransformationSystem/Utilities/ReplicationCLIParameters.py +3 -3
- DIRAC/TransformationSystem/scripts/dirac_production_runjoblocal.py +2 -4
- DIRAC/TransformationSystem/test/Test_replicationTransformation.py +5 -6
- DIRAC/Workflow/Modules/test/Test_Modules.py +5 -0
- DIRAC/WorkloadManagementSystem/Agent/JobAgent.py +1 -5
- DIRAC/WorkloadManagementSystem/Agent/JobCleaningAgent.py +11 -7
- DIRAC/WorkloadManagementSystem/Agent/PilotSyncAgent.py +4 -3
- DIRAC/WorkloadManagementSystem/Agent/PushJobAgent.py +13 -13
- DIRAC/WorkloadManagementSystem/Agent/SiteDirector.py +10 -13
- DIRAC/WorkloadManagementSystem/Agent/StalledJobAgent.py +18 -51
- DIRAC/WorkloadManagementSystem/Agent/StatesAccountingAgent.py +41 -1
- DIRAC/WorkloadManagementSystem/Agent/test/Test_Agent_JobAgent.py +2 -0
- DIRAC/WorkloadManagementSystem/Agent/test/Test_Agent_JobCleaningAgent.py +7 -9
- DIRAC/WorkloadManagementSystem/Agent/test/Test_Agent_PushJobAgent.py +1 -0
- DIRAC/WorkloadManagementSystem/Agent/test/Test_Agent_SiteDirector.py +8 -2
- DIRAC/WorkloadManagementSystem/Agent/test/Test_Agent_StalledJobAgent.py +4 -5
- DIRAC/WorkloadManagementSystem/Client/DownloadInputData.py +7 -5
- DIRAC/WorkloadManagementSystem/Client/JobMonitoringClient.py +10 -11
- DIRAC/WorkloadManagementSystem/Client/JobState/JobManifest.py +32 -261
- DIRAC/WorkloadManagementSystem/Client/JobStateUpdateClient.py +3 -0
- DIRAC/WorkloadManagementSystem/Client/JobStatus.py +8 -152
- DIRAC/WorkloadManagementSystem/Client/SandboxStoreClient.py +25 -38
- DIRAC/WorkloadManagementSystem/Client/WMSClient.py +2 -3
- DIRAC/WorkloadManagementSystem/Client/test/Test_Client_DownloadInputData.py +29 -0
- DIRAC/WorkloadManagementSystem/ConfigTemplate.cfg +4 -8
- DIRAC/WorkloadManagementSystem/DB/JobDB.py +40 -69
- DIRAC/WorkloadManagementSystem/DB/JobDBUtils.py +18 -147
- DIRAC/WorkloadManagementSystem/DB/JobParametersDB.py +9 -9
- DIRAC/WorkloadManagementSystem/DB/PilotAgentsDB.py +3 -2
- DIRAC/WorkloadManagementSystem/DB/SandboxMetadataDB.py +28 -39
- DIRAC/WorkloadManagementSystem/DB/StatusUtils.py +125 -0
- DIRAC/WorkloadManagementSystem/DB/tests/Test_JobDB.py +1 -1
- DIRAC/WorkloadManagementSystem/DB/tests/Test_StatusUtils.py +28 -0
- DIRAC/WorkloadManagementSystem/Executor/JobSanity.py +3 -3
- DIRAC/WorkloadManagementSystem/FutureClient/JobStateUpdateClient.py +2 -14
- DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py +14 -9
- DIRAC/WorkloadManagementSystem/JobWrapper/test/Test_JobWrapper.py +36 -10
- DIRAC/WorkloadManagementSystem/JobWrapper/test/Test_JobWrapperTemplate.py +4 -0
- DIRAC/WorkloadManagementSystem/Service/JobManagerHandler.py +33 -154
- DIRAC/WorkloadManagementSystem/Service/JobMonitoringHandler.py +5 -323
- DIRAC/WorkloadManagementSystem/Service/JobStateUpdateHandler.py +0 -16
- DIRAC/WorkloadManagementSystem/Service/PilotManagerHandler.py +6 -102
- DIRAC/WorkloadManagementSystem/Service/SandboxStoreHandler.py +5 -51
- DIRAC/WorkloadManagementSystem/Service/WMSAdministratorHandler.py +16 -79
- DIRAC/WorkloadManagementSystem/Utilities/JobModel.py +28 -199
- DIRAC/WorkloadManagementSystem/Utilities/JobParameters.py +65 -3
- DIRAC/WorkloadManagementSystem/Utilities/JobStatusUtility.py +2 -64
- DIRAC/WorkloadManagementSystem/Utilities/ParametricJob.py +7 -171
- DIRAC/WorkloadManagementSystem/Utilities/PilotCStoJSONSynchronizer.py +73 -7
- DIRAC/WorkloadManagementSystem/Utilities/PilotWrapper.py +2 -0
- DIRAC/WorkloadManagementSystem/Utilities/RemoteRunner.py +16 -0
- DIRAC/WorkloadManagementSystem/Utilities/Utils.py +36 -1
- DIRAC/WorkloadManagementSystem/Utilities/jobAdministration.py +15 -0
- DIRAC/WorkloadManagementSystem/Utilities/test/Test_JobModel.py +1 -5
- DIRAC/WorkloadManagementSystem/Utilities/test/Test_ParametricJob.py +45 -128
- DIRAC/WorkloadManagementSystem/Utilities/test/Test_PilotWrapper.py +16 -0
- DIRAC/__init__.py +55 -54
- {dirac-9.0.0a54.dist-info → dirac-9.0.7.dist-info}/METADATA +6 -4
- {dirac-9.0.0a54.dist-info → dirac-9.0.7.dist-info}/RECORD +160 -160
- {dirac-9.0.0a54.dist-info → dirac-9.0.7.dist-info}/WHEEL +1 -1
- {dirac-9.0.0a54.dist-info → dirac-9.0.7.dist-info}/entry_points.txt +0 -3
- DIRAC/Core/Utilities/test/Test_List.py +0 -150
- DIRAC/Core/Utilities/test/Test_Time.py +0 -88
- DIRAC/TransformationSystem/scripts/dirac_transformation_archive.py +0 -30
- DIRAC/TransformationSystem/scripts/dirac_transformation_clean.py +0 -30
- DIRAC/TransformationSystem/scripts/dirac_transformation_remove_output.py +0 -30
- DIRAC/WorkloadManagementSystem/Utilities/test/Test_JobManager.py +0 -58
- {dirac-9.0.0a54.dist-info → dirac-9.0.7.dist-info}/licenses/LICENSE +0 -0
- {dirac-9.0.0a54.dist-info → dirac-9.0.7.dist-info}/top_level.txt +0 -0
|
@@ -1,266 +1,37 @@
|
|
|
1
|
-
from
|
|
1
|
+
from __future__ import annotations
|
|
2
2
|
|
|
3
|
-
from
|
|
4
|
-
from DIRAC.Core.Utilities import List
|
|
5
|
-
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
|
|
6
|
-
from DIRAC.Core.Utilities.JDL import loadJDLAsCFG, dumpCFGAsJDL
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
class JobManifest:
|
|
10
|
-
def __init__(self, manifest=""):
|
|
11
|
-
self.__manifest = CFG()
|
|
12
|
-
self.__dirty = False
|
|
13
|
-
self.__ops = False
|
|
14
|
-
if manifest:
|
|
15
|
-
result = self.load(manifest)
|
|
16
|
-
if not result["OK"]:
|
|
17
|
-
raise Exception(result["Message"])
|
|
18
|
-
|
|
19
|
-
def isDirty(self):
|
|
20
|
-
return self.__dirty
|
|
21
|
-
|
|
22
|
-
def setDirty(self):
|
|
23
|
-
self.__dirty = True
|
|
24
|
-
|
|
25
|
-
def clearDirty(self):
|
|
26
|
-
self.__dirty = False
|
|
27
|
-
|
|
28
|
-
def load(self, dataString):
|
|
29
|
-
"""
|
|
30
|
-
Auto discover format type based on [ .. ] of JDL
|
|
31
|
-
"""
|
|
32
|
-
dataString = dataString.strip()
|
|
33
|
-
if dataString[0] == "[" and dataString[-1] == "]":
|
|
34
|
-
return self.loadJDL(dataString)
|
|
35
|
-
else:
|
|
36
|
-
return self.loadCFG(dataString)
|
|
37
|
-
|
|
38
|
-
def loadJDL(self, jdlString):
|
|
39
|
-
"""
|
|
40
|
-
Load job manifest from JDL format
|
|
41
|
-
"""
|
|
42
|
-
result = loadJDLAsCFG(jdlString.strip())
|
|
43
|
-
if not result["OK"]:
|
|
44
|
-
self.__manifest = CFG()
|
|
45
|
-
return result
|
|
46
|
-
self.__manifest = result["Value"][0]
|
|
47
|
-
return S_OK()
|
|
48
|
-
|
|
49
|
-
def loadCFG(self, cfgString):
|
|
50
|
-
"""
|
|
51
|
-
Load job manifest from CFG format
|
|
52
|
-
"""
|
|
53
|
-
try:
|
|
54
|
-
self.__manifest.loadFromBuffer(cfgString)
|
|
55
|
-
except Exception as e:
|
|
56
|
-
return S_ERROR(f"Can't load manifest from cfg: {str(e)}")
|
|
57
|
-
return S_OK()
|
|
58
|
-
|
|
59
|
-
def dumpAsCFG(self):
|
|
60
|
-
return str(self.__manifest)
|
|
61
|
-
|
|
62
|
-
def getAsCFG(self):
|
|
63
|
-
return self.__manifest.clone()
|
|
64
|
-
|
|
65
|
-
def dumpAsJDL(self):
|
|
66
|
-
return dumpCFGAsJDL(self.__manifest)
|
|
67
|
-
|
|
68
|
-
def __getCSValue(self, varName, defaultVal=None):
|
|
69
|
-
if not self.__ops:
|
|
70
|
-
self.__ops = Operations(group=self.__manifest["OwnerGroup"])
|
|
71
|
-
if varName[0] != "/":
|
|
72
|
-
varName = f"JobDescription/{varName}"
|
|
73
|
-
return self.__ops.getValue(varName, defaultVal)
|
|
74
|
-
|
|
75
|
-
def __checkNumericalVar(self, varName, defaultVal, minVal, maxVal):
|
|
76
|
-
"""
|
|
77
|
-
Check a numerical var
|
|
78
|
-
"""
|
|
79
|
-
initialVal = False
|
|
80
|
-
if varName not in self.__manifest:
|
|
81
|
-
varValue = self.__getCSValue(f"Default{varName}", defaultVal)
|
|
82
|
-
else:
|
|
83
|
-
varValue = self.__manifest[varName]
|
|
84
|
-
initialVal = varValue
|
|
85
|
-
try:
|
|
86
|
-
varValue = int(varValue)
|
|
87
|
-
except ValueError:
|
|
88
|
-
return S_ERROR(f"{varName} must be a number")
|
|
89
|
-
minVal = self.__getCSValue(f"Min{varName}", minVal)
|
|
90
|
-
maxVal = self.__getCSValue(f"Max{varName}", maxVal)
|
|
91
|
-
varValue = max(minVal, min(varValue, maxVal))
|
|
92
|
-
if initialVal != varValue:
|
|
93
|
-
self.__manifest.setOption(varName, varValue)
|
|
94
|
-
return S_OK(varValue)
|
|
95
|
-
|
|
96
|
-
def __checkChoiceVar(self, varName, defaultVal, choices):
|
|
97
|
-
"""
|
|
98
|
-
Check a choice var
|
|
99
|
-
"""
|
|
100
|
-
initialVal = False
|
|
101
|
-
if varName not in self.__manifest:
|
|
102
|
-
varValue = self.__getCSValue(f"Default{varName}", defaultVal)
|
|
103
|
-
else:
|
|
104
|
-
varValue = self.__manifest[varName]
|
|
105
|
-
initialVal = varValue
|
|
106
|
-
if varValue not in self.__getCSValue(f"Choices{varName}", choices):
|
|
107
|
-
return S_ERROR(f"{varValue} is not a valid value for {varName}")
|
|
108
|
-
if initialVal != varValue:
|
|
109
|
-
self.__manifest.setOption(varName, varValue)
|
|
110
|
-
return S_OK(varValue)
|
|
111
|
-
|
|
112
|
-
def __checkMultiChoice(self, varName, choices):
|
|
113
|
-
"""
|
|
114
|
-
Check a multi choice var
|
|
115
|
-
"""
|
|
116
|
-
initialVal = False
|
|
117
|
-
if varName not in self.__manifest:
|
|
118
|
-
return S_OK()
|
|
119
|
-
else:
|
|
120
|
-
varValue = self.__manifest[varName]
|
|
121
|
-
initialVal = varValue
|
|
122
|
-
choices = self.__getCSValue(f"Choices{varName}", choices)
|
|
123
|
-
for v in List.fromChar(varValue):
|
|
124
|
-
if v not in choices:
|
|
125
|
-
return S_ERROR(f"{v} is not a valid value for {varName}")
|
|
126
|
-
if initialVal != varValue:
|
|
127
|
-
self.__manifest.setOption(varName, varValue)
|
|
128
|
-
return S_OK(varValue)
|
|
3
|
+
from DIRACCommon.WorkloadManagementSystem.Client.JobState.JobManifest import * # noqa: F401, F403
|
|
129
4
|
|
|
130
|
-
|
|
131
|
-
"""
|
|
132
|
-
Check Maximum Number of Input Data files allowed
|
|
133
|
-
"""
|
|
134
|
-
varName = "InputData"
|
|
135
|
-
if varName not in self.__manifest:
|
|
136
|
-
return S_OK()
|
|
137
|
-
varValue = self.__manifest[varName]
|
|
138
|
-
if len(List.fromChar(varValue)) > maxNumber:
|
|
139
|
-
return S_ERROR(
|
|
140
|
-
"Number of Input Data Files (%s) greater than current limit: %s"
|
|
141
|
-
% (len(List.fromChar(varValue)), maxNumber)
|
|
142
|
-
)
|
|
143
|
-
return S_OK()
|
|
144
|
-
|
|
145
|
-
def __contains__(self, key):
|
|
146
|
-
"""Check if the manifest has the required key"""
|
|
147
|
-
return key in self.__manifest
|
|
5
|
+
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
|
|
148
6
|
|
|
149
|
-
def setOptionsFromDict(self, varDict):
|
|
150
|
-
for k in sorted(varDict):
|
|
151
|
-
self.setOption(k, varDict[k])
|
|
152
7
|
|
|
8
|
+
def makeJobManifestConfig(ownerGroup: str) -> JobManifestConfig:
|
|
9
|
+
ops = Operations(group=ownerGroup)
|
|
10
|
+
|
|
11
|
+
allowedJobTypesForGroup = ops.getValue(
|
|
12
|
+
"JobDescription/ChoicesJobType",
|
|
13
|
+
ops.getValue("JobDescription/AllowedJobTypes", ["User", "Test", "Hospital"])
|
|
14
|
+
+ ops.getValue("Transformations/DataProcessing", []),
|
|
15
|
+
)
|
|
16
|
+
|
|
17
|
+
return {
|
|
18
|
+
"defaultForGroup": {
|
|
19
|
+
"CPUTime": ops.getValue("JobDescription/DefaultCPUTime", 86400),
|
|
20
|
+
"Priority": ops.getValue("JobDescription/DefaultPriority", 1),
|
|
21
|
+
},
|
|
22
|
+
"minForGroup": {
|
|
23
|
+
"CPUTime": ops.getValue("JobDescription/MinCPUTime", 100),
|
|
24
|
+
"Priority": ops.getValue("JobDescription/MinPriority", 0),
|
|
25
|
+
},
|
|
26
|
+
"maxForGroup": {
|
|
27
|
+
"CPUTime": ops.getValue("JobDescription/MaxCPUTime", 500000),
|
|
28
|
+
"Priority": ops.getValue("JobDescription/MaxPriority", 10),
|
|
29
|
+
},
|
|
30
|
+
"allowedJobTypesForGroup": allowedJobTypesForGroup,
|
|
31
|
+
"maxInputData": Operations().getValue("JobDescription/MaxInputData", 500),
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class JobManifest(JobManifest): # noqa: F405 pylint: disable=function-redefined
|
|
153
36
|
def check(self):
|
|
154
|
-
""
|
|
155
|
-
Check that the manifest is OK
|
|
156
|
-
"""
|
|
157
|
-
for k in ["Owner", "OwnerGroup"]:
|
|
158
|
-
if k not in self.__manifest:
|
|
159
|
-
return S_ERROR(f"Missing var {k} in manifest")
|
|
160
|
-
|
|
161
|
-
# Check CPUTime
|
|
162
|
-
result = self.__checkNumericalVar("CPUTime", 86400, 100, 500000)
|
|
163
|
-
if not result["OK"]:
|
|
164
|
-
return result
|
|
165
|
-
|
|
166
|
-
result = self.__checkNumericalVar("Priority", 1, 0, 10)
|
|
167
|
-
if not result["OK"]:
|
|
168
|
-
return result
|
|
169
|
-
|
|
170
|
-
maxInputData = Operations().getValue("JobDescription/MaxInputData", 500)
|
|
171
|
-
result = self.__checkMaxInputData(maxInputData)
|
|
172
|
-
if not result["OK"]:
|
|
173
|
-
return result
|
|
174
|
-
|
|
175
|
-
operation = Operations(group=self.__manifest["OwnerGroup"])
|
|
176
|
-
allowedJobTypes = operation.getValue("JobDescription/AllowedJobTypes", ["User", "Test", "Hospital"])
|
|
177
|
-
transformationTypes = operation.getValue("Transformations/DataProcessing", [])
|
|
178
|
-
result = self.__checkMultiChoice("JobType", allowedJobTypes + transformationTypes)
|
|
179
|
-
if not result["OK"]:
|
|
180
|
-
return result
|
|
181
|
-
return S_OK()
|
|
182
|
-
|
|
183
|
-
def createSection(self, secName, contents=False):
|
|
184
|
-
if secName not in self.__manifest:
|
|
185
|
-
if contents and not isinstance(contents, CFG):
|
|
186
|
-
return S_ERROR(f"Contents for section {secName} is not a cfg object")
|
|
187
|
-
self.__dirty = True
|
|
188
|
-
return S_OK(self.__manifest.createNewSection(secName, contents=contents))
|
|
189
|
-
return S_ERROR(f"Section {secName} already exists")
|
|
190
|
-
|
|
191
|
-
def getSection(self, secName):
|
|
192
|
-
self.__dirty = True
|
|
193
|
-
if secName not in self.__manifest:
|
|
194
|
-
return S_ERROR(f"{secName} does not exist")
|
|
195
|
-
sec = self.__manifest[secName]
|
|
196
|
-
if not sec:
|
|
197
|
-
return S_ERROR(f"{secName} section empty")
|
|
198
|
-
return S_OK(sec)
|
|
199
|
-
|
|
200
|
-
def setSectionContents(self, secName, contents):
|
|
201
|
-
if contents and not isinstance(contents, CFG):
|
|
202
|
-
return S_ERROR(f"Contents for section {secName} is not a cfg object")
|
|
203
|
-
self.__dirty = True
|
|
204
|
-
if secName in self.__manifest:
|
|
205
|
-
self.__manifest[secName].reset()
|
|
206
|
-
self.__manifest[secName].mergeWith(contents)
|
|
207
|
-
else:
|
|
208
|
-
self.__manifest.createNewSection(secName, contents=contents)
|
|
209
|
-
|
|
210
|
-
def setOption(self, varName, varValue):
|
|
211
|
-
"""
|
|
212
|
-
Set a var in job manifest
|
|
213
|
-
"""
|
|
214
|
-
self.__dirty = True
|
|
215
|
-
levels = List.fromChar(varName, "/")
|
|
216
|
-
cfg = self.__manifest
|
|
217
|
-
for l in levels[:-1]:
|
|
218
|
-
if l not in cfg:
|
|
219
|
-
cfg.createNewSection(l)
|
|
220
|
-
cfg = cfg[l]
|
|
221
|
-
cfg.setOption(levels[-1], varValue)
|
|
222
|
-
|
|
223
|
-
def remove(self, opName):
|
|
224
|
-
levels = List.fromChar(opName, "/")
|
|
225
|
-
cfg = self.__manifest
|
|
226
|
-
for l in levels[:-1]:
|
|
227
|
-
if l not in cfg:
|
|
228
|
-
return S_ERROR(f"{opName} does not exist")
|
|
229
|
-
cfg = cfg[l]
|
|
230
|
-
if cfg.deleteKey(levels[-1]):
|
|
231
|
-
self.__dirty = True
|
|
232
|
-
return S_OK()
|
|
233
|
-
return S_ERROR(f"{opName} does not exist")
|
|
234
|
-
|
|
235
|
-
def getOption(self, varName, defaultValue=None):
|
|
236
|
-
"""
|
|
237
|
-
Get a variable from the job manifest
|
|
238
|
-
"""
|
|
239
|
-
cfg = self.__manifest
|
|
240
|
-
return cfg.getOption(varName, defaultValue)
|
|
241
|
-
|
|
242
|
-
def getOptionList(self, section=""):
|
|
243
|
-
"""
|
|
244
|
-
Get a list of variables in a section of the job manifest
|
|
245
|
-
"""
|
|
246
|
-
cfg = self.__manifest.getRecursive(section)
|
|
247
|
-
if not cfg or "value" not in cfg:
|
|
248
|
-
return []
|
|
249
|
-
cfg = cfg["value"]
|
|
250
|
-
return cfg.listOptions()
|
|
251
|
-
|
|
252
|
-
def isOption(self, opName):
|
|
253
|
-
"""
|
|
254
|
-
Check if it is a valid option
|
|
255
|
-
"""
|
|
256
|
-
return self.__manifest.isOption(opName)
|
|
257
|
-
|
|
258
|
-
def getSectionList(self, section=""):
|
|
259
|
-
"""
|
|
260
|
-
Get a list of sections in the job manifest
|
|
261
|
-
"""
|
|
262
|
-
cfg = self.__manifest.getRecursive(section)
|
|
263
|
-
if not cfg or "value" not in cfg:
|
|
264
|
-
return []
|
|
265
|
-
cfg = cfg["value"]
|
|
266
|
-
return cfg.listSections()
|
|
37
|
+
return super().check(config=makeJobManifestConfig(self.__manifest["OwnerGroup"]))
|
|
@@ -28,3 +28,6 @@ class JobStateUpdateClient(Client):
|
|
|
28
28
|
|
|
29
29
|
else:
|
|
30
30
|
self.serverURL = url
|
|
31
|
+
|
|
32
|
+
def setJobStatus(self, jobID, status="", minorStatus="", source="Unknown", datetime_=None, force=False, **kwargs):
|
|
33
|
+
return self._getRPC(**kwargs).setJobStatus(jobID, status, minorStatus, source, datetime_, force)
|
|
@@ -1,154 +1,10 @@
|
|
|
1
|
-
"""
|
|
2
|
-
This module contains constants and lists for the possible job states.
|
|
3
|
-
"""
|
|
4
|
-
|
|
5
|
-
from DIRAC import gLogger, S_OK, S_ERROR
|
|
6
|
-
from DIRAC.Core.Utilities.StateMachine import State, StateMachine
|
|
7
|
-
from DIRAC.Core.Utilities.Decorators import deprecated
|
|
8
|
-
|
|
9
|
-
from DIRAC.WorkloadManagementSystem.Client.JobMonitoringClient import JobMonitoringClient
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
#:
|
|
13
|
-
SUBMITTING = "Submitting"
|
|
14
|
-
#:
|
|
15
|
-
RECEIVED = "Received"
|
|
16
|
-
#:
|
|
17
|
-
CHECKING = "Checking"
|
|
18
|
-
#:
|
|
19
|
-
STAGING = "Staging"
|
|
20
|
-
#:
|
|
21
|
-
SCOUTING = "Scouting"
|
|
22
|
-
#:
|
|
23
|
-
WAITING = "Waiting"
|
|
24
|
-
#:
|
|
25
|
-
MATCHED = "Matched"
|
|
26
|
-
#: The Rescheduled status is effectively never stored in the DB.
|
|
27
|
-
#: It could be considered a "virtual" status, and might even be dropped.
|
|
28
|
-
RESCHEDULED = "Rescheduled"
|
|
29
|
-
#:
|
|
30
|
-
RUNNING = "Running"
|
|
31
|
-
#:
|
|
32
|
-
STALLED = "Stalled"
|
|
33
|
-
#:
|
|
34
|
-
COMPLETING = "Completing"
|
|
35
|
-
#:
|
|
36
|
-
DONE = "Done"
|
|
37
|
-
#:
|
|
38
|
-
COMPLETED = "Completed"
|
|
39
|
-
#:
|
|
40
|
-
FAILED = "Failed"
|
|
41
|
-
#:
|
|
42
|
-
DELETED = "Deleted"
|
|
43
|
-
#:
|
|
44
|
-
KILLED = "Killed"
|
|
45
|
-
|
|
46
|
-
#: Possible job states
|
|
47
|
-
JOB_STATES = [
|
|
48
|
-
SUBMITTING,
|
|
49
|
-
RECEIVED,
|
|
50
|
-
CHECKING,
|
|
51
|
-
SCOUTING,
|
|
52
|
-
STAGING,
|
|
53
|
-
WAITING,
|
|
54
|
-
MATCHED,
|
|
55
|
-
RESCHEDULED,
|
|
56
|
-
RUNNING,
|
|
57
|
-
STALLED,
|
|
58
|
-
COMPLETING,
|
|
59
|
-
DONE,
|
|
60
|
-
COMPLETED,
|
|
61
|
-
FAILED,
|
|
62
|
-
DELETED,
|
|
63
|
-
KILLED,
|
|
64
|
-
]
|
|
65
|
-
|
|
66
|
-
# Job States when the payload work has finished
|
|
67
|
-
JOB_FINAL_STATES = [DONE, COMPLETED, FAILED, KILLED]
|
|
68
|
-
|
|
69
|
-
# WMS internal job States indicating the job object won't be updated
|
|
70
|
-
JOB_REALLY_FINAL_STATES = [DELETED]
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
class JobsStateMachine(StateMachine):
|
|
74
|
-
"""Jobs state machine"""
|
|
75
|
-
|
|
76
|
-
def __init__(self, state):
|
|
77
|
-
"""c'tor
|
|
78
|
-
Defines the state machine transactions
|
|
79
|
-
"""
|
|
80
|
-
super().__init__(state)
|
|
1
|
+
"""Backward compatibility wrapper - moved to DIRACCommon
|
|
81
2
|
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
KILLED: State(14, [DELETED], defState=KILLED),
|
|
86
|
-
FAILED: State(13, [RESCHEDULED, DELETED], defState=FAILED),
|
|
87
|
-
DONE: State(12, [DELETED], defState=DONE),
|
|
88
|
-
COMPLETED: State(11, [DONE, FAILED], defState=COMPLETED),
|
|
89
|
-
COMPLETING: State(10, [DONE, FAILED, COMPLETED, STALLED, KILLED], defState=COMPLETING),
|
|
90
|
-
STALLED: State(9, [RUNNING, FAILED, KILLED], defState=STALLED),
|
|
91
|
-
RUNNING: State(8, [STALLED, DONE, FAILED, RESCHEDULED, COMPLETING, KILLED, RECEIVED], defState=RUNNING),
|
|
92
|
-
RESCHEDULED: State(7, [WAITING, RECEIVED, DELETED, FAILED, KILLED], defState=RESCHEDULED),
|
|
93
|
-
MATCHED: State(6, [RUNNING, FAILED, RESCHEDULED, KILLED], defState=MATCHED),
|
|
94
|
-
WAITING: State(5, [MATCHED, RESCHEDULED, DELETED, KILLED], defState=WAITING),
|
|
95
|
-
STAGING: State(4, [CHECKING, WAITING, FAILED, KILLED], defState=STAGING),
|
|
96
|
-
SCOUTING: State(3, [CHECKING, FAILED, STALLED, KILLED], defState=SCOUTING),
|
|
97
|
-
CHECKING: State(2, [SCOUTING, STAGING, WAITING, RESCHEDULED, FAILED, DELETED, KILLED], defState=CHECKING),
|
|
98
|
-
RECEIVED: State(1, [SCOUTING, CHECKING, STAGING, WAITING, FAILED, DELETED, KILLED], defState=RECEIVED),
|
|
99
|
-
SUBMITTING: State(0, [RECEIVED, CHECKING, DELETED, KILLED], defState=SUBMITTING), # initial state
|
|
100
|
-
}
|
|
3
|
+
This module has been moved to DIRACCommon.WorkloadManagementSystem.Client.JobStatus to avoid
|
|
4
|
+
circular dependencies and allow DiracX to use these utilities without
|
|
5
|
+
triggering DIRAC's global state initialization.
|
|
101
6
|
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
if not currentStatus:
|
|
107
|
-
if not jobMonitoringClient:
|
|
108
|
-
from DIRAC.WorkloadManagementSystem.Client.JobMonitoringClient import JobMonitoringClient
|
|
109
|
-
|
|
110
|
-
jobMonitoringClient = JobMonitoringClient()
|
|
111
|
-
|
|
112
|
-
res = jobMonitoringClient.getJobsStatus(jobID)
|
|
113
|
-
if not res["OK"]:
|
|
114
|
-
return res
|
|
115
|
-
try:
|
|
116
|
-
currentStatus = res["Value"][jobID]["Status"]
|
|
117
|
-
except KeyError:
|
|
118
|
-
return S_ERROR("Job does not exist")
|
|
119
|
-
|
|
120
|
-
res = JobsStateMachine(currentStatus).getNextState(candidateState)
|
|
121
|
-
if not res["OK"]:
|
|
122
|
-
return res
|
|
123
|
-
|
|
124
|
-
# If the JobsStateMachine does not accept the candidate, return an ERROR
|
|
125
|
-
if candidateState != res["Value"]:
|
|
126
|
-
gLogger.error(
|
|
127
|
-
"Job Status Error",
|
|
128
|
-
f"{jobID} can't move from {currentStatus} to {candidateState}",
|
|
129
|
-
)
|
|
130
|
-
return S_ERROR("Job state transition not allowed")
|
|
131
|
-
return S_OK()
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
def filterJobStateTransition(jobIDs, candidateState):
|
|
135
|
-
"""Given a list of jobIDs, return a list that are allowed to transition
|
|
136
|
-
to the given candidate state.
|
|
137
|
-
"""
|
|
138
|
-
allowedJobs = []
|
|
139
|
-
|
|
140
|
-
if not isinstance(jobIDs, list):
|
|
141
|
-
jobIDs = [jobIDs]
|
|
142
|
-
|
|
143
|
-
res = JobMonitoringClient().getJobsStatus(jobIDs)
|
|
144
|
-
if not res["OK"]:
|
|
145
|
-
return res
|
|
146
|
-
|
|
147
|
-
for jobID in jobIDs:
|
|
148
|
-
if jobID in res["Value"]:
|
|
149
|
-
curState = res["Value"][jobID]["Status"]
|
|
150
|
-
stateRes = JobsStateMachine(curState).getNextState(candidateState)
|
|
151
|
-
if stateRes["OK"]:
|
|
152
|
-
if stateRes["Value"] == candidateState:
|
|
153
|
-
allowedJobs.append(jobID)
|
|
154
|
-
return S_OK(allowedJobs)
|
|
7
|
+
All exports are maintained for backward compatibility.
|
|
8
|
+
"""
|
|
9
|
+
# Re-export everything from DIRACCommon for backward compatibility
|
|
10
|
+
from DIRACCommon.WorkloadManagementSystem.Client.JobStatus import * # noqa: F401, F403
|
|
@@ -1,13 +1,18 @@
|
|
|
1
1
|
""" Client for the SandboxStore.
|
|
2
2
|
Will connect to the WorkloadManagement/SandboxStore service.
|
|
3
3
|
"""
|
|
4
|
+
from __future__ import annotations
|
|
4
5
|
|
|
5
6
|
import hashlib
|
|
6
7
|
import os
|
|
7
8
|
import re
|
|
8
9
|
import tarfile
|
|
9
10
|
import tempfile
|
|
11
|
+
from contextlib import contextmanager
|
|
10
12
|
from io import BytesIO, StringIO
|
|
13
|
+
from typing import Literal
|
|
14
|
+
|
|
15
|
+
import zstandard
|
|
11
16
|
|
|
12
17
|
from DIRAC import S_ERROR, S_OK, gLogger
|
|
13
18
|
from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getVOForGroup
|
|
@@ -18,9 +23,27 @@ from DIRAC.Core.Utilities.ReturnValues import returnSingleResult
|
|
|
18
23
|
from DIRAC.Resources.Storage.StorageElement import StorageElement
|
|
19
24
|
|
|
20
25
|
|
|
26
|
+
@contextmanager
|
|
27
|
+
def ZstdCompatibleTarFile(tarFileName: os.PathLike, *, mode: Literal["r"] = "r"):
|
|
28
|
+
"""Context manager to extend tarfile.open to support zstd compressed files.
|
|
29
|
+
|
|
30
|
+
This is only needed for Python <=3.13.
|
|
31
|
+
"""
|
|
32
|
+
with open(tarFileName, "rb") as f:
|
|
33
|
+
magic = f.read(4)
|
|
34
|
+
# Read magic bytes to determine compression format
|
|
35
|
+
if magic.startswith(b"\x28\xb5\x2f\xfd"): # zstd magic number
|
|
36
|
+
dctx = zstandard.ZstdDecompressor()
|
|
37
|
+
with open(tarFileName, "rb") as f, dctx.stream_reader(f) as decompressor:
|
|
38
|
+
with tarfile.open(fileobj=decompressor, mode=f"{mode}|") as tf:
|
|
39
|
+
yield tf
|
|
40
|
+
else:
|
|
41
|
+
with tarfile.open(name=tarFileName, mode=mode) as tf:
|
|
42
|
+
yield tf
|
|
43
|
+
|
|
44
|
+
|
|
21
45
|
class SandboxStoreClient:
|
|
22
46
|
__validSandboxTypes = ("Input", "Output")
|
|
23
|
-
__smdb = None
|
|
24
47
|
|
|
25
48
|
def __init__(self, rpcClient=None, transferClient=None, smdb=False, **kwargs):
|
|
26
49
|
"""Constructor
|
|
@@ -37,21 +60,8 @@ class SandboxStoreClient:
|
|
|
37
60
|
self.__transferClient = transferClient
|
|
38
61
|
self.__kwargs = kwargs
|
|
39
62
|
self.__vo = None
|
|
40
|
-
SandboxStoreClient.__smdb = smdb
|
|
41
63
|
if "delegatedGroup" in kwargs:
|
|
42
64
|
self.__vo = getVOForGroup(kwargs["delegatedGroup"])
|
|
43
|
-
if SandboxStoreClient.__smdb is True:
|
|
44
|
-
try:
|
|
45
|
-
from DIRAC.WorkloadManagementSystem.DB.SandboxMetadataDB import SandboxMetadataDB
|
|
46
|
-
|
|
47
|
-
SandboxStoreClient.__smdb = SandboxMetadataDB()
|
|
48
|
-
result = SandboxStoreClient.__smdb._getConnection() # pylint: disable=protected-access
|
|
49
|
-
if not result["OK"]:
|
|
50
|
-
SandboxStoreClient.__smdb = False
|
|
51
|
-
else:
|
|
52
|
-
result["Value"].close()
|
|
53
|
-
except (ImportError, RuntimeError, AttributeError):
|
|
54
|
-
SandboxStoreClient.__smdb = False
|
|
55
65
|
|
|
56
66
|
def __getRPCClient(self):
|
|
57
67
|
"""Get an RPC client for SB service"""
|
|
@@ -206,7 +216,7 @@ class SandboxStoreClient:
|
|
|
206
216
|
|
|
207
217
|
try:
|
|
208
218
|
sandboxSize = 0
|
|
209
|
-
with
|
|
219
|
+
with ZstdCompatibleTarFile(tarFileName, mode="r") as tf:
|
|
210
220
|
for tarinfo in tf:
|
|
211
221
|
tf.extract(tarinfo, path=destinationDir)
|
|
212
222
|
sandboxSize += tarinfo.size
|
|
@@ -227,29 +237,6 @@ class SandboxStoreClient:
|
|
|
227
237
|
##############
|
|
228
238
|
# Jobs
|
|
229
239
|
|
|
230
|
-
def assignSandboxesToJob(self, jobId, sbList, ownerName="", ownerGroup=""):
|
|
231
|
-
"""
|
|
232
|
-
Assign sandboxes to a job.
|
|
233
|
-
sbList must be a list of sandboxes and relation types
|
|
234
|
-
sbList = [ ( "SB:SEName|SEPFN", "Input" ), ( "SB:SEName|SEPFN", "Output" ) ]
|
|
235
|
-
"""
|
|
236
|
-
eId = f"Job:{jobId}"
|
|
237
|
-
for sbT in sbList:
|
|
238
|
-
if sbT[1] not in self.__validSandboxTypes:
|
|
239
|
-
return S_ERROR(f"Invalid Sandbox type {sbT[1]}")
|
|
240
|
-
if SandboxStoreClient.__smdb and ownerName and ownerGroup:
|
|
241
|
-
return SandboxStoreClient.__smdb.assignSandboxesToEntities({eId: sbList}, ownerName, ownerGroup)
|
|
242
|
-
return self.__getRPCClient().assignSandboxesToEntities({eId: sbList}, ownerName, ownerGroup)
|
|
243
|
-
|
|
244
|
-
def unassignJobs(self, jobIdList):
|
|
245
|
-
"""Unassign SB to a job"""
|
|
246
|
-
if isinstance(jobIdList, int):
|
|
247
|
-
jobIdList = [jobIdList]
|
|
248
|
-
entitiesList = []
|
|
249
|
-
for jobId in jobIdList:
|
|
250
|
-
entitiesList.append(f"Job:{jobId}")
|
|
251
|
-
return self.__getRPCClient().unassignEntities(entitiesList)
|
|
252
|
-
|
|
253
240
|
def downloadSandboxForJob(self, jobId, sbType, destinationPath="", inMemory=False, unpack=True):
|
|
254
241
|
"""Download SB for a job"""
|
|
255
242
|
result = self.__getRPCClient().getSandboxesAssignedToEntity(f"Job:{jobId}")
|
|
@@ -2,11 +2,10 @@
|
|
|
2
2
|
methods necessary to communicate with the Workload Management System
|
|
3
3
|
"""
|
|
4
4
|
import os
|
|
5
|
-
from io import StringIO
|
|
6
5
|
import time
|
|
6
|
+
from io import StringIO
|
|
7
7
|
|
|
8
|
-
from DIRAC import
|
|
9
|
-
|
|
8
|
+
from DIRAC import S_ERROR, S_OK, gLogger
|
|
10
9
|
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
|
|
11
10
|
from DIRAC.Core.Utilities import File
|
|
12
11
|
from DIRAC.Core.Utilities.ClassAd.ClassAdLight import ClassAd
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
"""Test for WMS clients."""
|
|
2
2
|
# pylint: disable=protected-access, missing-docstring, invalid-name
|
|
3
3
|
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
import shutil
|
|
4
6
|
import pytest
|
|
5
7
|
|
|
6
8
|
from unittest.mock import MagicMock
|
|
@@ -280,3 +282,30 @@ def test_DLI_execute_NoLocal(mocker, dli, mockSE):
|
|
|
280
282
|
assert res["Value"]["Failed"]
|
|
281
283
|
assert "/a/lfn/1.txt" in res["Value"]["Failed"], res
|
|
282
284
|
assert res["Value"]["Failed"][0] == "/a/lfn/1.txt", res
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
def test_DLI_execute_jobIDPath(mocker, dli, mockSE):
|
|
288
|
+
"""Specify a jobIDPath. Output should be in the jobIDPath directory."""
|
|
289
|
+
mocker.patch("DIRAC.WorkloadManagementSystem.Client.DownloadInputData.gConfig.getValue", return_value=2)
|
|
290
|
+
|
|
291
|
+
# Create the jobIDPath directory
|
|
292
|
+
jobIDPath = Path().cwd() / "job" / "12345"
|
|
293
|
+
jobIDPath.mkdir(parents=True, exist_ok=True)
|
|
294
|
+
|
|
295
|
+
dli.configuration["JobIDPath"] = str(jobIDPath)
|
|
296
|
+
|
|
297
|
+
mocker.patch("DIRAC.WorkloadManagementSystem.Client.DownloadInputData.gConfig.getValue", return_value=2)
|
|
298
|
+
mockObjectSE = mockSE.return_value
|
|
299
|
+
mockObjectSE.getFileMetadata.return_value = S_OK(
|
|
300
|
+
{"Successful": {"/a/lfn/1.txt": {"Cached": 1, "Accessible": 0}}, "Failed": {}}
|
|
301
|
+
)
|
|
302
|
+
dli._downloadFromSE = MagicMock(side_effect=[S_ERROR("Failed to down"), S_OK({"path": jobIDPath / "1.txt"})])
|
|
303
|
+
dli._isCache = MagicMock(return_value=True)
|
|
304
|
+
res = dli.execute(dataToResolve=["/a/lfn/1.txt"])
|
|
305
|
+
|
|
306
|
+
assert res["OK"]
|
|
307
|
+
assert not res["Value"]["Failed"]
|
|
308
|
+
assert "/a/lfn/1.txt" in res["Value"]["Successful"], res
|
|
309
|
+
|
|
310
|
+
# Check that the output path is in the jobIDPath directory
|
|
311
|
+
assert res["Value"]["Successful"]["/a/lfn/1.txt"]["path"] == jobIDPath / "1.txt", res
|
|
@@ -263,14 +263,10 @@ Agents
|
|
|
263
263
|
# the DN of the certificate proxy used to submit pilots. If not found here, what is in Operations/Pilot section of the CS will be used
|
|
264
264
|
PilotDN =
|
|
265
265
|
|
|
266
|
-
# List of
|
|
267
|
-
|
|
268
|
-
# List of
|
|
269
|
-
|
|
270
|
-
# List of CE types that will be treated by this SiteDirector (No value can refer to any type of CE defined in the CS)
|
|
271
|
-
CETypes =
|
|
272
|
-
# List of Tags that are required to be present in the CE/Queue definition
|
|
273
|
-
Tags =
|
|
266
|
+
# Site = # List of CEs that will be treated by this SiteDirector (No value can refer to any CE defined in the CS)
|
|
267
|
+
# CEs = # List of CE types that will be treated by this SiteDirector (No value can refer to any type of CE defined in the CS)
|
|
268
|
+
# CETypes = # List of Tags that are required to be present in the CE/Queue definition
|
|
269
|
+
# Tags =
|
|
274
270
|
|
|
275
271
|
# How many cycles to skip if queue is not working
|
|
276
272
|
FailedQueueCycleFactor = 10
|