DIRAC 9.0.0a54__py3-none-any.whl → 9.0.7__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (166) hide show
  1. DIRAC/AccountingSystem/Client/AccountingCLI.py +0 -140
  2. DIRAC/AccountingSystem/Client/DataStoreClient.py +0 -13
  3. DIRAC/AccountingSystem/Client/Types/BaseAccountingType.py +0 -7
  4. DIRAC/AccountingSystem/ConfigTemplate.cfg +0 -5
  5. DIRAC/AccountingSystem/Service/DataStoreHandler.py +0 -72
  6. DIRAC/ConfigurationSystem/Client/Helpers/CSGlobals.py +0 -9
  7. DIRAC/ConfigurationSystem/Client/Helpers/Registry.py +34 -32
  8. DIRAC/ConfigurationSystem/Client/Helpers/Resources.py +11 -43
  9. DIRAC/ConfigurationSystem/Client/Helpers/test/Test_Helpers.py +0 -16
  10. DIRAC/ConfigurationSystem/Client/LocalConfiguration.py +14 -8
  11. DIRAC/ConfigurationSystem/Client/PathFinder.py +47 -8
  12. DIRAC/ConfigurationSystem/Client/SyncPlugins/CERNLDAPSyncPlugin.py +4 -1
  13. DIRAC/ConfigurationSystem/Client/VOMS2CSSynchronizer.py +9 -2
  14. DIRAC/ConfigurationSystem/Client/test/Test_PathFinder.py +41 -1
  15. DIRAC/ConfigurationSystem/private/RefresherBase.py +4 -2
  16. DIRAC/Core/DISET/ServiceReactor.py +11 -3
  17. DIRAC/Core/DISET/private/BaseClient.py +1 -2
  18. DIRAC/Core/DISET/private/Transports/M2SSLTransport.py +9 -7
  19. DIRAC/Core/Security/DiracX.py +12 -7
  20. DIRAC/Core/Security/IAMService.py +4 -3
  21. DIRAC/Core/Security/ProxyInfo.py +9 -5
  22. DIRAC/Core/Security/test/test_diracx_token_from_pem.py +161 -0
  23. DIRAC/Core/Tornado/Client/ClientSelector.py +4 -1
  24. DIRAC/Core/Tornado/Server/TornadoService.py +1 -1
  25. DIRAC/Core/Utilities/ClassAd/ClassAdLight.py +4 -290
  26. DIRAC/Core/Utilities/DErrno.py +5 -309
  27. DIRAC/Core/Utilities/Extensions.py +10 -1
  28. DIRAC/Core/Utilities/Graphs/GraphData.py +1 -1
  29. DIRAC/Core/Utilities/JDL.py +1 -195
  30. DIRAC/Core/Utilities/List.py +1 -124
  31. DIRAC/Core/Utilities/MySQL.py +101 -97
  32. DIRAC/Core/Utilities/Os.py +32 -1
  33. DIRAC/Core/Utilities/Platform.py +2 -107
  34. DIRAC/Core/Utilities/ReturnValues.py +7 -252
  35. DIRAC/Core/Utilities/StateMachine.py +12 -178
  36. DIRAC/Core/Utilities/TimeUtilities.py +10 -253
  37. DIRAC/Core/Utilities/test/Test_JDL.py +0 -3
  38. DIRAC/Core/Utilities/test/Test_Profiler.py +20 -20
  39. DIRAC/Core/scripts/dirac_agent.py +1 -1
  40. DIRAC/Core/scripts/dirac_apptainer_exec.py +16 -7
  41. DIRAC/Core/scripts/dirac_platform.py +1 -92
  42. DIRAC/DataManagementSystem/Agent/FTS3Agent.py +8 -7
  43. DIRAC/DataManagementSystem/Agent/RequestOperations/RemoveFile.py +7 -6
  44. DIRAC/DataManagementSystem/Client/FTS3Job.py +71 -34
  45. DIRAC/DataManagementSystem/DB/FTS3DB.py +3 -0
  46. DIRAC/DataManagementSystem/DB/FileCatalogComponents/DatasetManager/DatasetManager.py +1 -1
  47. DIRAC/DataManagementSystem/Utilities/DMSHelpers.py +6 -2
  48. DIRAC/DataManagementSystem/scripts/dirac_dms_create_moving_request.py +2 -0
  49. DIRAC/DataManagementSystem/scripts/dirac_dms_protocol_matrix.py +0 -1
  50. DIRAC/FrameworkSystem/Client/ComponentInstaller.py +4 -2
  51. DIRAC/FrameworkSystem/DB/ProxyDB.py +9 -5
  52. DIRAC/FrameworkSystem/Utilities/TokenManagementUtilities.py +3 -2
  53. DIRAC/FrameworkSystem/Utilities/diracx.py +2 -74
  54. DIRAC/FrameworkSystem/private/authorization/AuthServer.py +2 -2
  55. DIRAC/FrameworkSystem/scripts/dirac_login.py +2 -2
  56. DIRAC/FrameworkSystem/scripts/dirac_proxy_init.py +1 -1
  57. DIRAC/Interfaces/API/Dirac.py +27 -13
  58. DIRAC/Interfaces/API/DiracAdmin.py +42 -7
  59. DIRAC/Interfaces/API/Job.py +1 -0
  60. DIRAC/Interfaces/scripts/dirac_admin_allow_site.py +7 -1
  61. DIRAC/Interfaces/scripts/dirac_admin_ban_site.py +7 -1
  62. DIRAC/Interfaces/scripts/dirac_wms_job_parameters.py +0 -1
  63. DIRAC/MonitoringSystem/Client/Types/WMSHistory.py +4 -0
  64. DIRAC/MonitoringSystem/Client/WebAppClient.py +26 -0
  65. DIRAC/MonitoringSystem/ConfigTemplate.cfg +9 -0
  66. DIRAC/MonitoringSystem/DB/MonitoringDB.py +6 -25
  67. DIRAC/MonitoringSystem/Service/MonitoringHandler.py +0 -33
  68. DIRAC/MonitoringSystem/Service/WebAppHandler.py +599 -0
  69. DIRAC/MonitoringSystem/private/MainReporter.py +0 -3
  70. DIRAC/ProductionSystem/scripts/dirac_prod_get_trans.py +2 -3
  71. DIRAC/RequestManagementSystem/Agent/RequestExecutingAgent.py +8 -6
  72. DIRAC/RequestManagementSystem/ConfigTemplate.cfg +6 -6
  73. DIRAC/RequestManagementSystem/DB/test/RMSTestScenari.py +2 -0
  74. DIRAC/ResourceStatusSystem/Client/SiteStatus.py +4 -2
  75. DIRAC/ResourceStatusSystem/Command/FreeDiskSpaceCommand.py +3 -1
  76. DIRAC/ResourceStatusSystem/Utilities/CSHelpers.py +2 -31
  77. DIRAC/ResourceStatusSystem/scripts/dirac_rss_set_status.py +18 -4
  78. DIRAC/Resources/Catalog/RucioFileCatalogClient.py +1 -1
  79. DIRAC/Resources/Computing/AREXComputingElement.py +19 -3
  80. DIRAC/Resources/Computing/BatchSystems/Condor.py +126 -108
  81. DIRAC/Resources/Computing/BatchSystems/SLURM.py +5 -1
  82. DIRAC/Resources/Computing/BatchSystems/test/Test_SLURM.py +46 -0
  83. DIRAC/Resources/Computing/HTCondorCEComputingElement.py +37 -43
  84. DIRAC/Resources/Computing/SingularityComputingElement.py +6 -1
  85. DIRAC/Resources/Computing/test/Test_HTCondorCEComputingElement.py +67 -49
  86. DIRAC/Resources/Computing/test/Test_PoolComputingElement.py +2 -1
  87. DIRAC/Resources/IdProvider/CheckInIdProvider.py +13 -0
  88. DIRAC/Resources/IdProvider/IdProviderFactory.py +11 -3
  89. DIRAC/Resources/Storage/StorageBase.py +4 -2
  90. DIRAC/Resources/Storage/StorageElement.py +4 -4
  91. DIRAC/TransformationSystem/Agent/TaskManagerAgentBase.py +10 -16
  92. DIRAC/TransformationSystem/Agent/TransformationAgent.py +22 -1
  93. DIRAC/TransformationSystem/Agent/TransformationCleaningAgent.py +15 -15
  94. DIRAC/TransformationSystem/Client/Transformation.py +2 -1
  95. DIRAC/TransformationSystem/Client/TransformationClient.py +0 -7
  96. DIRAC/TransformationSystem/Client/Utilities.py +9 -0
  97. DIRAC/TransformationSystem/Service/TransformationManagerHandler.py +0 -336
  98. DIRAC/TransformationSystem/Utilities/ReplicationCLIParameters.py +3 -3
  99. DIRAC/TransformationSystem/scripts/dirac_production_runjoblocal.py +2 -4
  100. DIRAC/TransformationSystem/test/Test_replicationTransformation.py +5 -6
  101. DIRAC/Workflow/Modules/test/Test_Modules.py +5 -0
  102. DIRAC/WorkloadManagementSystem/Agent/JobAgent.py +1 -5
  103. DIRAC/WorkloadManagementSystem/Agent/JobCleaningAgent.py +11 -7
  104. DIRAC/WorkloadManagementSystem/Agent/PilotSyncAgent.py +4 -3
  105. DIRAC/WorkloadManagementSystem/Agent/PushJobAgent.py +13 -13
  106. DIRAC/WorkloadManagementSystem/Agent/SiteDirector.py +10 -13
  107. DIRAC/WorkloadManagementSystem/Agent/StalledJobAgent.py +18 -51
  108. DIRAC/WorkloadManagementSystem/Agent/StatesAccountingAgent.py +41 -1
  109. DIRAC/WorkloadManagementSystem/Agent/test/Test_Agent_JobAgent.py +2 -0
  110. DIRAC/WorkloadManagementSystem/Agent/test/Test_Agent_JobCleaningAgent.py +7 -9
  111. DIRAC/WorkloadManagementSystem/Agent/test/Test_Agent_PushJobAgent.py +1 -0
  112. DIRAC/WorkloadManagementSystem/Agent/test/Test_Agent_SiteDirector.py +8 -2
  113. DIRAC/WorkloadManagementSystem/Agent/test/Test_Agent_StalledJobAgent.py +4 -5
  114. DIRAC/WorkloadManagementSystem/Client/DownloadInputData.py +7 -5
  115. DIRAC/WorkloadManagementSystem/Client/JobMonitoringClient.py +10 -11
  116. DIRAC/WorkloadManagementSystem/Client/JobState/JobManifest.py +32 -261
  117. DIRAC/WorkloadManagementSystem/Client/JobStateUpdateClient.py +3 -0
  118. DIRAC/WorkloadManagementSystem/Client/JobStatus.py +8 -152
  119. DIRAC/WorkloadManagementSystem/Client/SandboxStoreClient.py +25 -38
  120. DIRAC/WorkloadManagementSystem/Client/WMSClient.py +2 -3
  121. DIRAC/WorkloadManagementSystem/Client/test/Test_Client_DownloadInputData.py +29 -0
  122. DIRAC/WorkloadManagementSystem/ConfigTemplate.cfg +4 -8
  123. DIRAC/WorkloadManagementSystem/DB/JobDB.py +40 -69
  124. DIRAC/WorkloadManagementSystem/DB/JobDBUtils.py +18 -147
  125. DIRAC/WorkloadManagementSystem/DB/JobParametersDB.py +9 -9
  126. DIRAC/WorkloadManagementSystem/DB/PilotAgentsDB.py +3 -2
  127. DIRAC/WorkloadManagementSystem/DB/SandboxMetadataDB.py +28 -39
  128. DIRAC/WorkloadManagementSystem/DB/StatusUtils.py +125 -0
  129. DIRAC/WorkloadManagementSystem/DB/tests/Test_JobDB.py +1 -1
  130. DIRAC/WorkloadManagementSystem/DB/tests/Test_StatusUtils.py +28 -0
  131. DIRAC/WorkloadManagementSystem/Executor/JobSanity.py +3 -3
  132. DIRAC/WorkloadManagementSystem/FutureClient/JobStateUpdateClient.py +2 -14
  133. DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py +14 -9
  134. DIRAC/WorkloadManagementSystem/JobWrapper/test/Test_JobWrapper.py +36 -10
  135. DIRAC/WorkloadManagementSystem/JobWrapper/test/Test_JobWrapperTemplate.py +4 -0
  136. DIRAC/WorkloadManagementSystem/Service/JobManagerHandler.py +33 -154
  137. DIRAC/WorkloadManagementSystem/Service/JobMonitoringHandler.py +5 -323
  138. DIRAC/WorkloadManagementSystem/Service/JobStateUpdateHandler.py +0 -16
  139. DIRAC/WorkloadManagementSystem/Service/PilotManagerHandler.py +6 -102
  140. DIRAC/WorkloadManagementSystem/Service/SandboxStoreHandler.py +5 -51
  141. DIRAC/WorkloadManagementSystem/Service/WMSAdministratorHandler.py +16 -79
  142. DIRAC/WorkloadManagementSystem/Utilities/JobModel.py +28 -199
  143. DIRAC/WorkloadManagementSystem/Utilities/JobParameters.py +65 -3
  144. DIRAC/WorkloadManagementSystem/Utilities/JobStatusUtility.py +2 -64
  145. DIRAC/WorkloadManagementSystem/Utilities/ParametricJob.py +7 -171
  146. DIRAC/WorkloadManagementSystem/Utilities/PilotCStoJSONSynchronizer.py +73 -7
  147. DIRAC/WorkloadManagementSystem/Utilities/PilotWrapper.py +2 -0
  148. DIRAC/WorkloadManagementSystem/Utilities/RemoteRunner.py +16 -0
  149. DIRAC/WorkloadManagementSystem/Utilities/Utils.py +36 -1
  150. DIRAC/WorkloadManagementSystem/Utilities/jobAdministration.py +15 -0
  151. DIRAC/WorkloadManagementSystem/Utilities/test/Test_JobModel.py +1 -5
  152. DIRAC/WorkloadManagementSystem/Utilities/test/Test_ParametricJob.py +45 -128
  153. DIRAC/WorkloadManagementSystem/Utilities/test/Test_PilotWrapper.py +16 -0
  154. DIRAC/__init__.py +55 -54
  155. {dirac-9.0.0a54.dist-info → dirac-9.0.7.dist-info}/METADATA +6 -4
  156. {dirac-9.0.0a54.dist-info → dirac-9.0.7.dist-info}/RECORD +160 -160
  157. {dirac-9.0.0a54.dist-info → dirac-9.0.7.dist-info}/WHEEL +1 -1
  158. {dirac-9.0.0a54.dist-info → dirac-9.0.7.dist-info}/entry_points.txt +0 -3
  159. DIRAC/Core/Utilities/test/Test_List.py +0 -150
  160. DIRAC/Core/Utilities/test/Test_Time.py +0 -88
  161. DIRAC/TransformationSystem/scripts/dirac_transformation_archive.py +0 -30
  162. DIRAC/TransformationSystem/scripts/dirac_transformation_clean.py +0 -30
  163. DIRAC/TransformationSystem/scripts/dirac_transformation_remove_output.py +0 -30
  164. DIRAC/WorkloadManagementSystem/Utilities/test/Test_JobManager.py +0 -58
  165. {dirac-9.0.0a54.dist-info → dirac-9.0.7.dist-info}/licenses/LICENSE +0 -0
  166. {dirac-9.0.0a54.dist-info → dirac-9.0.7.dist-info}/top_level.txt +0 -0
@@ -1,266 +1,37 @@
1
- from diraccfg import CFG
1
+ from __future__ import annotations
2
2
 
3
- from DIRAC import S_OK, S_ERROR
4
- from DIRAC.Core.Utilities import List
5
- from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
6
- from DIRAC.Core.Utilities.JDL import loadJDLAsCFG, dumpCFGAsJDL
7
-
8
-
9
- class JobManifest:
10
- def __init__(self, manifest=""):
11
- self.__manifest = CFG()
12
- self.__dirty = False
13
- self.__ops = False
14
- if manifest:
15
- result = self.load(manifest)
16
- if not result["OK"]:
17
- raise Exception(result["Message"])
18
-
19
- def isDirty(self):
20
- return self.__dirty
21
-
22
- def setDirty(self):
23
- self.__dirty = True
24
-
25
- def clearDirty(self):
26
- self.__dirty = False
27
-
28
- def load(self, dataString):
29
- """
30
- Auto discover format type based on [ .. ] of JDL
31
- """
32
- dataString = dataString.strip()
33
- if dataString[0] == "[" and dataString[-1] == "]":
34
- return self.loadJDL(dataString)
35
- else:
36
- return self.loadCFG(dataString)
37
-
38
- def loadJDL(self, jdlString):
39
- """
40
- Load job manifest from JDL format
41
- """
42
- result = loadJDLAsCFG(jdlString.strip())
43
- if not result["OK"]:
44
- self.__manifest = CFG()
45
- return result
46
- self.__manifest = result["Value"][0]
47
- return S_OK()
48
-
49
- def loadCFG(self, cfgString):
50
- """
51
- Load job manifest from CFG format
52
- """
53
- try:
54
- self.__manifest.loadFromBuffer(cfgString)
55
- except Exception as e:
56
- return S_ERROR(f"Can't load manifest from cfg: {str(e)}")
57
- return S_OK()
58
-
59
- def dumpAsCFG(self):
60
- return str(self.__manifest)
61
-
62
- def getAsCFG(self):
63
- return self.__manifest.clone()
64
-
65
- def dumpAsJDL(self):
66
- return dumpCFGAsJDL(self.__manifest)
67
-
68
- def __getCSValue(self, varName, defaultVal=None):
69
- if not self.__ops:
70
- self.__ops = Operations(group=self.__manifest["OwnerGroup"])
71
- if varName[0] != "/":
72
- varName = f"JobDescription/{varName}"
73
- return self.__ops.getValue(varName, defaultVal)
74
-
75
- def __checkNumericalVar(self, varName, defaultVal, minVal, maxVal):
76
- """
77
- Check a numerical var
78
- """
79
- initialVal = False
80
- if varName not in self.__manifest:
81
- varValue = self.__getCSValue(f"Default{varName}", defaultVal)
82
- else:
83
- varValue = self.__manifest[varName]
84
- initialVal = varValue
85
- try:
86
- varValue = int(varValue)
87
- except ValueError:
88
- return S_ERROR(f"{varName} must be a number")
89
- minVal = self.__getCSValue(f"Min{varName}", minVal)
90
- maxVal = self.__getCSValue(f"Max{varName}", maxVal)
91
- varValue = max(minVal, min(varValue, maxVal))
92
- if initialVal != varValue:
93
- self.__manifest.setOption(varName, varValue)
94
- return S_OK(varValue)
95
-
96
- def __checkChoiceVar(self, varName, defaultVal, choices):
97
- """
98
- Check a choice var
99
- """
100
- initialVal = False
101
- if varName not in self.__manifest:
102
- varValue = self.__getCSValue(f"Default{varName}", defaultVal)
103
- else:
104
- varValue = self.__manifest[varName]
105
- initialVal = varValue
106
- if varValue not in self.__getCSValue(f"Choices{varName}", choices):
107
- return S_ERROR(f"{varValue} is not a valid value for {varName}")
108
- if initialVal != varValue:
109
- self.__manifest.setOption(varName, varValue)
110
- return S_OK(varValue)
111
-
112
- def __checkMultiChoice(self, varName, choices):
113
- """
114
- Check a multi choice var
115
- """
116
- initialVal = False
117
- if varName not in self.__manifest:
118
- return S_OK()
119
- else:
120
- varValue = self.__manifest[varName]
121
- initialVal = varValue
122
- choices = self.__getCSValue(f"Choices{varName}", choices)
123
- for v in List.fromChar(varValue):
124
- if v not in choices:
125
- return S_ERROR(f"{v} is not a valid value for {varName}")
126
- if initialVal != varValue:
127
- self.__manifest.setOption(varName, varValue)
128
- return S_OK(varValue)
3
+ from DIRACCommon.WorkloadManagementSystem.Client.JobState.JobManifest import * # noqa: F401, F403
129
4
 
130
- def __checkMaxInputData(self, maxNumber):
131
- """
132
- Check Maximum Number of Input Data files allowed
133
- """
134
- varName = "InputData"
135
- if varName not in self.__manifest:
136
- return S_OK()
137
- varValue = self.__manifest[varName]
138
- if len(List.fromChar(varValue)) > maxNumber:
139
- return S_ERROR(
140
- "Number of Input Data Files (%s) greater than current limit: %s"
141
- % (len(List.fromChar(varValue)), maxNumber)
142
- )
143
- return S_OK()
144
-
145
- def __contains__(self, key):
146
- """Check if the manifest has the required key"""
147
- return key in self.__manifest
5
+ from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
148
6
 
149
- def setOptionsFromDict(self, varDict):
150
- for k in sorted(varDict):
151
- self.setOption(k, varDict[k])
152
7
 
8
+ def makeJobManifestConfig(ownerGroup: str) -> JobManifestConfig:
9
+ ops = Operations(group=ownerGroup)
10
+
11
+ allowedJobTypesForGroup = ops.getValue(
12
+ "JobDescription/ChoicesJobType",
13
+ ops.getValue("JobDescription/AllowedJobTypes", ["User", "Test", "Hospital"])
14
+ + ops.getValue("Transformations/DataProcessing", []),
15
+ )
16
+
17
+ return {
18
+ "defaultForGroup": {
19
+ "CPUTime": ops.getValue("JobDescription/DefaultCPUTime", 86400),
20
+ "Priority": ops.getValue("JobDescription/DefaultPriority", 1),
21
+ },
22
+ "minForGroup": {
23
+ "CPUTime": ops.getValue("JobDescription/MinCPUTime", 100),
24
+ "Priority": ops.getValue("JobDescription/MinPriority", 0),
25
+ },
26
+ "maxForGroup": {
27
+ "CPUTime": ops.getValue("JobDescription/MaxCPUTime", 500000),
28
+ "Priority": ops.getValue("JobDescription/MaxPriority", 10),
29
+ },
30
+ "allowedJobTypesForGroup": allowedJobTypesForGroup,
31
+ "maxInputData": Operations().getValue("JobDescription/MaxInputData", 500),
32
+ }
33
+
34
+
35
+ class JobManifest(JobManifest): # noqa: F405 pylint: disable=function-redefined
153
36
  def check(self):
154
- """
155
- Check that the manifest is OK
156
- """
157
- for k in ["Owner", "OwnerGroup"]:
158
- if k not in self.__manifest:
159
- return S_ERROR(f"Missing var {k} in manifest")
160
-
161
- # Check CPUTime
162
- result = self.__checkNumericalVar("CPUTime", 86400, 100, 500000)
163
- if not result["OK"]:
164
- return result
165
-
166
- result = self.__checkNumericalVar("Priority", 1, 0, 10)
167
- if not result["OK"]:
168
- return result
169
-
170
- maxInputData = Operations().getValue("JobDescription/MaxInputData", 500)
171
- result = self.__checkMaxInputData(maxInputData)
172
- if not result["OK"]:
173
- return result
174
-
175
- operation = Operations(group=self.__manifest["OwnerGroup"])
176
- allowedJobTypes = operation.getValue("JobDescription/AllowedJobTypes", ["User", "Test", "Hospital"])
177
- transformationTypes = operation.getValue("Transformations/DataProcessing", [])
178
- result = self.__checkMultiChoice("JobType", allowedJobTypes + transformationTypes)
179
- if not result["OK"]:
180
- return result
181
- return S_OK()
182
-
183
- def createSection(self, secName, contents=False):
184
- if secName not in self.__manifest:
185
- if contents and not isinstance(contents, CFG):
186
- return S_ERROR(f"Contents for section {secName} is not a cfg object")
187
- self.__dirty = True
188
- return S_OK(self.__manifest.createNewSection(secName, contents=contents))
189
- return S_ERROR(f"Section {secName} already exists")
190
-
191
- def getSection(self, secName):
192
- self.__dirty = True
193
- if secName not in self.__manifest:
194
- return S_ERROR(f"{secName} does not exist")
195
- sec = self.__manifest[secName]
196
- if not sec:
197
- return S_ERROR(f"{secName} section empty")
198
- return S_OK(sec)
199
-
200
- def setSectionContents(self, secName, contents):
201
- if contents and not isinstance(contents, CFG):
202
- return S_ERROR(f"Contents for section {secName} is not a cfg object")
203
- self.__dirty = True
204
- if secName in self.__manifest:
205
- self.__manifest[secName].reset()
206
- self.__manifest[secName].mergeWith(contents)
207
- else:
208
- self.__manifest.createNewSection(secName, contents=contents)
209
-
210
- def setOption(self, varName, varValue):
211
- """
212
- Set a var in job manifest
213
- """
214
- self.__dirty = True
215
- levels = List.fromChar(varName, "/")
216
- cfg = self.__manifest
217
- for l in levels[:-1]:
218
- if l not in cfg:
219
- cfg.createNewSection(l)
220
- cfg = cfg[l]
221
- cfg.setOption(levels[-1], varValue)
222
-
223
- def remove(self, opName):
224
- levels = List.fromChar(opName, "/")
225
- cfg = self.__manifest
226
- for l in levels[:-1]:
227
- if l not in cfg:
228
- return S_ERROR(f"{opName} does not exist")
229
- cfg = cfg[l]
230
- if cfg.deleteKey(levels[-1]):
231
- self.__dirty = True
232
- return S_OK()
233
- return S_ERROR(f"{opName} does not exist")
234
-
235
- def getOption(self, varName, defaultValue=None):
236
- """
237
- Get a variable from the job manifest
238
- """
239
- cfg = self.__manifest
240
- return cfg.getOption(varName, defaultValue)
241
-
242
- def getOptionList(self, section=""):
243
- """
244
- Get a list of variables in a section of the job manifest
245
- """
246
- cfg = self.__manifest.getRecursive(section)
247
- if not cfg or "value" not in cfg:
248
- return []
249
- cfg = cfg["value"]
250
- return cfg.listOptions()
251
-
252
- def isOption(self, opName):
253
- """
254
- Check if it is a valid option
255
- """
256
- return self.__manifest.isOption(opName)
257
-
258
- def getSectionList(self, section=""):
259
- """
260
- Get a list of sections in the job manifest
261
- """
262
- cfg = self.__manifest.getRecursive(section)
263
- if not cfg or "value" not in cfg:
264
- return []
265
- cfg = cfg["value"]
266
- return cfg.listSections()
37
+ return super().check(config=makeJobManifestConfig(self.__manifest["OwnerGroup"]))
@@ -28,3 +28,6 @@ class JobStateUpdateClient(Client):
28
28
 
29
29
  else:
30
30
  self.serverURL = url
31
+
32
+ def setJobStatus(self, jobID, status="", minorStatus="", source="Unknown", datetime_=None, force=False, **kwargs):
33
+ return self._getRPC(**kwargs).setJobStatus(jobID, status, minorStatus, source, datetime_, force)
@@ -1,154 +1,10 @@
1
- """
2
- This module contains constants and lists for the possible job states.
3
- """
4
-
5
- from DIRAC import gLogger, S_OK, S_ERROR
6
- from DIRAC.Core.Utilities.StateMachine import State, StateMachine
7
- from DIRAC.Core.Utilities.Decorators import deprecated
8
-
9
- from DIRAC.WorkloadManagementSystem.Client.JobMonitoringClient import JobMonitoringClient
10
-
11
-
12
- #:
13
- SUBMITTING = "Submitting"
14
- #:
15
- RECEIVED = "Received"
16
- #:
17
- CHECKING = "Checking"
18
- #:
19
- STAGING = "Staging"
20
- #:
21
- SCOUTING = "Scouting"
22
- #:
23
- WAITING = "Waiting"
24
- #:
25
- MATCHED = "Matched"
26
- #: The Rescheduled status is effectively never stored in the DB.
27
- #: It could be considered a "virtual" status, and might even be dropped.
28
- RESCHEDULED = "Rescheduled"
29
- #:
30
- RUNNING = "Running"
31
- #:
32
- STALLED = "Stalled"
33
- #:
34
- COMPLETING = "Completing"
35
- #:
36
- DONE = "Done"
37
- #:
38
- COMPLETED = "Completed"
39
- #:
40
- FAILED = "Failed"
41
- #:
42
- DELETED = "Deleted"
43
- #:
44
- KILLED = "Killed"
45
-
46
- #: Possible job states
47
- JOB_STATES = [
48
- SUBMITTING,
49
- RECEIVED,
50
- CHECKING,
51
- SCOUTING,
52
- STAGING,
53
- WAITING,
54
- MATCHED,
55
- RESCHEDULED,
56
- RUNNING,
57
- STALLED,
58
- COMPLETING,
59
- DONE,
60
- COMPLETED,
61
- FAILED,
62
- DELETED,
63
- KILLED,
64
- ]
65
-
66
- # Job States when the payload work has finished
67
- JOB_FINAL_STATES = [DONE, COMPLETED, FAILED, KILLED]
68
-
69
- # WMS internal job States indicating the job object won't be updated
70
- JOB_REALLY_FINAL_STATES = [DELETED]
71
-
72
-
73
- class JobsStateMachine(StateMachine):
74
- """Jobs state machine"""
75
-
76
- def __init__(self, state):
77
- """c'tor
78
- Defines the state machine transactions
79
- """
80
- super().__init__(state)
1
+ """Backward compatibility wrapper - moved to DIRACCommon
81
2
 
82
- # States transitions
83
- self.states = {
84
- DELETED: State(15), # final state
85
- KILLED: State(14, [DELETED], defState=KILLED),
86
- FAILED: State(13, [RESCHEDULED, DELETED], defState=FAILED),
87
- DONE: State(12, [DELETED], defState=DONE),
88
- COMPLETED: State(11, [DONE, FAILED], defState=COMPLETED),
89
- COMPLETING: State(10, [DONE, FAILED, COMPLETED, STALLED, KILLED], defState=COMPLETING),
90
- STALLED: State(9, [RUNNING, FAILED, KILLED], defState=STALLED),
91
- RUNNING: State(8, [STALLED, DONE, FAILED, RESCHEDULED, COMPLETING, KILLED, RECEIVED], defState=RUNNING),
92
- RESCHEDULED: State(7, [WAITING, RECEIVED, DELETED, FAILED, KILLED], defState=RESCHEDULED),
93
- MATCHED: State(6, [RUNNING, FAILED, RESCHEDULED, KILLED], defState=MATCHED),
94
- WAITING: State(5, [MATCHED, RESCHEDULED, DELETED, KILLED], defState=WAITING),
95
- STAGING: State(4, [CHECKING, WAITING, FAILED, KILLED], defState=STAGING),
96
- SCOUTING: State(3, [CHECKING, FAILED, STALLED, KILLED], defState=SCOUTING),
97
- CHECKING: State(2, [SCOUTING, STAGING, WAITING, RESCHEDULED, FAILED, DELETED, KILLED], defState=CHECKING),
98
- RECEIVED: State(1, [SCOUTING, CHECKING, STAGING, WAITING, FAILED, DELETED, KILLED], defState=RECEIVED),
99
- SUBMITTING: State(0, [RECEIVED, CHECKING, DELETED, KILLED], defState=SUBMITTING), # initial state
100
- }
3
+ This module has been moved to DIRACCommon.WorkloadManagementSystem.Client.JobStatus to avoid
4
+ circular dependencies and allow DiracX to use these utilities without
5
+ triggering DIRAC's global state initialization.
101
6
 
102
-
103
- @deprecated("Use filterJobStateTransition instead")
104
- def checkJobStateTransition(jobID, candidateState, currentStatus=None, jobMonitoringClient=None):
105
- """Utility to check if a job state transition is allowed"""
106
- if not currentStatus:
107
- if not jobMonitoringClient:
108
- from DIRAC.WorkloadManagementSystem.Client.JobMonitoringClient import JobMonitoringClient
109
-
110
- jobMonitoringClient = JobMonitoringClient()
111
-
112
- res = jobMonitoringClient.getJobsStatus(jobID)
113
- if not res["OK"]:
114
- return res
115
- try:
116
- currentStatus = res["Value"][jobID]["Status"]
117
- except KeyError:
118
- return S_ERROR("Job does not exist")
119
-
120
- res = JobsStateMachine(currentStatus).getNextState(candidateState)
121
- if not res["OK"]:
122
- return res
123
-
124
- # If the JobsStateMachine does not accept the candidate, return an ERROR
125
- if candidateState != res["Value"]:
126
- gLogger.error(
127
- "Job Status Error",
128
- f"{jobID} can't move from {currentStatus} to {candidateState}",
129
- )
130
- return S_ERROR("Job state transition not allowed")
131
- return S_OK()
132
-
133
-
134
- def filterJobStateTransition(jobIDs, candidateState):
135
- """Given a list of jobIDs, return a list that are allowed to transition
136
- to the given candidate state.
137
- """
138
- allowedJobs = []
139
-
140
- if not isinstance(jobIDs, list):
141
- jobIDs = [jobIDs]
142
-
143
- res = JobMonitoringClient().getJobsStatus(jobIDs)
144
- if not res["OK"]:
145
- return res
146
-
147
- for jobID in jobIDs:
148
- if jobID in res["Value"]:
149
- curState = res["Value"][jobID]["Status"]
150
- stateRes = JobsStateMachine(curState).getNextState(candidateState)
151
- if stateRes["OK"]:
152
- if stateRes["Value"] == candidateState:
153
- allowedJobs.append(jobID)
154
- return S_OK(allowedJobs)
7
+ All exports are maintained for backward compatibility.
8
+ """
9
+ # Re-export everything from DIRACCommon for backward compatibility
10
+ from DIRACCommon.WorkloadManagementSystem.Client.JobStatus import * # noqa: F401, F403
@@ -1,13 +1,18 @@
1
1
  """ Client for the SandboxStore.
2
2
  Will connect to the WorkloadManagement/SandboxStore service.
3
3
  """
4
+ from __future__ import annotations
4
5
 
5
6
  import hashlib
6
7
  import os
7
8
  import re
8
9
  import tarfile
9
10
  import tempfile
11
+ from contextlib import contextmanager
10
12
  from io import BytesIO, StringIO
13
+ from typing import Literal
14
+
15
+ import zstandard
11
16
 
12
17
  from DIRAC import S_ERROR, S_OK, gLogger
13
18
  from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getVOForGroup
@@ -18,9 +23,27 @@ from DIRAC.Core.Utilities.ReturnValues import returnSingleResult
18
23
  from DIRAC.Resources.Storage.StorageElement import StorageElement
19
24
 
20
25
 
26
+ @contextmanager
27
+ def ZstdCompatibleTarFile(tarFileName: os.PathLike, *, mode: Literal["r"] = "r"):
28
+ """Context manager to extend tarfile.open to support zstd compressed files.
29
+
30
+ This is only needed for Python <=3.13.
31
+ """
32
+ with open(tarFileName, "rb") as f:
33
+ magic = f.read(4)
34
+ # Read magic bytes to determine compression format
35
+ if magic.startswith(b"\x28\xb5\x2f\xfd"): # zstd magic number
36
+ dctx = zstandard.ZstdDecompressor()
37
+ with open(tarFileName, "rb") as f, dctx.stream_reader(f) as decompressor:
38
+ with tarfile.open(fileobj=decompressor, mode=f"{mode}|") as tf:
39
+ yield tf
40
+ else:
41
+ with tarfile.open(name=tarFileName, mode=mode) as tf:
42
+ yield tf
43
+
44
+
21
45
  class SandboxStoreClient:
22
46
  __validSandboxTypes = ("Input", "Output")
23
- __smdb = None
24
47
 
25
48
  def __init__(self, rpcClient=None, transferClient=None, smdb=False, **kwargs):
26
49
  """Constructor
@@ -37,21 +60,8 @@ class SandboxStoreClient:
37
60
  self.__transferClient = transferClient
38
61
  self.__kwargs = kwargs
39
62
  self.__vo = None
40
- SandboxStoreClient.__smdb = smdb
41
63
  if "delegatedGroup" in kwargs:
42
64
  self.__vo = getVOForGroup(kwargs["delegatedGroup"])
43
- if SandboxStoreClient.__smdb is True:
44
- try:
45
- from DIRAC.WorkloadManagementSystem.DB.SandboxMetadataDB import SandboxMetadataDB
46
-
47
- SandboxStoreClient.__smdb = SandboxMetadataDB()
48
- result = SandboxStoreClient.__smdb._getConnection() # pylint: disable=protected-access
49
- if not result["OK"]:
50
- SandboxStoreClient.__smdb = False
51
- else:
52
- result["Value"].close()
53
- except (ImportError, RuntimeError, AttributeError):
54
- SandboxStoreClient.__smdb = False
55
65
 
56
66
  def __getRPCClient(self):
57
67
  """Get an RPC client for SB service"""
@@ -206,7 +216,7 @@ class SandboxStoreClient:
206
216
 
207
217
  try:
208
218
  sandboxSize = 0
209
- with tarfile.open(name=tarFileName, mode="r") as tf:
219
+ with ZstdCompatibleTarFile(tarFileName, mode="r") as tf:
210
220
  for tarinfo in tf:
211
221
  tf.extract(tarinfo, path=destinationDir)
212
222
  sandboxSize += tarinfo.size
@@ -227,29 +237,6 @@ class SandboxStoreClient:
227
237
  ##############
228
238
  # Jobs
229
239
 
230
- def assignSandboxesToJob(self, jobId, sbList, ownerName="", ownerGroup=""):
231
- """
232
- Assign sandboxes to a job.
233
- sbList must be a list of sandboxes and relation types
234
- sbList = [ ( "SB:SEName|SEPFN", "Input" ), ( "SB:SEName|SEPFN", "Output" ) ]
235
- """
236
- eId = f"Job:{jobId}"
237
- for sbT in sbList:
238
- if sbT[1] not in self.__validSandboxTypes:
239
- return S_ERROR(f"Invalid Sandbox type {sbT[1]}")
240
- if SandboxStoreClient.__smdb and ownerName and ownerGroup:
241
- return SandboxStoreClient.__smdb.assignSandboxesToEntities({eId: sbList}, ownerName, ownerGroup)
242
- return self.__getRPCClient().assignSandboxesToEntities({eId: sbList}, ownerName, ownerGroup)
243
-
244
- def unassignJobs(self, jobIdList):
245
- """Unassign SB to a job"""
246
- if isinstance(jobIdList, int):
247
- jobIdList = [jobIdList]
248
- entitiesList = []
249
- for jobId in jobIdList:
250
- entitiesList.append(f"Job:{jobId}")
251
- return self.__getRPCClient().unassignEntities(entitiesList)
252
-
253
240
  def downloadSandboxForJob(self, jobId, sbType, destinationPath="", inMemory=False, unpack=True):
254
241
  """Download SB for a job"""
255
242
  result = self.__getRPCClient().getSandboxesAssignedToEntity(f"Job:{jobId}")
@@ -2,11 +2,10 @@
2
2
  methods necessary to communicate with the Workload Management System
3
3
  """
4
4
  import os
5
- from io import StringIO
6
5
  import time
6
+ from io import StringIO
7
7
 
8
- from DIRAC import S_OK, S_ERROR, gLogger
9
-
8
+ from DIRAC import S_ERROR, S_OK, gLogger
10
9
  from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
11
10
  from DIRAC.Core.Utilities import File
12
11
  from DIRAC.Core.Utilities.ClassAd.ClassAdLight import ClassAd
@@ -1,6 +1,8 @@
1
1
  """Test for WMS clients."""
2
2
  # pylint: disable=protected-access, missing-docstring, invalid-name
3
3
 
4
+ from pathlib import Path
5
+ import shutil
4
6
  import pytest
5
7
 
6
8
  from unittest.mock import MagicMock
@@ -280,3 +282,30 @@ def test_DLI_execute_NoLocal(mocker, dli, mockSE):
280
282
  assert res["Value"]["Failed"]
281
283
  assert "/a/lfn/1.txt" in res["Value"]["Failed"], res
282
284
  assert res["Value"]["Failed"][0] == "/a/lfn/1.txt", res
285
+
286
+
287
+ def test_DLI_execute_jobIDPath(mocker, dli, mockSE):
288
+ """Specify a jobIDPath. Output should be in the jobIDPath directory."""
289
+ mocker.patch("DIRAC.WorkloadManagementSystem.Client.DownloadInputData.gConfig.getValue", return_value=2)
290
+
291
+ # Create the jobIDPath directory
292
+ jobIDPath = Path().cwd() / "job" / "12345"
293
+ jobIDPath.mkdir(parents=True, exist_ok=True)
294
+
295
+ dli.configuration["JobIDPath"] = str(jobIDPath)
296
+
297
+ mocker.patch("DIRAC.WorkloadManagementSystem.Client.DownloadInputData.gConfig.getValue", return_value=2)
298
+ mockObjectSE = mockSE.return_value
299
+ mockObjectSE.getFileMetadata.return_value = S_OK(
300
+ {"Successful": {"/a/lfn/1.txt": {"Cached": 1, "Accessible": 0}}, "Failed": {}}
301
+ )
302
+ dli._downloadFromSE = MagicMock(side_effect=[S_ERROR("Failed to down"), S_OK({"path": jobIDPath / "1.txt"})])
303
+ dli._isCache = MagicMock(return_value=True)
304
+ res = dli.execute(dataToResolve=["/a/lfn/1.txt"])
305
+
306
+ assert res["OK"]
307
+ assert not res["Value"]["Failed"]
308
+ assert "/a/lfn/1.txt" in res["Value"]["Successful"], res
309
+
310
+ # Check that the output path is in the jobIDPath directory
311
+ assert res["Value"]["Successful"]["/a/lfn/1.txt"]["path"] == jobIDPath / "1.txt", res
@@ -263,14 +263,10 @@ Agents
263
263
  # the DN of the certificate proxy used to submit pilots. If not found here, what is in Operations/Pilot section of the CS will be used
264
264
  PilotDN =
265
265
 
266
- # List of sites that will be treated by this SiteDirector (No value can refer to any Site defined in the CS)
267
- Site =
268
- # List of CEs that will be treated by this SiteDirector (No value can refer to any CE defined in the CS)
269
- CEs =
270
- # List of CE types that will be treated by this SiteDirector (No value can refer to any type of CE defined in the CS)
271
- CETypes =
272
- # List of Tags that are required to be present in the CE/Queue definition
273
- Tags =
266
+ # Site = # List of CEs that will be treated by this SiteDirector (No value can refer to any CE defined in the CS)
267
+ # CEs = # List of CE types that will be treated by this SiteDirector (No value can refer to any type of CE defined in the CS)
268
+ # CETypes = # List of Tags that are required to be present in the CE/Queue definition
269
+ # Tags =
274
270
 
275
271
  # How many cycles to skip if queue is not working
276
272
  FailedQueueCycleFactor = 10