DIRAC 9.0.0a54__py3-none-any.whl → 9.0.7__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (166) hide show
  1. DIRAC/AccountingSystem/Client/AccountingCLI.py +0 -140
  2. DIRAC/AccountingSystem/Client/DataStoreClient.py +0 -13
  3. DIRAC/AccountingSystem/Client/Types/BaseAccountingType.py +0 -7
  4. DIRAC/AccountingSystem/ConfigTemplate.cfg +0 -5
  5. DIRAC/AccountingSystem/Service/DataStoreHandler.py +0 -72
  6. DIRAC/ConfigurationSystem/Client/Helpers/CSGlobals.py +0 -9
  7. DIRAC/ConfigurationSystem/Client/Helpers/Registry.py +34 -32
  8. DIRAC/ConfigurationSystem/Client/Helpers/Resources.py +11 -43
  9. DIRAC/ConfigurationSystem/Client/Helpers/test/Test_Helpers.py +0 -16
  10. DIRAC/ConfigurationSystem/Client/LocalConfiguration.py +14 -8
  11. DIRAC/ConfigurationSystem/Client/PathFinder.py +47 -8
  12. DIRAC/ConfigurationSystem/Client/SyncPlugins/CERNLDAPSyncPlugin.py +4 -1
  13. DIRAC/ConfigurationSystem/Client/VOMS2CSSynchronizer.py +9 -2
  14. DIRAC/ConfigurationSystem/Client/test/Test_PathFinder.py +41 -1
  15. DIRAC/ConfigurationSystem/private/RefresherBase.py +4 -2
  16. DIRAC/Core/DISET/ServiceReactor.py +11 -3
  17. DIRAC/Core/DISET/private/BaseClient.py +1 -2
  18. DIRAC/Core/DISET/private/Transports/M2SSLTransport.py +9 -7
  19. DIRAC/Core/Security/DiracX.py +12 -7
  20. DIRAC/Core/Security/IAMService.py +4 -3
  21. DIRAC/Core/Security/ProxyInfo.py +9 -5
  22. DIRAC/Core/Security/test/test_diracx_token_from_pem.py +161 -0
  23. DIRAC/Core/Tornado/Client/ClientSelector.py +4 -1
  24. DIRAC/Core/Tornado/Server/TornadoService.py +1 -1
  25. DIRAC/Core/Utilities/ClassAd/ClassAdLight.py +4 -290
  26. DIRAC/Core/Utilities/DErrno.py +5 -309
  27. DIRAC/Core/Utilities/Extensions.py +10 -1
  28. DIRAC/Core/Utilities/Graphs/GraphData.py +1 -1
  29. DIRAC/Core/Utilities/JDL.py +1 -195
  30. DIRAC/Core/Utilities/List.py +1 -124
  31. DIRAC/Core/Utilities/MySQL.py +101 -97
  32. DIRAC/Core/Utilities/Os.py +32 -1
  33. DIRAC/Core/Utilities/Platform.py +2 -107
  34. DIRAC/Core/Utilities/ReturnValues.py +7 -252
  35. DIRAC/Core/Utilities/StateMachine.py +12 -178
  36. DIRAC/Core/Utilities/TimeUtilities.py +10 -253
  37. DIRAC/Core/Utilities/test/Test_JDL.py +0 -3
  38. DIRAC/Core/Utilities/test/Test_Profiler.py +20 -20
  39. DIRAC/Core/scripts/dirac_agent.py +1 -1
  40. DIRAC/Core/scripts/dirac_apptainer_exec.py +16 -7
  41. DIRAC/Core/scripts/dirac_platform.py +1 -92
  42. DIRAC/DataManagementSystem/Agent/FTS3Agent.py +8 -7
  43. DIRAC/DataManagementSystem/Agent/RequestOperations/RemoveFile.py +7 -6
  44. DIRAC/DataManagementSystem/Client/FTS3Job.py +71 -34
  45. DIRAC/DataManagementSystem/DB/FTS3DB.py +3 -0
  46. DIRAC/DataManagementSystem/DB/FileCatalogComponents/DatasetManager/DatasetManager.py +1 -1
  47. DIRAC/DataManagementSystem/Utilities/DMSHelpers.py +6 -2
  48. DIRAC/DataManagementSystem/scripts/dirac_dms_create_moving_request.py +2 -0
  49. DIRAC/DataManagementSystem/scripts/dirac_dms_protocol_matrix.py +0 -1
  50. DIRAC/FrameworkSystem/Client/ComponentInstaller.py +4 -2
  51. DIRAC/FrameworkSystem/DB/ProxyDB.py +9 -5
  52. DIRAC/FrameworkSystem/Utilities/TokenManagementUtilities.py +3 -2
  53. DIRAC/FrameworkSystem/Utilities/diracx.py +2 -74
  54. DIRAC/FrameworkSystem/private/authorization/AuthServer.py +2 -2
  55. DIRAC/FrameworkSystem/scripts/dirac_login.py +2 -2
  56. DIRAC/FrameworkSystem/scripts/dirac_proxy_init.py +1 -1
  57. DIRAC/Interfaces/API/Dirac.py +27 -13
  58. DIRAC/Interfaces/API/DiracAdmin.py +42 -7
  59. DIRAC/Interfaces/API/Job.py +1 -0
  60. DIRAC/Interfaces/scripts/dirac_admin_allow_site.py +7 -1
  61. DIRAC/Interfaces/scripts/dirac_admin_ban_site.py +7 -1
  62. DIRAC/Interfaces/scripts/dirac_wms_job_parameters.py +0 -1
  63. DIRAC/MonitoringSystem/Client/Types/WMSHistory.py +4 -0
  64. DIRAC/MonitoringSystem/Client/WebAppClient.py +26 -0
  65. DIRAC/MonitoringSystem/ConfigTemplate.cfg +9 -0
  66. DIRAC/MonitoringSystem/DB/MonitoringDB.py +6 -25
  67. DIRAC/MonitoringSystem/Service/MonitoringHandler.py +0 -33
  68. DIRAC/MonitoringSystem/Service/WebAppHandler.py +599 -0
  69. DIRAC/MonitoringSystem/private/MainReporter.py +0 -3
  70. DIRAC/ProductionSystem/scripts/dirac_prod_get_trans.py +2 -3
  71. DIRAC/RequestManagementSystem/Agent/RequestExecutingAgent.py +8 -6
  72. DIRAC/RequestManagementSystem/ConfigTemplate.cfg +6 -6
  73. DIRAC/RequestManagementSystem/DB/test/RMSTestScenari.py +2 -0
  74. DIRAC/ResourceStatusSystem/Client/SiteStatus.py +4 -2
  75. DIRAC/ResourceStatusSystem/Command/FreeDiskSpaceCommand.py +3 -1
  76. DIRAC/ResourceStatusSystem/Utilities/CSHelpers.py +2 -31
  77. DIRAC/ResourceStatusSystem/scripts/dirac_rss_set_status.py +18 -4
  78. DIRAC/Resources/Catalog/RucioFileCatalogClient.py +1 -1
  79. DIRAC/Resources/Computing/AREXComputingElement.py +19 -3
  80. DIRAC/Resources/Computing/BatchSystems/Condor.py +126 -108
  81. DIRAC/Resources/Computing/BatchSystems/SLURM.py +5 -1
  82. DIRAC/Resources/Computing/BatchSystems/test/Test_SLURM.py +46 -0
  83. DIRAC/Resources/Computing/HTCondorCEComputingElement.py +37 -43
  84. DIRAC/Resources/Computing/SingularityComputingElement.py +6 -1
  85. DIRAC/Resources/Computing/test/Test_HTCondorCEComputingElement.py +67 -49
  86. DIRAC/Resources/Computing/test/Test_PoolComputingElement.py +2 -1
  87. DIRAC/Resources/IdProvider/CheckInIdProvider.py +13 -0
  88. DIRAC/Resources/IdProvider/IdProviderFactory.py +11 -3
  89. DIRAC/Resources/Storage/StorageBase.py +4 -2
  90. DIRAC/Resources/Storage/StorageElement.py +4 -4
  91. DIRAC/TransformationSystem/Agent/TaskManagerAgentBase.py +10 -16
  92. DIRAC/TransformationSystem/Agent/TransformationAgent.py +22 -1
  93. DIRAC/TransformationSystem/Agent/TransformationCleaningAgent.py +15 -15
  94. DIRAC/TransformationSystem/Client/Transformation.py +2 -1
  95. DIRAC/TransformationSystem/Client/TransformationClient.py +0 -7
  96. DIRAC/TransformationSystem/Client/Utilities.py +9 -0
  97. DIRAC/TransformationSystem/Service/TransformationManagerHandler.py +0 -336
  98. DIRAC/TransformationSystem/Utilities/ReplicationCLIParameters.py +3 -3
  99. DIRAC/TransformationSystem/scripts/dirac_production_runjoblocal.py +2 -4
  100. DIRAC/TransformationSystem/test/Test_replicationTransformation.py +5 -6
  101. DIRAC/Workflow/Modules/test/Test_Modules.py +5 -0
  102. DIRAC/WorkloadManagementSystem/Agent/JobAgent.py +1 -5
  103. DIRAC/WorkloadManagementSystem/Agent/JobCleaningAgent.py +11 -7
  104. DIRAC/WorkloadManagementSystem/Agent/PilotSyncAgent.py +4 -3
  105. DIRAC/WorkloadManagementSystem/Agent/PushJobAgent.py +13 -13
  106. DIRAC/WorkloadManagementSystem/Agent/SiteDirector.py +10 -13
  107. DIRAC/WorkloadManagementSystem/Agent/StalledJobAgent.py +18 -51
  108. DIRAC/WorkloadManagementSystem/Agent/StatesAccountingAgent.py +41 -1
  109. DIRAC/WorkloadManagementSystem/Agent/test/Test_Agent_JobAgent.py +2 -0
  110. DIRAC/WorkloadManagementSystem/Agent/test/Test_Agent_JobCleaningAgent.py +7 -9
  111. DIRAC/WorkloadManagementSystem/Agent/test/Test_Agent_PushJobAgent.py +1 -0
  112. DIRAC/WorkloadManagementSystem/Agent/test/Test_Agent_SiteDirector.py +8 -2
  113. DIRAC/WorkloadManagementSystem/Agent/test/Test_Agent_StalledJobAgent.py +4 -5
  114. DIRAC/WorkloadManagementSystem/Client/DownloadInputData.py +7 -5
  115. DIRAC/WorkloadManagementSystem/Client/JobMonitoringClient.py +10 -11
  116. DIRAC/WorkloadManagementSystem/Client/JobState/JobManifest.py +32 -261
  117. DIRAC/WorkloadManagementSystem/Client/JobStateUpdateClient.py +3 -0
  118. DIRAC/WorkloadManagementSystem/Client/JobStatus.py +8 -152
  119. DIRAC/WorkloadManagementSystem/Client/SandboxStoreClient.py +25 -38
  120. DIRAC/WorkloadManagementSystem/Client/WMSClient.py +2 -3
  121. DIRAC/WorkloadManagementSystem/Client/test/Test_Client_DownloadInputData.py +29 -0
  122. DIRAC/WorkloadManagementSystem/ConfigTemplate.cfg +4 -8
  123. DIRAC/WorkloadManagementSystem/DB/JobDB.py +40 -69
  124. DIRAC/WorkloadManagementSystem/DB/JobDBUtils.py +18 -147
  125. DIRAC/WorkloadManagementSystem/DB/JobParametersDB.py +9 -9
  126. DIRAC/WorkloadManagementSystem/DB/PilotAgentsDB.py +3 -2
  127. DIRAC/WorkloadManagementSystem/DB/SandboxMetadataDB.py +28 -39
  128. DIRAC/WorkloadManagementSystem/DB/StatusUtils.py +125 -0
  129. DIRAC/WorkloadManagementSystem/DB/tests/Test_JobDB.py +1 -1
  130. DIRAC/WorkloadManagementSystem/DB/tests/Test_StatusUtils.py +28 -0
  131. DIRAC/WorkloadManagementSystem/Executor/JobSanity.py +3 -3
  132. DIRAC/WorkloadManagementSystem/FutureClient/JobStateUpdateClient.py +2 -14
  133. DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py +14 -9
  134. DIRAC/WorkloadManagementSystem/JobWrapper/test/Test_JobWrapper.py +36 -10
  135. DIRAC/WorkloadManagementSystem/JobWrapper/test/Test_JobWrapperTemplate.py +4 -0
  136. DIRAC/WorkloadManagementSystem/Service/JobManagerHandler.py +33 -154
  137. DIRAC/WorkloadManagementSystem/Service/JobMonitoringHandler.py +5 -323
  138. DIRAC/WorkloadManagementSystem/Service/JobStateUpdateHandler.py +0 -16
  139. DIRAC/WorkloadManagementSystem/Service/PilotManagerHandler.py +6 -102
  140. DIRAC/WorkloadManagementSystem/Service/SandboxStoreHandler.py +5 -51
  141. DIRAC/WorkloadManagementSystem/Service/WMSAdministratorHandler.py +16 -79
  142. DIRAC/WorkloadManagementSystem/Utilities/JobModel.py +28 -199
  143. DIRAC/WorkloadManagementSystem/Utilities/JobParameters.py +65 -3
  144. DIRAC/WorkloadManagementSystem/Utilities/JobStatusUtility.py +2 -64
  145. DIRAC/WorkloadManagementSystem/Utilities/ParametricJob.py +7 -171
  146. DIRAC/WorkloadManagementSystem/Utilities/PilotCStoJSONSynchronizer.py +73 -7
  147. DIRAC/WorkloadManagementSystem/Utilities/PilotWrapper.py +2 -0
  148. DIRAC/WorkloadManagementSystem/Utilities/RemoteRunner.py +16 -0
  149. DIRAC/WorkloadManagementSystem/Utilities/Utils.py +36 -1
  150. DIRAC/WorkloadManagementSystem/Utilities/jobAdministration.py +15 -0
  151. DIRAC/WorkloadManagementSystem/Utilities/test/Test_JobModel.py +1 -5
  152. DIRAC/WorkloadManagementSystem/Utilities/test/Test_ParametricJob.py +45 -128
  153. DIRAC/WorkloadManagementSystem/Utilities/test/Test_PilotWrapper.py +16 -0
  154. DIRAC/__init__.py +55 -54
  155. {dirac-9.0.0a54.dist-info → dirac-9.0.7.dist-info}/METADATA +6 -4
  156. {dirac-9.0.0a54.dist-info → dirac-9.0.7.dist-info}/RECORD +160 -160
  157. {dirac-9.0.0a54.dist-info → dirac-9.0.7.dist-info}/WHEEL +1 -1
  158. {dirac-9.0.0a54.dist-info → dirac-9.0.7.dist-info}/entry_points.txt +0 -3
  159. DIRAC/Core/Utilities/test/Test_List.py +0 -150
  160. DIRAC/Core/Utilities/test/Test_Time.py +0 -88
  161. DIRAC/TransformationSystem/scripts/dirac_transformation_archive.py +0 -30
  162. DIRAC/TransformationSystem/scripts/dirac_transformation_clean.py +0 -30
  163. DIRAC/TransformationSystem/scripts/dirac_transformation_remove_output.py +0 -30
  164. DIRAC/WorkloadManagementSystem/Utilities/test/Test_JobManager.py +0 -58
  165. {dirac-9.0.0a54.dist-info → dirac-9.0.7.dist-info}/licenses/LICENSE +0 -0
  166. {dirac-9.0.0a54.dist-info → dirac-9.0.7.dist-info}/top_level.txt +0 -0
@@ -4,176 +4,12 @@
4
4
  getParameterVectorLength() - to get the total size of the bunch of parametric jobs
5
5
  generateParametricJobs() - to get a list of expanded descriptions of all the jobs
6
6
  """
7
- import re
8
7
 
9
- from DIRAC.Core.Utilities.ClassAd.ClassAdLight import ClassAd
10
- from DIRAC.Core.Utilities.ReturnValues import S_OK, S_ERROR
11
- from DIRAC.Core.Utilities.DErrno import EWMSJDL
8
+ # Import from DIRACCommon for backward compatibility
9
+ from DIRACCommon.WorkloadManagementSystem.Utilities.ParametricJob import (
10
+ getParameterVectorLength,
11
+ generateParametricJobs,
12
+ )
12
13
 
13
-
14
- def __getParameterSequence(nPar, parList=[], parStart=1, parStep=0, parFactor=1):
15
- if parList:
16
- if nPar != len(parList):
17
- return []
18
- else:
19
- parameterList = list(parList)
20
- else:
21
- # The first parameter must have the same type as the other ones even if not defined explicitly
22
- parameterList = [parStart * type(parFactor)(1) + type(parStep)(0)]
23
- for np in range(1, nPar):
24
- parameterList.append(parameterList[np - 1] * parFactor + parStep)
25
-
26
- return parameterList
27
-
28
-
29
- def getParameterVectorLength(jobClassAd):
30
- """Get the length of parameter vector in the parametric job description
31
-
32
- :param jobClassAd: ClassAd job description object
33
- :return: result structure with the Value: int number of parameter values, None if not a parametric job
34
- """
35
-
36
- nParValues = None
37
- attributes = jobClassAd.getAttributes()
38
- for attribute in attributes:
39
- if attribute.startswith("Parameters"):
40
- if jobClassAd.isAttributeList(attribute):
41
- parameterList = jobClassAd.getListFromExpression(attribute)
42
- nThisParValues = len(parameterList)
43
- else:
44
- nThisParValues = jobClassAd.getAttributeInt(attribute)
45
- if nParValues is not None and nParValues != nThisParValues:
46
- return S_ERROR(
47
- EWMSJDL,
48
- "Different length of parameter vectors: for %s, %s != %d" % (attribute, nParValues, nThisParValues),
49
- )
50
- nParValues = nThisParValues
51
- if nParValues is not None and nParValues <= 0:
52
- return S_ERROR(EWMSJDL, "Illegal number of job parameters %d" % (nParValues))
53
- return S_OK(nParValues)
54
-
55
-
56
- def __updateAttribute(classAd, attribute, parName, parValue):
57
- # If there is something to do:
58
- pattern = r"%%\(%s\)s" % parName
59
- if parName == "0":
60
- pattern = "%s"
61
- expr = classAd.get_expression(attribute)
62
- if not re.search(pattern, expr):
63
- return False
64
-
65
- pattern = "%%(%s)s" % parName
66
- if parName == "0":
67
- pattern = "%s"
68
-
69
- parValue = parValue.strip()
70
- if classAd.isAttributeList(attribute):
71
- parValue = parValue.strip()
72
- if parValue.startswith("{"):
73
- parValue = parValue.lstrip("{").rstrip("}").strip()
74
-
75
- expr = classAd.get_expression(attribute)
76
- newexpr = expr.replace(pattern, str(parValue))
77
- classAd.set_expression(attribute, newexpr)
78
- return True
79
-
80
-
81
- def generateParametricJobs(jobClassAd):
82
- """Generate a series of ClassAd job descriptions expanding
83
- job parameters
84
-
85
- :param jobClassAd: ClassAd job description object
86
- :return: list of ClassAd job description objects
87
- """
88
- if not jobClassAd.lookupAttribute("Parameters"):
89
- return S_OK([jobClassAd.asJDL()])
90
-
91
- result = getParameterVectorLength(jobClassAd)
92
- if not result["OK"]:
93
- return result
94
- nParValues = result["Value"]
95
- if nParValues is None:
96
- return S_ERROR(EWMSJDL, "Can not determine the number of job parameters")
97
-
98
- parameterDict = {}
99
- attributes = jobClassAd.getAttributes()
100
- for attribute in attributes:
101
- for key in ["Parameters", "ParameterStart", "ParameterStep", "ParameterFactor"]:
102
- if attribute.startswith(key):
103
- seqID = "0" if "." not in attribute else attribute.split(".")[1]
104
- parameterDict.setdefault(seqID, {})
105
- if key == "Parameters":
106
- if jobClassAd.isAttributeList(attribute):
107
- parList = jobClassAd.getListFromExpression(attribute)
108
- if len(parList) != nParValues:
109
- return S_ERROR(EWMSJDL, "Inconsistent parametric job description")
110
- parameterDict[seqID]["ParameterList"] = parList
111
- else:
112
- if attribute != "Parameters":
113
- return S_ERROR(EWMSJDL, "Inconsistent parametric job description")
114
- nPar = jobClassAd.getAttributeInt(attribute)
115
- if nPar is None:
116
- value = jobClassAd.get_expression(attribute)
117
- return S_ERROR(EWMSJDL, f"Inconsistent parametric job description: {attribute}={value}")
118
- parameterDict[seqID]["Parameters"] = nPar
119
- else:
120
- value = jobClassAd.getAttributeInt(attribute)
121
- if value is None:
122
- value = jobClassAd.getAttributeFloat(attribute)
123
- if value is None:
124
- value = jobClassAd.get_expression(attribute)
125
- return S_ERROR(f"Illegal value for {attribute} JDL field: {value}")
126
- parameterDict[seqID][key] = value
127
-
128
- if "0" in parameterDict and not parameterDict.get("0"):
129
- parameterDict.pop("0")
130
-
131
- parameterLists = {}
132
- for seqID in parameterDict:
133
- parList = __getParameterSequence(
134
- nParValues,
135
- parList=parameterDict[seqID].get("ParameterList", []),
136
- parStart=parameterDict[seqID].get("ParameterStart", 1),
137
- parStep=parameterDict[seqID].get("ParameterStep", 0),
138
- parFactor=parameterDict[seqID].get("ParameterFactor", 1),
139
- )
140
- if not parList:
141
- return S_ERROR(EWMSJDL, "Inconsistent parametric job description")
142
-
143
- parameterLists[seqID] = parList
144
-
145
- jobDescList = []
146
- jobDesc = jobClassAd.asJDL()
147
- # Width of the sequential parameter number
148
- zLength = len(str(nParValues - 1))
149
- for n in range(nParValues):
150
- newJobDesc = jobDesc
151
- newJobDesc = newJobDesc.replace("%n", str(n).zfill(zLength))
152
- newClassAd = ClassAd(newJobDesc)
153
- for seqID in parameterLists:
154
- parameter = parameterLists[seqID][n]
155
- for attribute in newClassAd.getAttributes():
156
- __updateAttribute(newClassAd, attribute, seqID, str(parameter))
157
-
158
- for seqID in parameterLists:
159
- for attribute in ["Parameters", "ParameterStart", "ParameterStep", "ParameterFactor"]:
160
- if seqID == "0":
161
- newClassAd.deleteAttribute(attribute)
162
- else:
163
- newClassAd.deleteAttribute(f"{attribute}.{seqID}")
164
-
165
- parameter = parameterLists[seqID][n]
166
- if seqID == "0":
167
- attribute = "Parameter"
168
- else:
169
- attribute = f"Parameter.{seqID}"
170
- if isinstance(parameter, str) and parameter.startswith("{"):
171
- newClassAd.insertAttributeInt(attribute, str(parameter))
172
- else:
173
- newClassAd.insertAttributeString(attribute, str(parameter))
174
-
175
- newClassAd.insertAttributeInt("ParameterNumber", n)
176
- newJDL = newClassAd.asJDL()
177
- jobDescList.append(newJDL)
178
-
179
- return S_OK(jobDescList)
14
+ # Re-export for backward compatibility
15
+ __all__ = ["getParameterVectorLength", "generateParametricJobs"]
@@ -1,16 +1,16 @@
1
- """ CStoJSONSynchronizer
2
- Module that keeps the pilot parameters file synchronized with the information
3
- in the Operations/Pilot section of the CS. If there are additions in the CS,
4
- these are incorporated to the file.
5
- The module uploads to a web server the latest version of the pilot scripts.
1
+ """CStoJSONSynchronizer
2
+ Module that keeps the pilot parameters file synchronized with the information
3
+ in the Operations/Pilot section of the CS. If there are additions in the CS,
4
+ these are incorporated to the file.
5
+ The module uploads to a web server the latest version of the pilot scripts.
6
6
  """
7
+
7
8
  import datetime
8
9
  import glob
9
10
  import os
10
11
  import shutil
11
12
  import tarfile
12
13
  from typing import Any
13
-
14
14
  from git import Repo
15
15
 
16
16
  from DIRAC import S_OK, gConfig, gLogger
@@ -19,6 +19,67 @@ from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
19
19
  from DIRAC.ConfigurationSystem.Client.Helpers.Path import cfgPath
20
20
  from DIRAC.Core.Utilities.ReturnValues import DOKReturnType, DReturnType
21
21
 
22
+ import socket
23
+ from urllib.parse import urlparse
24
+
25
+
26
+ def exclude_master_cs_aliases(urls: list[str], master_cs_url: str) -> list[str]:
27
+ """
28
+ Excludes URLs that are DNS aliases of the given MasterCS server URL.
29
+
30
+ This function resolves the IP addresses of the MasterCS server and each URL in the input list.
31
+ It returns a new list containing only those URLs whose hostnames do not resolve to any of the
32
+ MasterCS server's IP addresses, effectively excluding all DNS aliases of the MasterCS server.
33
+
34
+ Args:
35
+ urls (list[str]): A list of URLs to filter. Each URL should be a string in a valid URL format.
36
+ master_cs_url (str): The reference URL (e.g., MasterCS server URL) whose DNS aliases are to be excluded.
37
+
38
+ Returns:
39
+ list[str]: A new list of URLs with all aliases of the MasterCS server removed.
40
+ If the MasterCS hostname cannot be resolved, the original list is returned unchanged.
41
+
42
+ Example:
43
+ >>> urls = [
44
+ ... 'dips://lbvobox303.cern.ch:9135/Configuration/Server',
45
+ ... 'dips://ccwlcglhcb02.in2p3.fr:9135/Configuration/Server',
46
+ ... 'dips://lbvobox302.cern.ch:9135/Configuration/Server',
47
+ ... ]
48
+ >>> master_cs_url = "dips://mastercs.cern.ch:9135/Configuration/Server"
49
+ >>> exclude_master_cs_aliases(urls, master_cs_url)
50
+ ['dips://ccwlcglhcb02.in2p3.fr:9135/Configuration/Server']
51
+
52
+ Notes:
53
+ - If the MasterCS hostname cannot be resolved, the function returns the original list.
54
+ - If a hostname in the input list cannot be resolved, it is included in the result.
55
+ - The comparison is based on IP addresses, not hostnames.
56
+ """
57
+ master_cs_hostname = urlparse(master_cs_url).hostname
58
+ if not master_cs_hostname:
59
+ return urls
60
+
61
+ # Resolve IP addresses for the MasterCS hostname
62
+ try:
63
+ master_cs_ips = set(socket.gethostbyname_ex(master_cs_hostname)[2])
64
+ except socket.gaierror:
65
+ return urls
66
+
67
+ # Function to get IPs for a hostname
68
+ def get_ips(hostname):
69
+ try:
70
+ return set(socket.gethostbyname_ex(hostname)[2])
71
+ except socket.gaierror:
72
+ return set()
73
+
74
+ filtered_urls = []
75
+ for url in urls:
76
+ hostname = urlparse(url).hostname
77
+ ips = get_ips(hostname)
78
+ if not ips & master_cs_ips:
79
+ filtered_urls.append(url)
80
+
81
+ return filtered_urls
82
+
22
83
 
23
84
  class PilotCStoJSONSynchronizer:
24
85
  """
@@ -151,9 +212,14 @@ class PilotCStoJSONSynchronizer:
151
212
  configurationServers = gConfig.getServersList()
152
213
  if not includeMasterCS:
153
214
  masterCS = gConfigurationData.getMasterServer()
154
- configurationServers = list(set(configurationServers) - {masterCS})
215
+ configurationServers = exclude_master_cs_aliases(configurationServers, masterCS)
216
+
155
217
  pilotDict["ConfigurationServers"] = configurationServers
156
218
 
219
+ preferredURLPatterns = gConfigurationData.extractOptionFromCFG("/DIRAC/PreferredURLPatterns")
220
+ if preferredURLPatterns:
221
+ pilotDict["PreferredURLPatterns"] = preferredURLPatterns
222
+
157
223
  self.log.debug("Got pilotDict", str(pilotDict))
158
224
 
159
225
  return S_OK(pilotDict)
@@ -417,6 +417,8 @@ def getPilotFilesCompressedEncodedDict(pilotFiles, proxy=None):
417
417
  from DIRAC.Core.Security.ProxyFile import writeChainToTemporaryFile # pylint: disable=import-error
418
418
 
419
419
  retVal = writeChainToTemporaryFile(proxy)
420
+ if not retVal["OK"]:
421
+ raise RuntimeError("Failed to write proxy to temporary file: %s" % retVal["Message"])
420
422
  proxyLocation = Path(retVal["Value"])
421
423
  proxy_string = proxyLocation.read_text()
422
424
  proxyLocation.unlink()
@@ -49,6 +49,11 @@ class RemoteRunner:
49
49
  f"site {self._workloadSite}, CE {self._workloadCE}, queue {self._workloadQueue}",
50
50
  )
51
51
 
52
+ # The CE interface needs to drop the token section from the proxy file to interact with the CE
53
+ # So we save the current proxy file location (which likely contains the DiracX token)
54
+ # and we will restore it at the end of the job
55
+ originalProxyLocation = os.environ.get("X509_USER_PROXY")
56
+
52
57
  # Set up Application Queue
53
58
  if not (result := self._setUpWorkloadCE(numberOfProcessors))["OK"]:
54
59
  result["Errno"] = DErrno.ERESUNA
@@ -87,6 +92,8 @@ class RemoteRunner:
87
92
  time.sleep(timeBetweenRetries)
88
93
  else:
89
94
  result["Errno"] = DErrno.EWMSSUBM
95
+ # Restore the original proxy location
96
+ os.environ["X509_USER_PROXY"] = originalProxyLocation
90
97
  return result
91
98
 
92
99
  jobID = result["Value"][0]
@@ -107,6 +114,8 @@ class RemoteRunner:
107
114
  time.sleep(timeBetweenRetries)
108
115
  else:
109
116
  result["Errno"] = DErrno.EWMSSTATUS
117
+ # Restore the original proxy location
118
+ os.environ["X509_USER_PROXY"] = originalProxyLocation
110
119
  return result
111
120
 
112
121
  jobStatus = result["Value"][jobID]
@@ -123,6 +132,8 @@ class RemoteRunner:
123
132
  time.sleep(timeBetweenRetries)
124
133
  else:
125
134
  result["Errno"] = DErrno.EWMSJMAN
135
+ # Restore the original proxy location
136
+ os.environ["X509_USER_PROXY"] = originalProxyLocation
126
137
  return result
127
138
 
128
139
  output, error = result["Value"]
@@ -131,6 +142,8 @@ class RemoteRunner:
131
142
  self.log.info("Checking the integrity of the outputs...")
132
143
  if not (result := self._checkOutputIntegrity("."))["OK"]:
133
144
  result["Errno"] = DErrno.EWMSJMAN
145
+ # Restore the original proxy location
146
+ os.environ["X509_USER_PROXY"] = originalProxyLocation
134
147
  return result
135
148
  self.log.info("The output has been retrieved and declared complete")
136
149
 
@@ -146,6 +159,9 @@ class RemoteRunner:
146
159
  self.log.warn("Failed to clean the output remotely", result["Message"])
147
160
  self.log.info("The job has been remotely removed")
148
161
 
162
+ # Restore the original proxy location
163
+ os.environ["X509_USER_PROXY"] = originalProxyLocation
164
+
149
165
  commandStatus = {"Done": 0, "Failed": -1, "Killed": -2}
150
166
  return S_OK((commandStatus[jobStatus], output, error))
151
167
 
@@ -4,9 +4,12 @@ import os
4
4
  import sys
5
5
  import json
6
6
 
7
- from DIRAC import gLogger, S_OK
7
+ from DIRAC import gLogger, S_OK, S_ERROR
8
8
  from DIRAC.Core.Utilities.File import mkDir
9
9
  from DIRAC.FrameworkSystem.private.standardLogging.Logging import Logging
10
+ from DIRAC.WorkloadManagementSystem.DB.JobLoggingDB import JobLoggingDB
11
+ from DIRAC.WorkloadManagementSystem.DB.JobDB import JobDB
12
+ from DIRAC.WorkloadManagementSystem.DB.TaskQueueDB import TaskQueueDB
10
13
 
11
14
 
12
15
  def createJobWrapper(
@@ -113,3 +116,35 @@ def createJobWrapper(
113
116
  if rootLocation != wrapperPath:
114
117
  generatedFiles["JobExecutableRelocatedPath"] = os.path.join(rootLocation, os.path.basename(jobExeFile))
115
118
  return S_OK(generatedFiles)
119
+
120
+
121
+ def rescheduleJobs(jobIDs: list[int], source: str = "") -> dict:
122
+ """Utility to reschedule jobs (not atomic, nor bulk)
123
+ Requires direct access to the JobDB and TaskQueueDB
124
+
125
+ :param jobIDs: list of jobIDs
126
+ :param source: source of the reschedule
127
+ :return: S_OK/S_ERROR
128
+ :rtype: dict
129
+
130
+ """
131
+
132
+ failedJobs = []
133
+
134
+ for jobID in jobIDs:
135
+ result = JobDB().rescheduleJob(jobID)
136
+ if not result["OK"]:
137
+ failedJobs.append(jobID)
138
+ continue
139
+ TaskQueueDB().deleteJob(jobID)
140
+ JobLoggingDB().addLoggingRecord(
141
+ result["JobID"],
142
+ status=result["Status"],
143
+ minorStatus=result["MinorStatus"],
144
+ applicationStatus="Unknown",
145
+ source=source,
146
+ )
147
+
148
+ if failedJobs:
149
+ return S_ERROR(f"Failed to reschedule jobs {failedJobs}")
150
+ return S_OK()
@@ -0,0 +1,15 @@
1
+ from DIRAC.WorkloadManagementSystem.Client import JobStatus
2
+
3
+
4
+ def _filterJobStateTransition(jobStates, candidateState):
5
+ """Given a dictionary of jobs states,
6
+ return a list of jobs that are allowed to transition to the given candidate state.
7
+ """
8
+ allowedJobs = []
9
+
10
+ for js in jobStates.items():
11
+ stateRes = JobStatus.JobsStateMachine(js[1]["Status"]).getNextState(candidateState)
12
+ if stateRes["OK"]:
13
+ if stateRes["Value"] == candidateState:
14
+ allowedJobs.append(js[0])
15
+ return allowedJobs
@@ -175,11 +175,7 @@ def test_logLevelValidator_invalid():
175
175
 
176
176
  def test_platformValidator_valid():
177
177
  """Test the platform validator with valid input."""
178
- with patch(
179
- "DIRAC.WorkloadManagementSystem.Utilities.JobModel.getDIRACPlatforms",
180
- return_value=S_OK(["x86_64-slc6-gcc62-opt"]),
181
- ):
182
- job = BaseJobDescriptionModel(executable=EXECUTABLE, platform="x86_64-slc6-gcc62-opt")
178
+ job = BaseJobDescriptionModel(executable=EXECUTABLE, platform="x86_64-slc6-gcc62-opt")
183
179
  assert job.platform == "x86_64-slc6-gcc62-opt"
184
180
 
185
181
 
@@ -1,138 +1,55 @@
1
- """ This is a test of the parametric job generation tools
2
- """
1
+ """ This is a test of the parametric job generation tools"""
3
2
  # pylint: disable= missing-docstring
4
3
 
5
4
  import pytest
6
5
 
6
+ # Test imports from DIRAC to verify backward compatibility
7
7
  from DIRAC.WorkloadManagementSystem.Utilities.ParametricJob import generateParametricJobs, getParameterVectorLength
8
8
  from DIRAC.Core.Utilities.ClassAd.ClassAdLight import ClassAd
9
9
 
10
- TEST_JDL_NO_PARAMETERS = """
11
- [
12
- Executable = "my_executable";
13
- Arguments = "%s";
14
- JobName = "Test_%n";
15
- ]
16
- """
17
10
 
18
- TEST_JDL_SIMPLE = """
19
- [
20
- Executable = "my_executable";
21
- Arguments = "%s";
22
- JobName = "Test_%n";
23
- Parameters = { "a", "b", "c" }
24
- ]
25
- """
26
-
27
- TEST_JDL_SIMPLE_BUNCH = """
28
- [
29
- Executable = "my_executable";
30
- Arguments = "%s";
31
- JobName = "Test_%n";
32
- Parameters = 3;
33
- ParameterStart = 5;
34
- ]
35
- """
36
-
37
- TEST_JDL_SIMPLE_PROGRESSION = """
38
- [
39
- Executable = "my_executable";
40
- Arguments = "%s";
41
- JobName = "Test_%n";
42
- Parameters = 3;
43
- ParameterStart = 1;
44
- ParameterStep = 1;
45
- ParameterFactor = 2;
46
- ]
47
- """
48
-
49
- TEST_JDL_MULTI = """
50
- [
51
- Executable = "my_executable";
52
- Arguments = "%(A)s %(B)s";
53
- JobName = "Test_%n";
54
- Parameters = 3;
55
- ParameterStart.A = 1;
56
- ParameterStep.A = 1;
57
- ParameterFactor.A = 2;
58
- Parameters.B = { "a","b","c" };
59
- ]
60
- """
61
-
62
- TEST_JDL_MULTI_BAD = """
63
- [
64
- Executable = "my_executable";
65
- Arguments = "%(A)s %(B)s";
66
- JobName = "Test_%n";
67
- Parameters = 3;
68
- ParameterStart.A = 1;
69
- ParameterStep.A = 1;
70
- ParameterFactor.A = 2;
71
- Parameters.B = { "a","b","c","d" };
72
- ]
73
- """
74
-
75
-
76
- @pytest.mark.parametrize(
77
- "jdl, expectedArguments",
78
- [
79
- (TEST_JDL_SIMPLE, ["a", "b", "c"]),
80
- (TEST_JDL_SIMPLE_BUNCH, ["5", "5", "5"]),
81
- (TEST_JDL_SIMPLE_PROGRESSION, ["1", "3", "7"]),
82
- (TEST_JDL_MULTI, ["1 a", "3 b", "7 c"]),
83
- (TEST_JDL_NO_PARAMETERS, []),
84
- ],
85
- )
86
- def test_getParameterVectorLength_successful(jdl: str, expectedArguments: list[str]):
11
+ def test_backward_compatibility_import():
12
+ """Test that imports from DIRAC still work (backward compatibility)"""
87
13
  # Arrange
88
- jobDescription = ClassAd(jdl)
89
-
90
- # Act
91
- result = getParameterVectorLength(jobDescription)
92
-
93
- # Assert
94
- assert result["OK"], result["Message"]
95
- if expectedArguments:
96
- assert result["Value"] == len(expectedArguments)
97
- else:
98
- assert result["Value"] == None
99
-
100
-
101
- @pytest.mark.parametrize("jdl", [TEST_JDL_MULTI_BAD])
102
- def test_getParameterVectorLength_unsuccessful(jdl: str):
103
- # Arrange
104
- jobDescription = ClassAd(jdl)
105
-
106
- # Act
107
- result = getParameterVectorLength(jobDescription)
108
-
109
- # Assert
110
- assert not result["OK"], result["Value"]
111
-
112
-
113
- @pytest.mark.parametrize(
114
- "jdl, expectedArguments",
14
+ jdl = """
115
15
  [
116
- (TEST_JDL_SIMPLE, ["a", "b", "c"]),
117
- (TEST_JDL_SIMPLE_BUNCH, ["5", "5", "5"]),
118
- (TEST_JDL_SIMPLE_PROGRESSION, ["1", "3", "7"]),
119
- (TEST_JDL_MULTI, ["1 a", "3 b", "7 c"]),
120
- ],
121
- )
122
- def test_generateParametricJobs(jdl: str, expectedArguments: list[str]):
123
- # Arrange
124
- parametricJobDescription = ClassAd(jdl)
125
-
126
- # Act
127
- result = generateParametricJobs(parametricJobDescription)
128
-
129
- # Assert
130
- assert result["OK"], result["Message"]
131
- assert result["Value"]
132
- jobDescList = result["Value"]
133
- assert len(jobDescList) == len(expectedArguments)
134
-
135
- for i in range(len(jobDescList)):
136
- jobDescription = ClassAd(jobDescList[i])
137
- assert jobDescription.getAttributeString("JobName") == f"Test_{i}"
138
- assert jobDescription.getAttributeString("Arguments") == expectedArguments[i]
16
+ Executable = "my_executable";
17
+ Arguments = "%s";
18
+ JobName = "Test_%n";
19
+ Parameters = { "a", "b", "c" }
20
+ ]
21
+ """
22
+
23
+ # Act - Test that we can import and use the functions from DIRAC
24
+ jobDescription = ClassAd(jdl)
25
+ vector_result = getParameterVectorLength(jobDescription)
26
+ generate_result = generateParametricJobs(jobDescription)
27
+
28
+ # Assert - Verify functions work correctly
29
+ assert vector_result["OK"]
30
+ assert vector_result["Value"] == 3
31
+ assert generate_result["OK"]
32
+ assert len(generate_result["Value"]) == 3
33
+
34
+
35
+ # Import and run the comprehensive tests from DIRACCommon to avoid duplication
36
+ # This ensures the DIRAC re-exports work with the full test suite
37
+ try:
38
+ from DIRACCommon.tests.WorkloadManagementSystem.Utilities.test_ParametricJob import (
39
+ test_getParameterVectorLength_successful,
40
+ test_getParameterVectorLength_unsuccessful,
41
+ test_generateParametricJobs,
42
+ )
43
+
44
+ # Re-export the DIRACCommon tests so they run as part of DIRAC test suite
45
+ # This validates that the backward compatibility imports work correctly
46
+ __all__ = [
47
+ "test_backward_compatibility_import",
48
+ "test_getParameterVectorLength_successful",
49
+ "test_getParameterVectorLength_unsuccessful",
50
+ "test_generateParametricJobs",
51
+ ]
52
+
53
+ except ImportError:
54
+ # If DIRACCommon tests can't be imported, just run the backward compatibility test
55
+ __all__ = ["test_backward_compatibility_import"]
@@ -136,3 +136,19 @@ def test_scriptPilot3_4():
136
136
  assert 'os.environ["someName"]="someValue"' in res
137
137
  assert "lhcb-portal.cern.ch" in res
138
138
  assert """locations += ["file:/cvmfs/dirac.egi.eu/pilot"]""" in res
139
+
140
+
141
+ def test_scriptPilot3_5():
142
+ """test script creation"""
143
+ res = pilotWrapperScript(
144
+ pilotFilesCompressedEncodedDict={"proxy": "thisIsSomeProxy"},
145
+ pilotOptions="-l LHCb --architectureScript=dirac-architecture --CVMFS_locations=/cvmfs/lhcb.cern.ch -e LHCb -N atlasce1.lnf.infn.it -Q condor -n LCG.Frascati.it --wnVO=lhcb --architectureScript=dirac-apptainer-exec dirac-architecture -o lbRunOnly",
146
+ envVariables={"someName": "someValue", "someMore": "oneMore"},
147
+ location="lhcb-portal.cern.ch",
148
+ CVMFS_locations=[],
149
+ )
150
+ assert 'os.environ["someName"]="someValue"' in res
151
+ assert "lhcb-portal.cern.ch" in res
152
+ assert """locations += ["file:/cvmfs/dirac.egi.eu/pilot"]""" in res
153
+ assert "dirac-architecture" in res
154
+ assert "dirac-apptainer-exec dirac-architecture" in res