DIRAC 9.0.0a54__py3-none-any.whl → 9.0.7__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (166) hide show
  1. DIRAC/AccountingSystem/Client/AccountingCLI.py +0 -140
  2. DIRAC/AccountingSystem/Client/DataStoreClient.py +0 -13
  3. DIRAC/AccountingSystem/Client/Types/BaseAccountingType.py +0 -7
  4. DIRAC/AccountingSystem/ConfigTemplate.cfg +0 -5
  5. DIRAC/AccountingSystem/Service/DataStoreHandler.py +0 -72
  6. DIRAC/ConfigurationSystem/Client/Helpers/CSGlobals.py +0 -9
  7. DIRAC/ConfigurationSystem/Client/Helpers/Registry.py +34 -32
  8. DIRAC/ConfigurationSystem/Client/Helpers/Resources.py +11 -43
  9. DIRAC/ConfigurationSystem/Client/Helpers/test/Test_Helpers.py +0 -16
  10. DIRAC/ConfigurationSystem/Client/LocalConfiguration.py +14 -8
  11. DIRAC/ConfigurationSystem/Client/PathFinder.py +47 -8
  12. DIRAC/ConfigurationSystem/Client/SyncPlugins/CERNLDAPSyncPlugin.py +4 -1
  13. DIRAC/ConfigurationSystem/Client/VOMS2CSSynchronizer.py +9 -2
  14. DIRAC/ConfigurationSystem/Client/test/Test_PathFinder.py +41 -1
  15. DIRAC/ConfigurationSystem/private/RefresherBase.py +4 -2
  16. DIRAC/Core/DISET/ServiceReactor.py +11 -3
  17. DIRAC/Core/DISET/private/BaseClient.py +1 -2
  18. DIRAC/Core/DISET/private/Transports/M2SSLTransport.py +9 -7
  19. DIRAC/Core/Security/DiracX.py +12 -7
  20. DIRAC/Core/Security/IAMService.py +4 -3
  21. DIRAC/Core/Security/ProxyInfo.py +9 -5
  22. DIRAC/Core/Security/test/test_diracx_token_from_pem.py +161 -0
  23. DIRAC/Core/Tornado/Client/ClientSelector.py +4 -1
  24. DIRAC/Core/Tornado/Server/TornadoService.py +1 -1
  25. DIRAC/Core/Utilities/ClassAd/ClassAdLight.py +4 -290
  26. DIRAC/Core/Utilities/DErrno.py +5 -309
  27. DIRAC/Core/Utilities/Extensions.py +10 -1
  28. DIRAC/Core/Utilities/Graphs/GraphData.py +1 -1
  29. DIRAC/Core/Utilities/JDL.py +1 -195
  30. DIRAC/Core/Utilities/List.py +1 -124
  31. DIRAC/Core/Utilities/MySQL.py +101 -97
  32. DIRAC/Core/Utilities/Os.py +32 -1
  33. DIRAC/Core/Utilities/Platform.py +2 -107
  34. DIRAC/Core/Utilities/ReturnValues.py +7 -252
  35. DIRAC/Core/Utilities/StateMachine.py +12 -178
  36. DIRAC/Core/Utilities/TimeUtilities.py +10 -253
  37. DIRAC/Core/Utilities/test/Test_JDL.py +0 -3
  38. DIRAC/Core/Utilities/test/Test_Profiler.py +20 -20
  39. DIRAC/Core/scripts/dirac_agent.py +1 -1
  40. DIRAC/Core/scripts/dirac_apptainer_exec.py +16 -7
  41. DIRAC/Core/scripts/dirac_platform.py +1 -92
  42. DIRAC/DataManagementSystem/Agent/FTS3Agent.py +8 -7
  43. DIRAC/DataManagementSystem/Agent/RequestOperations/RemoveFile.py +7 -6
  44. DIRAC/DataManagementSystem/Client/FTS3Job.py +71 -34
  45. DIRAC/DataManagementSystem/DB/FTS3DB.py +3 -0
  46. DIRAC/DataManagementSystem/DB/FileCatalogComponents/DatasetManager/DatasetManager.py +1 -1
  47. DIRAC/DataManagementSystem/Utilities/DMSHelpers.py +6 -2
  48. DIRAC/DataManagementSystem/scripts/dirac_dms_create_moving_request.py +2 -0
  49. DIRAC/DataManagementSystem/scripts/dirac_dms_protocol_matrix.py +0 -1
  50. DIRAC/FrameworkSystem/Client/ComponentInstaller.py +4 -2
  51. DIRAC/FrameworkSystem/DB/ProxyDB.py +9 -5
  52. DIRAC/FrameworkSystem/Utilities/TokenManagementUtilities.py +3 -2
  53. DIRAC/FrameworkSystem/Utilities/diracx.py +2 -74
  54. DIRAC/FrameworkSystem/private/authorization/AuthServer.py +2 -2
  55. DIRAC/FrameworkSystem/scripts/dirac_login.py +2 -2
  56. DIRAC/FrameworkSystem/scripts/dirac_proxy_init.py +1 -1
  57. DIRAC/Interfaces/API/Dirac.py +27 -13
  58. DIRAC/Interfaces/API/DiracAdmin.py +42 -7
  59. DIRAC/Interfaces/API/Job.py +1 -0
  60. DIRAC/Interfaces/scripts/dirac_admin_allow_site.py +7 -1
  61. DIRAC/Interfaces/scripts/dirac_admin_ban_site.py +7 -1
  62. DIRAC/Interfaces/scripts/dirac_wms_job_parameters.py +0 -1
  63. DIRAC/MonitoringSystem/Client/Types/WMSHistory.py +4 -0
  64. DIRAC/MonitoringSystem/Client/WebAppClient.py +26 -0
  65. DIRAC/MonitoringSystem/ConfigTemplate.cfg +9 -0
  66. DIRAC/MonitoringSystem/DB/MonitoringDB.py +6 -25
  67. DIRAC/MonitoringSystem/Service/MonitoringHandler.py +0 -33
  68. DIRAC/MonitoringSystem/Service/WebAppHandler.py +599 -0
  69. DIRAC/MonitoringSystem/private/MainReporter.py +0 -3
  70. DIRAC/ProductionSystem/scripts/dirac_prod_get_trans.py +2 -3
  71. DIRAC/RequestManagementSystem/Agent/RequestExecutingAgent.py +8 -6
  72. DIRAC/RequestManagementSystem/ConfigTemplate.cfg +6 -6
  73. DIRAC/RequestManagementSystem/DB/test/RMSTestScenari.py +2 -0
  74. DIRAC/ResourceStatusSystem/Client/SiteStatus.py +4 -2
  75. DIRAC/ResourceStatusSystem/Command/FreeDiskSpaceCommand.py +3 -1
  76. DIRAC/ResourceStatusSystem/Utilities/CSHelpers.py +2 -31
  77. DIRAC/ResourceStatusSystem/scripts/dirac_rss_set_status.py +18 -4
  78. DIRAC/Resources/Catalog/RucioFileCatalogClient.py +1 -1
  79. DIRAC/Resources/Computing/AREXComputingElement.py +19 -3
  80. DIRAC/Resources/Computing/BatchSystems/Condor.py +126 -108
  81. DIRAC/Resources/Computing/BatchSystems/SLURM.py +5 -1
  82. DIRAC/Resources/Computing/BatchSystems/test/Test_SLURM.py +46 -0
  83. DIRAC/Resources/Computing/HTCondorCEComputingElement.py +37 -43
  84. DIRAC/Resources/Computing/SingularityComputingElement.py +6 -1
  85. DIRAC/Resources/Computing/test/Test_HTCondorCEComputingElement.py +67 -49
  86. DIRAC/Resources/Computing/test/Test_PoolComputingElement.py +2 -1
  87. DIRAC/Resources/IdProvider/CheckInIdProvider.py +13 -0
  88. DIRAC/Resources/IdProvider/IdProviderFactory.py +11 -3
  89. DIRAC/Resources/Storage/StorageBase.py +4 -2
  90. DIRAC/Resources/Storage/StorageElement.py +4 -4
  91. DIRAC/TransformationSystem/Agent/TaskManagerAgentBase.py +10 -16
  92. DIRAC/TransformationSystem/Agent/TransformationAgent.py +22 -1
  93. DIRAC/TransformationSystem/Agent/TransformationCleaningAgent.py +15 -15
  94. DIRAC/TransformationSystem/Client/Transformation.py +2 -1
  95. DIRAC/TransformationSystem/Client/TransformationClient.py +0 -7
  96. DIRAC/TransformationSystem/Client/Utilities.py +9 -0
  97. DIRAC/TransformationSystem/Service/TransformationManagerHandler.py +0 -336
  98. DIRAC/TransformationSystem/Utilities/ReplicationCLIParameters.py +3 -3
  99. DIRAC/TransformationSystem/scripts/dirac_production_runjoblocal.py +2 -4
  100. DIRAC/TransformationSystem/test/Test_replicationTransformation.py +5 -6
  101. DIRAC/Workflow/Modules/test/Test_Modules.py +5 -0
  102. DIRAC/WorkloadManagementSystem/Agent/JobAgent.py +1 -5
  103. DIRAC/WorkloadManagementSystem/Agent/JobCleaningAgent.py +11 -7
  104. DIRAC/WorkloadManagementSystem/Agent/PilotSyncAgent.py +4 -3
  105. DIRAC/WorkloadManagementSystem/Agent/PushJobAgent.py +13 -13
  106. DIRAC/WorkloadManagementSystem/Agent/SiteDirector.py +10 -13
  107. DIRAC/WorkloadManagementSystem/Agent/StalledJobAgent.py +18 -51
  108. DIRAC/WorkloadManagementSystem/Agent/StatesAccountingAgent.py +41 -1
  109. DIRAC/WorkloadManagementSystem/Agent/test/Test_Agent_JobAgent.py +2 -0
  110. DIRAC/WorkloadManagementSystem/Agent/test/Test_Agent_JobCleaningAgent.py +7 -9
  111. DIRAC/WorkloadManagementSystem/Agent/test/Test_Agent_PushJobAgent.py +1 -0
  112. DIRAC/WorkloadManagementSystem/Agent/test/Test_Agent_SiteDirector.py +8 -2
  113. DIRAC/WorkloadManagementSystem/Agent/test/Test_Agent_StalledJobAgent.py +4 -5
  114. DIRAC/WorkloadManagementSystem/Client/DownloadInputData.py +7 -5
  115. DIRAC/WorkloadManagementSystem/Client/JobMonitoringClient.py +10 -11
  116. DIRAC/WorkloadManagementSystem/Client/JobState/JobManifest.py +32 -261
  117. DIRAC/WorkloadManagementSystem/Client/JobStateUpdateClient.py +3 -0
  118. DIRAC/WorkloadManagementSystem/Client/JobStatus.py +8 -152
  119. DIRAC/WorkloadManagementSystem/Client/SandboxStoreClient.py +25 -38
  120. DIRAC/WorkloadManagementSystem/Client/WMSClient.py +2 -3
  121. DIRAC/WorkloadManagementSystem/Client/test/Test_Client_DownloadInputData.py +29 -0
  122. DIRAC/WorkloadManagementSystem/ConfigTemplate.cfg +4 -8
  123. DIRAC/WorkloadManagementSystem/DB/JobDB.py +40 -69
  124. DIRAC/WorkloadManagementSystem/DB/JobDBUtils.py +18 -147
  125. DIRAC/WorkloadManagementSystem/DB/JobParametersDB.py +9 -9
  126. DIRAC/WorkloadManagementSystem/DB/PilotAgentsDB.py +3 -2
  127. DIRAC/WorkloadManagementSystem/DB/SandboxMetadataDB.py +28 -39
  128. DIRAC/WorkloadManagementSystem/DB/StatusUtils.py +125 -0
  129. DIRAC/WorkloadManagementSystem/DB/tests/Test_JobDB.py +1 -1
  130. DIRAC/WorkloadManagementSystem/DB/tests/Test_StatusUtils.py +28 -0
  131. DIRAC/WorkloadManagementSystem/Executor/JobSanity.py +3 -3
  132. DIRAC/WorkloadManagementSystem/FutureClient/JobStateUpdateClient.py +2 -14
  133. DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py +14 -9
  134. DIRAC/WorkloadManagementSystem/JobWrapper/test/Test_JobWrapper.py +36 -10
  135. DIRAC/WorkloadManagementSystem/JobWrapper/test/Test_JobWrapperTemplate.py +4 -0
  136. DIRAC/WorkloadManagementSystem/Service/JobManagerHandler.py +33 -154
  137. DIRAC/WorkloadManagementSystem/Service/JobMonitoringHandler.py +5 -323
  138. DIRAC/WorkloadManagementSystem/Service/JobStateUpdateHandler.py +0 -16
  139. DIRAC/WorkloadManagementSystem/Service/PilotManagerHandler.py +6 -102
  140. DIRAC/WorkloadManagementSystem/Service/SandboxStoreHandler.py +5 -51
  141. DIRAC/WorkloadManagementSystem/Service/WMSAdministratorHandler.py +16 -79
  142. DIRAC/WorkloadManagementSystem/Utilities/JobModel.py +28 -199
  143. DIRAC/WorkloadManagementSystem/Utilities/JobParameters.py +65 -3
  144. DIRAC/WorkloadManagementSystem/Utilities/JobStatusUtility.py +2 -64
  145. DIRAC/WorkloadManagementSystem/Utilities/ParametricJob.py +7 -171
  146. DIRAC/WorkloadManagementSystem/Utilities/PilotCStoJSONSynchronizer.py +73 -7
  147. DIRAC/WorkloadManagementSystem/Utilities/PilotWrapper.py +2 -0
  148. DIRAC/WorkloadManagementSystem/Utilities/RemoteRunner.py +16 -0
  149. DIRAC/WorkloadManagementSystem/Utilities/Utils.py +36 -1
  150. DIRAC/WorkloadManagementSystem/Utilities/jobAdministration.py +15 -0
  151. DIRAC/WorkloadManagementSystem/Utilities/test/Test_JobModel.py +1 -5
  152. DIRAC/WorkloadManagementSystem/Utilities/test/Test_ParametricJob.py +45 -128
  153. DIRAC/WorkloadManagementSystem/Utilities/test/Test_PilotWrapper.py +16 -0
  154. DIRAC/__init__.py +55 -54
  155. {dirac-9.0.0a54.dist-info → dirac-9.0.7.dist-info}/METADATA +6 -4
  156. {dirac-9.0.0a54.dist-info → dirac-9.0.7.dist-info}/RECORD +160 -160
  157. {dirac-9.0.0a54.dist-info → dirac-9.0.7.dist-info}/WHEEL +1 -1
  158. {dirac-9.0.0a54.dist-info → dirac-9.0.7.dist-info}/entry_points.txt +0 -3
  159. DIRAC/Core/Utilities/test/Test_List.py +0 -150
  160. DIRAC/Core/Utilities/test/Test_Time.py +0 -88
  161. DIRAC/TransformationSystem/scripts/dirac_transformation_archive.py +0 -30
  162. DIRAC/TransformationSystem/scripts/dirac_transformation_clean.py +0 -30
  163. DIRAC/TransformationSystem/scripts/dirac_transformation_remove_output.py +0 -30
  164. DIRAC/WorkloadManagementSystem/Utilities/test/Test_JobManager.py +0 -58
  165. {dirac-9.0.0a54.dist-info → dirac-9.0.7.dist-info}/licenses/LICENSE +0 -0
  166. {dirac-9.0.0a54.dist-info → dirac-9.0.7.dist-info}/top_level.txt +0 -0
@@ -1,7 +1,12 @@
1
1
  """ Collection of utilities for finding paths in the CS
2
2
  """
3
+ import re
4
+ from copy import deepcopy
5
+ from collections.abc import Iterable
3
6
  from urllib import parse
4
7
 
8
+ from cachetools import cached, TTLCache
9
+
5
10
  from DIRAC.Core.Utilities import List
6
11
  from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData
7
12
  from DIRAC.ConfigurationSystem.Client.Helpers import Path
@@ -151,6 +156,43 @@ def getSystemURLs(system, failover=False):
151
156
  return urlDict
152
157
 
153
158
 
159
+ def groupURLsByPriority(urls: Iterable[str]) -> list[set[str]]:
160
+ """Group URLs by priority.
161
+
162
+ :param Iterable[str] preferredURLPatterns: patterns to check in ranked order
163
+ :param set[str] urls: URLs to check
164
+
165
+ :return: list[set[str]] -- list of URL groups, ordered by priority
166
+ """
167
+ return deepcopy(_groupURLsByPriority(frozenset(urls)))
168
+
169
+
170
+ @cached(cache=TTLCache(maxsize=1024, ttl=300))
171
+ def _groupURLsByPriority(urls: frozenset[str]) -> list[set[str]]:
172
+ preferredURLPatterns = []
173
+ if patterns := gConfigurationData.extractOptionFromCFG("/DIRAC/PreferredURLPatterns"):
174
+ preferredURLPatterns = [re.compile(pattern) for pattern in List.fromChar(patterns)]
175
+
176
+ urlGroups = [set() for _ in range(len(preferredURLPatterns) + 1)]
177
+ for url in urls:
178
+ urlGroups[findURLPriority(preferredURLPatterns, url)].add(url)
179
+ return urlGroups
180
+
181
+
182
+ def findURLPriority(preferredURLPatterns: list[re.Pattern[str]], url: str) -> int:
183
+ """Find which preferred URL pattern the URL matches.
184
+
185
+ :param str preferredURLPatterns: patterns to check in ranked order
186
+ :param str url: URL to check
187
+
188
+ :return: int -- index of the pattern that matched, smallest is the most preferred
189
+ """
190
+ for i, pattern in enumerate(preferredURLPatterns):
191
+ if re.match(pattern, url):
192
+ return i
193
+ return len(preferredURLPatterns)
194
+
195
+
154
196
  def getServiceURLs(system, service=None, failover=False):
155
197
  """Generate url.
156
198
 
@@ -168,8 +210,8 @@ def getServiceURLs(system, service=None, failover=False):
168
210
  # Add failover URLs at the end of the list
169
211
  failover = "Failover" if failover else ""
170
212
  for fURLs in ["", "Failover"] if failover else [""]:
171
- urlList = []
172
213
  urls = List.fromChar(gConfigurationData.extractOptionFromCFG(f"{systemSection}/{fURLs}URLs/{service}"))
214
+ urlList = set()
173
215
 
174
216
  # Be sure that urls not None
175
217
  for url in urls or []:
@@ -186,16 +228,13 @@ def getServiceURLs(system, service=None, failover=False):
186
228
 
187
229
  for srv in mainServers:
188
230
  _url = checkComponentURL(url.replace("$MAINSERVERS$", srv), system, service, pathMandatory=True)
189
- if _url not in urlList:
190
- urlList.append(_url)
231
+ urlList.add(_url)
191
232
  continue
192
233
 
193
- _url = checkComponentURL(url, system, service, pathMandatory=True)
194
- if _url not in urlList:
195
- urlList.append(_url)
234
+ urlList.add(checkComponentURL(url, system, service, pathMandatory=True))
196
235
 
197
- # Randomize list if needed
198
- resList.extend(List.randomize(urlList))
236
+ for urlGroup in groupURLsByPriority(urlList):
237
+ resList.extend(List.randomize(urlGroup))
199
238
 
200
239
  return resList
201
240
 
@@ -32,6 +32,9 @@ class CERNLDAPSyncPlugin:
32
32
  else:
33
33
  userDict["PrimaryCERNAccount"] = self._findOwnerAccountName(username, attributes)
34
34
 
35
+ if userDict["CERNAccountType"] in ["Primary", "Secondary"]:
36
+ userDict["CERNPersonId"] = attributes.get("employeeId", [None])[0]
37
+
35
38
  def _findOwnerAccountName(self, username, attributes):
36
39
  """Find the owner account from a CERN LDAP entry.
37
40
 
@@ -64,7 +67,7 @@ class CERNLDAPSyncPlugin:
64
67
  status, result, response, _ = self._connection.search(
65
68
  "OU=Users,OU=Organic Units,DC=cern,DC=ch",
66
69
  f"(CN={commonName})",
67
- attributes=["cernAccountOwner", "cernAccountType"],
70
+ attributes=["cernAccountOwner", "cernAccountType", "employeeId"],
68
71
  )
69
72
  if not status:
70
73
  raise ValueError(f"Bad status from LDAP search: {result}")
@@ -87,8 +87,15 @@ def _getUserNameFromDN(dn, vo):
87
87
  return nname
88
88
  else:
89
89
  robot = False
90
+ # only pop if the remains are sufficient (i.e. not just digits)
90
91
  if names[0].lower().startswith("robot"):
91
- names.pop(0)
92
+ nameok = False
93
+ if len(names) > 1:
94
+ for name in names[1:]:
95
+ if not name.isdigit():
96
+ nameok = True
97
+ if nameok:
98
+ names.pop(0)
92
99
  robot = True
93
100
  for name in list(names):
94
101
  if name[0].isdigit() or "@" in name:
@@ -587,7 +594,7 @@ class VOMS2CSSynchronizer:
587
594
 
588
595
  # Try to fill in the DiracX section
589
596
  if self.useIAM:
590
- iam_subs = self.iamSrv.getUsersSub()
597
+ iam_subs = self.iamSrv.getUsersSub(self.vo)
591
598
  diracx_vo_config = {"DiracX": {"CsSync": {"VOs": {self.vo: {"UserSubjects": iam_subs}}}}}
592
599
  iam_sub_cfg = CFG()
593
600
  iam_sub_cfg.loadFromDict(diracx_vo_config)
@@ -11,9 +11,26 @@ from DIRAC.ConfigurationSystem.private.ConfigurationData import ConfigurationDat
11
11
  localCFGData = ConfigurationData(False)
12
12
  mergedCFG = CFG()
13
13
  mergedCFG.loadFromBuffer(
14
- """
14
+ r"""
15
+ DIRAC
16
+ {
17
+ PreferredURLPatterns = dips://.*\.site:.*
18
+ PreferredURLPatterns += dips://.*\.other:.*
19
+ }
15
20
  Systems
16
21
  {
22
+ Configuration
23
+ {
24
+ URLs
25
+ {
26
+ Server = dips://server1.site:1234/Configuration/Server
27
+ Server += dips://server2.site:1234/Configuration/Server
28
+ Server += dips://server3.site:1234/Configuration/Server
29
+ Server += dips://server4.site:1234/Configuration/Server
30
+ Server += dips://server.other:1234/Configuration/Server
31
+ Server += dips://server.external:1234/Configuration/Server
32
+ }
33
+ }
17
34
  WorkloadManagement
18
35
  {
19
36
  URLs
@@ -181,6 +198,29 @@ def test_getServiceURLs(pathFinder, serviceName, service, failover, result):
181
198
  assert set(pathFinder.getServiceURLs(serviceName, service=service, failover=failover)) == result
182
199
 
183
200
 
201
+ def test_getServiceURLsOrdering(pathFinder):
202
+ """Ensure the PreferredURLPattern option is respected"""
203
+ all_results = set()
204
+ for _ in range(10_000):
205
+ urls = pathFinder.getServiceURLs("Configuration", service="Server")
206
+ assert set(urls) == {
207
+ "dips://server1.site:1234/Configuration/Server",
208
+ "dips://server2.site:1234/Configuration/Server",
209
+ "dips://server3.site:1234/Configuration/Server",
210
+ "dips://server4.site:1234/Configuration/Server",
211
+ "dips://server.other:1234/Configuration/Server",
212
+ "dips://server.external:1234/Configuration/Server",
213
+ }
214
+ # The second to last URL should always be "other"
215
+ assert urls[-2] == "dips://server.other:1234/Configuration/Server"
216
+ # The last URL should always be the one which isn't preferred
217
+ assert urls[-1] == "dips://server.external:1234/Configuration/Server"
218
+ all_results.add(tuple(urls))
219
+ # There are 4! = 24 possible orderings of the preferred URLs, we should have seen all
220
+ # of them at least once in 10_000 iterations
221
+ assert len(all_results) >= 24
222
+
223
+
184
224
  @pytest.mark.parametrize(
185
225
  "system, failover, result",
186
226
  [
@@ -1,7 +1,7 @@
1
1
  import time
2
2
 
3
3
  from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData
4
- from DIRAC.ConfigurationSystem.Client.PathFinder import getGatewayURLs
4
+ from DIRAC.ConfigurationSystem.Client.PathFinder import getGatewayURLs, groupURLsByPriority
5
5
  from DIRAC.Core.Utilities import List
6
6
  from DIRAC.Core.Utilities.EventDispatcher import gEventDispatcher
7
7
  from DIRAC.Core.Utilities.ReturnValues import S_ERROR, S_OK
@@ -138,7 +138,9 @@ class RefresherBase:
138
138
  if not initialServerList:
139
139
  return S_OK()
140
140
 
141
- randomServerList = List.randomize(initialServerList)
141
+ randomServerList = []
142
+ for urlGroup in groupURLsByPriority(initialServerList):
143
+ randomServerList.extend(List.randomize(urlGroup))
142
144
  gLogger.debug(f"Randomized server list is {', '.join(randomServerList)}")
143
145
 
144
146
  for sServer in randomServerList:
@@ -200,6 +200,7 @@ class ServiceReactor:
200
200
  services at the same time
201
201
  """
202
202
  sel = self.__getListeningSelector(svcName)
203
+ throttleExpires = None
203
204
  while self.__alive:
204
205
  clientTransport = None
205
206
  try:
@@ -223,12 +224,19 @@ class ServiceReactor:
223
224
  gLogger.warn(f"Client connected from banned ip {clientIP}")
224
225
  clientTransport.close()
225
226
  continue
227
+ # Handle throttling
228
+ if self.__services[svcName].wantsThrottle and throttleExpires is None:
229
+ throttleExpires = time.time() + THROTTLE_SERVICE_SLEEP_SECONDS
230
+ if throttleExpires:
231
+ if time.time() > throttleExpires:
232
+ throttleExpires = None
233
+ else:
234
+ gLogger.warn("Rejecting client due to throttling", str(clientTransport.getRemoteAddress()))
235
+ clientTransport.close()
236
+ continue
226
237
  # Handle connection
227
238
  self.__stats.connectionStablished()
228
239
  self.__services[svcName].handleConnection(clientTransport)
229
- while self.__services[svcName].wantsThrottle:
230
- gLogger.warn("Sleeping as service requested throttling", svcName)
231
- time.sleep(THROTTLE_SERVICE_SLEEP_SECONDS)
232
240
  # Renew context?
233
241
  now = time.time()
234
242
  renewed = False
@@ -323,7 +323,7 @@ class BaseClient:
323
323
  pass
324
324
 
325
325
  # We randomize the list, and add at the end the failover URLs (System/FailoverURLs/Component)
326
- urlsList = List.randomize(List.fromChar(urls, ",")) + failoverUrls
326
+ urlsList = List.fromChar(urls, ",") + failoverUrls
327
327
  self.__nbOfUrls = len(urlsList)
328
328
  self.__nbOfRetry = (
329
329
  2 if self.__nbOfUrls > 2 else 3
@@ -445,7 +445,6 @@ and this is thread {cThID}
445
445
  return self.__initStatus
446
446
  if self.__enableThreadCheck:
447
447
  self.__checkThreadID()
448
-
449
448
  gLogger.debug(f"Trying to connect to: {self.serviceURL}")
450
449
  try:
451
450
  # Calls the transport method of the apropriate protocol.
@@ -110,19 +110,18 @@ class SSLTransport(BaseTransport):
110
110
  if self.serverMode():
111
111
  raise RuntimeError("SSLTransport is in server mode.")
112
112
 
113
- error = None
113
+ errors = []
114
114
  host, port = self.stServerAddress
115
115
 
116
116
  # The following piece of code was inspired by the python socket documentation
117
117
  # as well as the implementation of M2Crypto.httpslib.HTTPSConnection
118
118
 
119
- # We ignore the returned sockaddr because SSL.Connection.connect needs
120
- # a host name.
119
+ # Get all available addresses (IPv6 and IPv4) and try them in order
121
120
  try:
122
121
  addrInfoList = socket.getaddrinfo(host, port, socket.AF_UNSPEC, socket.SOCK_STREAM)
123
122
  except OSError as e:
124
123
  return S_ERROR(f"DNS lookup failed {e!r}")
125
- for family, _socketType, _proto, _canonname, _socketAddress in addrInfoList:
124
+ for family, _socketType, _proto, _canonname, socketAddress in addrInfoList:
126
125
  try:
127
126
  self.oSocket = SSL.Connection(self.__ctx, family=family)
128
127
 
@@ -138,7 +137,10 @@ class SSLTransport(BaseTransport):
138
137
  # set SNI server name since we know it at this point
139
138
  self.oSocket.set_tlsext_host_name(host)
140
139
 
141
- self.oSocket.connect((host, port))
140
+ # tell the connection which host we are connecting to so we can
141
+ # use the address we obtained from DNS
142
+ self.oSocket.set1_host(host)
143
+ self.oSocket.connect(socketAddress)
142
144
 
143
145
  # Once the connection is established, we can use the timeout
144
146
  # asked for RPC
@@ -151,12 +153,12 @@ class SSLTransport(BaseTransport):
151
153
  # They should be propagated upwards and caught by the BaseClient
152
154
  # not to enter the retry loop
153
155
  except OSError as e:
154
- error = f"{e}:{repr(e)}"
156
+ errors.append(f"{socketAddress} {e}:{repr(e)}")
155
157
 
156
158
  if self.oSocket is not None:
157
159
  self.close()
158
160
 
159
- return S_ERROR(error)
161
+ return S_ERROR("; ".join(errors))
160
162
 
161
163
  def initAsServer(self):
162
164
  """Prepare this server socket for use."""
@@ -40,14 +40,14 @@ from DIRAC.Core.Utilities.ReturnValues import convertToReturnValue, returnValueO
40
40
 
41
41
  PEM_BEGIN = "-----BEGIN DIRACX-----"
42
42
  PEM_END = "-----END DIRACX-----"
43
- RE_DIRACX_PEM = re.compile(rf"{PEM_BEGIN}\n(.*)\n{PEM_END}", re.MULTILINE | re.DOTALL)
43
+ RE_DIRACX_PEM = re.compile(rf"{PEM_BEGIN}\n(.*?)\n{PEM_END}", re.DOTALL)
44
44
 
45
45
 
46
46
  @convertToReturnValue
47
47
  def addTokenToPEM(pemPath, group):
48
48
  from DIRAC.Core.Base.Client import Client
49
49
 
50
- vo = Registry.getVOMSVOForGroup(group)
50
+ vo = Registry.getVOForGroup(group)
51
51
  if not vo:
52
52
  gLogger.error(f"ERROR: Could not find VO for group {group}, DiracX will not work!")
53
53
  disabledVOs = gConfig.getValue("/DiracX/DisabledVOs", [])
@@ -62,21 +62,26 @@ def addTokenToPEM(pemPath, group):
62
62
  token_type=token_content.get("token_type"),
63
63
  refresh_token=token_content.get("refresh_token"),
64
64
  )
65
-
66
65
  token_pem = f"{PEM_BEGIN}\n"
67
66
  data = base64.b64encode(serialize_credentials(token).encode("utf-8")).decode()
68
67
  token_pem += textwrap.fill(data, width=64)
69
68
  token_pem += f"\n{PEM_END}\n"
70
69
 
71
- with open(pemPath, "a") as f:
72
- f.write(token_pem)
70
+ pem = Path(pemPath).read_text()
71
+ # Remove any existing DiracX token there would be
72
+ new_pem = re.sub(RE_DIRACX_PEM, "", pem)
73
+ new_pem += token_pem
74
+
75
+ Path(pemPath).write_text(new_pem)
73
76
 
74
77
 
75
78
  def diracxTokenFromPEM(pemPath) -> dict[str, Any] | None:
76
79
  """Extract the DiracX token from the proxy PEM file"""
77
80
  pem = Path(pemPath).read_text()
78
- if match := RE_DIRACX_PEM.search(pem):
79
- match = match.group(1)
81
+ if match := RE_DIRACX_PEM.findall(pem):
82
+ if len(match) > 1:
83
+ raise ValueError("Found multiple DiracX tokens, this should never happen")
84
+ match = match[0]
80
85
  return json.loads(base64.b64decode(match).decode("utf-8"))
81
86
 
82
87
 
@@ -144,7 +144,7 @@ class IAMService:
144
144
  result = S_OK({"Users": users, "Errors": errors})
145
145
  return result
146
146
 
147
- def getUsersSub(self) -> dict[str, str]:
147
+ def getUsersSub(self, vo=None) -> dict[str, str]:
148
148
  """
149
149
  Return the mapping based on IAM sub:
150
150
  {nickname : sub}
@@ -152,6 +152,7 @@ class IAMService:
152
152
  iam_users_raw = self._getIamUserDump()
153
153
  diracx_user_section = {}
154
154
  for user_info in iam_users_raw:
155
+ userGroups = [grp["display"] for grp in user_info.get("groups", [])]
155
156
  # The nickname is available in the list of attributes
156
157
  # (if configured so)
157
158
  # in the form {'name': 'nickname', 'value': 'chaen'}
@@ -165,8 +166,8 @@ class IAMService:
165
166
  except (KeyError, IndexError):
166
167
  nickname = user_info["userName"]
167
168
  sub = user_info["id"]
168
-
169
- diracx_user_section[nickname] = sub
169
+ if not vo or vo in userGroups:
170
+ diracx_user_section[nickname] = sub
170
171
  # reorder it
171
172
  diracx_user_section = dict(sorted(diracx_user_section.items()))
172
173
 
@@ -1,10 +1,13 @@
1
1
  """
2
- Set of utilities to retrieve Information from proxy
2
+ Set of utilities to retrieve Information from proxy
3
3
  """
4
+
4
5
  import base64
5
6
 
6
7
  from DIRAC import S_ERROR, S_OK, gLogger
7
8
  from DIRAC.ConfigurationSystem.Client.Helpers import Registry
9
+ from DIRAC.ConfigurationSystem.Client.Helpers.CSGlobals import getVO
10
+
8
11
  from DIRAC.Core.Security import Locations
9
12
  from DIRAC.Core.Security.DiracX import diracxTokenFromPEM
10
13
  from DIRAC.Core.Security.VOMS import VOMS
@@ -207,10 +210,11 @@ def getVOfromProxyGroup():
207
210
  """
208
211
  Return the VO associated to the group in the proxy
209
212
  """
210
- voName = Registry.getVOForGroup("NoneExistingGroup")
213
+
211
214
  ret = getProxyInfo(disableVOMS=True)
212
- if not ret["OK"]:
213
- return S_OK(voName)
214
- if "group" in ret["Value"]:
215
+ if not ret["OK"] or "group" not in ret["Value"]:
216
+ voName = getVO()
217
+ else:
215
218
  voName = Registry.getVOForGroup(ret["Value"]["group"])
219
+
216
220
  return S_OK(voName)
@@ -0,0 +1,161 @@
1
+ import base64
2
+ import json
3
+ import pytest
4
+ import tempfile
5
+ from pathlib import Path
6
+ from unittest.mock import patch, mock_open
7
+
8
+ from DIRAC.Core.Security.DiracX import diracxTokenFromPEM, PEM_BEGIN, PEM_END, RE_DIRACX_PEM
9
+
10
+
11
+ class TestDiracxTokenFromPEM:
12
+ """Test cases for diracxTokenFromPEM function"""
13
+
14
+ def create_valid_token_data(self):
15
+ """Create valid token data for testing"""
16
+ return {
17
+ "access_token": "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.test",
18
+ "refresh_token": "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.refresh",
19
+ "expires_in": 3600,
20
+ "token_type": "Bearer",
21
+ }
22
+
23
+ def create_pem_content(self, token_data=None, include_other_content=True):
24
+ """Create PEM content with embedded DiracX token"""
25
+ if token_data is None:
26
+ token_data = self.create_valid_token_data()
27
+
28
+ # Encode token data
29
+ token_json = json.dumps(token_data)
30
+ encoded_token = base64.b64encode(token_json.encode("utf-8")).decode()
31
+
32
+ # Create PEM content
33
+ pem_content = ""
34
+ if include_other_content:
35
+ pem_content += "-----BEGIN CERTIFICATE-----\n"
36
+ pem_content += "MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA...\n"
37
+ pem_content += "-----END CERTIFICATE-----\n"
38
+
39
+ pem_content += f"{PEM_BEGIN}\n"
40
+ pem_content += encoded_token + "\n"
41
+ pem_content += f"{PEM_END}\n"
42
+
43
+ return pem_content
44
+
45
+ def test_valid_token_extraction(self):
46
+ """Test successful extraction of valid token from PEM file"""
47
+ token_data = self.create_valid_token_data()
48
+ pem_content = self.create_pem_content(token_data)
49
+
50
+ with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".pem") as f:
51
+ f.write(pem_content)
52
+ temp_path = f.name
53
+
54
+ try:
55
+ result = diracxTokenFromPEM(temp_path)
56
+ assert result == token_data
57
+ finally:
58
+ Path(temp_path).unlink()
59
+
60
+ def test_no_token_in_pem(self):
61
+ """Test behavior when no DiracX token is present in PEM file"""
62
+ pem_content = """-----BEGIN CERTIFICATE-----
63
+ MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA...
64
+ -----END CERTIFICATE-----"""
65
+
66
+ with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".pem") as f:
67
+ f.write(pem_content)
68
+ temp_path = f.name
69
+
70
+ try:
71
+ result = diracxTokenFromPEM(temp_path)
72
+ assert result is None
73
+ finally:
74
+ Path(temp_path).unlink()
75
+
76
+ def test_multiple_tokens_error(self):
77
+ """Test that multiple tokens raise ValueError"""
78
+ token_data = self.create_valid_token_data()
79
+
80
+ # Create PEM with two tokens
81
+ pem_content = self.create_pem_content(token_data)
82
+ pem_content += "\n" + self.create_pem_content(token_data, include_other_content=False)
83
+
84
+ with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".pem") as f:
85
+ f.write(pem_content)
86
+ temp_path = f.name
87
+
88
+ try:
89
+ with pytest.raises(ValueError, match="Found multiple DiracX tokens"):
90
+ diracxTokenFromPEM(temp_path)
91
+ finally:
92
+ Path(temp_path).unlink()
93
+
94
+ def test_malformed_base64(self):
95
+ """Test behavior with malformed base64 data"""
96
+ pem_content = f"""{PEM_BEGIN}
97
+ invalid_base64_data_that_will_cause_error!
98
+ {PEM_END}"""
99
+
100
+ with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".pem") as f:
101
+ f.write(pem_content)
102
+ temp_path = f.name
103
+
104
+ try:
105
+ with pytest.raises(Exception): # base64.b64decode will raise an exception
106
+ diracxTokenFromPEM(temp_path)
107
+ finally:
108
+ Path(temp_path).unlink()
109
+
110
+ def test_invalid_json_in_token(self):
111
+ """Test behavior with invalid JSON in token data"""
112
+ invalid_json = "this is not valid json"
113
+ encoded_invalid = base64.b64encode(invalid_json.encode("utf-8")).decode()
114
+
115
+ pem_content = f"""{PEM_BEGIN}
116
+ {encoded_invalid}
117
+ {PEM_END}"""
118
+
119
+ with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".pem") as f:
120
+ f.write(pem_content)
121
+ temp_path = f.name
122
+
123
+ try:
124
+ with pytest.raises(json.JSONDecodeError):
125
+ diracxTokenFromPEM(temp_path)
126
+ finally:
127
+ Path(temp_path).unlink()
128
+
129
+ def test_token_with_unicode_characters(self):
130
+ """Test token with unicode characters"""
131
+ unicode_token = {
132
+ "access_token": "token_with_unicode_ñ_é_ü",
133
+ "refresh_token": "refresh_with_emoji_🚀_🎉",
134
+ "expires_in": 3600,
135
+ "token_type": "Bearer",
136
+ }
137
+
138
+ pem_content = self.create_pem_content(unicode_token)
139
+
140
+ with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".pem") as f:
141
+ f.write(pem_content)
142
+ temp_path = f.name
143
+
144
+ try:
145
+ result = diracxTokenFromPEM(temp_path)
146
+ assert result == unicode_token
147
+ finally:
148
+ Path(temp_path).unlink()
149
+
150
+ def test_regex_pattern_validation(self):
151
+ """Test that the regex pattern correctly identifies DiracX tokens"""
152
+ # Test that the regex matches the expected pattern
153
+ token_data = self.create_valid_token_data()
154
+ token_json = json.dumps(token_data)
155
+ encoded_token = base64.b64encode(token_json.encode("utf-8")).decode()
156
+
157
+ test_content = f"{PEM_BEGIN}\n{encoded_token}\n{PEM_END}"
158
+ matches = RE_DIRACX_PEM.findall(test_content)
159
+
160
+ assert len(matches) == 1
161
+ assert matches[0] == encoded_token
@@ -17,7 +17,6 @@ from DIRAC.Core.DISET.RPCClient import RPCClient
17
17
  from DIRAC.Core.DISET.TransferClient import TransferClient
18
18
  from DIRAC.Core.Tornado.Client.TornadoClient import TornadoClient
19
19
 
20
-
21
20
  sLog = gLogger.getSubLogger(__name__)
22
21
 
23
22
 
@@ -82,6 +81,10 @@ def ClientSelector(disetClient, *args, **kwargs): # We use same interface as RP
82
81
  rpc = tornadoClient(*args, **kwargs)
83
82
  else:
84
83
  rpc = disetClient(*args, **kwargs)
84
+ except NotImplementedError as e:
85
+ # We catch explicitly NotImplementedError to avoid just printing "there's an error"
86
+ # If we mis-configured the CS for legacy adapted services, we MUST have an error.
87
+ raise e
85
88
  except Exception as e: # pylint: disable=broad-except
86
89
  # If anything went wrong in the resolution, we return default RPCClient
87
90
  # So the behaviour is exactly the same as before implementation of Tornado
@@ -73,7 +73,7 @@ class TornadoService(BaseRequestHandler): # pylint: disable=abstract-method
73
73
  :py:class:`BaseRequestHandler <DIRAC.Core.Tornado.Server.private.BaseRequestHandler.BaseRequestHandler>` for more details.
74
74
 
75
75
  In order to pass information around and keep some states, we use instance attributes.
76
- These are initialized in the :py:meth:`.initialize` method.
76
+ These are initialized in the ``initialize`` methods.
77
77
 
78
78
  The handler only define the ``post`` verb. Please refer to :py:meth:`.post` for the details.
79
79