DIRAC 9.0.0a42__py3-none-any.whl → 9.0.7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- DIRAC/AccountingSystem/Client/AccountingCLI.py +0 -140
- DIRAC/AccountingSystem/Client/DataStoreClient.py +0 -13
- DIRAC/AccountingSystem/Client/Types/BaseAccountingType.py +0 -7
- DIRAC/AccountingSystem/ConfigTemplate.cfg +0 -5
- DIRAC/AccountingSystem/Service/DataStoreHandler.py +0 -72
- DIRAC/ConfigurationSystem/Client/Helpers/CSGlobals.py +0 -9
- DIRAC/ConfigurationSystem/Client/Helpers/Registry.py +38 -26
- DIRAC/ConfigurationSystem/Client/Helpers/Resources.py +11 -43
- DIRAC/ConfigurationSystem/Client/Helpers/test/Test_Helpers.py +0 -16
- DIRAC/ConfigurationSystem/Client/LocalConfiguration.py +14 -8
- DIRAC/ConfigurationSystem/Client/PathFinder.py +47 -8
- DIRAC/ConfigurationSystem/Client/SyncPlugins/CERNLDAPSyncPlugin.py +4 -1
- DIRAC/ConfigurationSystem/Client/VOMS2CSSynchronizer.py +32 -19
- DIRAC/ConfigurationSystem/Client/test/Test_PathFinder.py +41 -1
- DIRAC/ConfigurationSystem/private/RefresherBase.py +4 -2
- DIRAC/Core/Base/API.py +4 -7
- DIRAC/Core/Base/SQLAlchemyDB.py +1 -0
- DIRAC/Core/DISET/ServiceReactor.py +11 -3
- DIRAC/Core/DISET/private/BaseClient.py +1 -2
- DIRAC/Core/DISET/private/Transports/M2SSLTransport.py +9 -7
- DIRAC/Core/DISET/private/Transports/SSL/M2Utils.py +3 -1
- DIRAC/Core/LCG/GOCDBClient.py +5 -7
- DIRAC/Core/Security/DiracX.py +31 -17
- DIRAC/Core/Security/IAMService.py +5 -10
- DIRAC/Core/Security/Locations.py +27 -18
- DIRAC/Core/Security/ProxyInfo.py +9 -5
- DIRAC/Core/Security/VOMSService.py +2 -4
- DIRAC/Core/Security/m2crypto/X509Certificate.py +4 -6
- DIRAC/Core/Security/m2crypto/asn1_utils.py +17 -5
- DIRAC/Core/Security/test/test_diracx_token_from_pem.py +161 -0
- DIRAC/Core/Tornado/Client/ClientSelector.py +4 -1
- DIRAC/Core/Tornado/Server/TornadoService.py +1 -1
- DIRAC/Core/Utilities/CGroups2.py +328 -0
- DIRAC/Core/Utilities/ClassAd/ClassAdLight.py +4 -290
- DIRAC/Core/Utilities/DErrno.py +5 -309
- DIRAC/Core/Utilities/Extensions.py +10 -1
- DIRAC/Core/Utilities/File.py +1 -1
- DIRAC/Core/Utilities/Graphs/GraphData.py +1 -1
- DIRAC/Core/Utilities/Graphs/GraphUtilities.py +6 -1
- DIRAC/Core/Utilities/JDL.py +1 -195
- DIRAC/Core/Utilities/List.py +1 -124
- DIRAC/Core/Utilities/MySQL.py +103 -99
- DIRAC/Core/Utilities/Os.py +32 -1
- DIRAC/Core/Utilities/Platform.py +2 -107
- DIRAC/Core/Utilities/Proxy.py +0 -4
- DIRAC/Core/Utilities/ReturnValues.py +7 -252
- DIRAC/Core/Utilities/StateMachine.py +12 -178
- DIRAC/Core/Utilities/Subprocess.py +35 -14
- DIRAC/Core/Utilities/TimeUtilities.py +10 -253
- DIRAC/Core/Utilities/test/Test_JDL.py +0 -3
- DIRAC/Core/Utilities/test/Test_Profiler.py +20 -20
- DIRAC/Core/scripts/dirac_agent.py +1 -1
- DIRAC/Core/scripts/dirac_apptainer_exec.py +72 -46
- DIRAC/Core/scripts/dirac_configure.py +1 -3
- DIRAC/Core/scripts/dirac_install_db.py +24 -6
- DIRAC/Core/scripts/dirac_platform.py +1 -92
- DIRAC/DataManagementSystem/Agent/FTS3Agent.py +8 -7
- DIRAC/DataManagementSystem/Agent/RequestOperations/RemoveFile.py +7 -6
- DIRAC/DataManagementSystem/Client/FTS3Job.py +71 -34
- DIRAC/DataManagementSystem/DB/FTS3DB.py +7 -3
- DIRAC/DataManagementSystem/DB/FileCatalogComponents/DatasetManager/DatasetManager.py +1 -1
- DIRAC/DataManagementSystem/DB/FileCatalogDB.sql +9 -9
- DIRAC/DataManagementSystem/DB/FileCatalogWithFkAndPsDB.sql +9 -9
- DIRAC/DataManagementSystem/Utilities/DMSHelpers.py +6 -2
- DIRAC/DataManagementSystem/scripts/dirac_admin_allow_se.py +13 -8
- DIRAC/DataManagementSystem/scripts/dirac_admin_ban_se.py +13 -8
- DIRAC/DataManagementSystem/scripts/dirac_dms_create_moving_request.py +2 -0
- DIRAC/DataManagementSystem/scripts/dirac_dms_protocol_matrix.py +0 -1
- DIRAC/FrameworkSystem/Client/BundleDeliveryClient.py +2 -7
- DIRAC/FrameworkSystem/Client/ComponentInstaller.py +9 -4
- DIRAC/FrameworkSystem/Client/ProxyManagerClient.py +5 -2
- DIRAC/FrameworkSystem/Client/SystemAdministratorClientCLI.py +11 -6
- DIRAC/FrameworkSystem/ConfigTemplate.cfg +2 -0
- DIRAC/FrameworkSystem/DB/AuthDB.py +3 -3
- DIRAC/FrameworkSystem/DB/InstalledComponentsDB.py +4 -4
- DIRAC/FrameworkSystem/DB/ProxyDB.py +11 -3
- DIRAC/FrameworkSystem/DB/TokenDB.py +1 -1
- DIRAC/FrameworkSystem/Service/ProxyManagerHandler.py +8 -6
- DIRAC/FrameworkSystem/Utilities/MonitoringUtilities.py +2 -19
- DIRAC/FrameworkSystem/Utilities/TokenManagementUtilities.py +3 -2
- DIRAC/FrameworkSystem/Utilities/diracx.py +36 -14
- DIRAC/FrameworkSystem/private/authorization/AuthServer.py +2 -2
- DIRAC/FrameworkSystem/scripts/dirac_admin_update_pilot.py +18 -11
- DIRAC/FrameworkSystem/scripts/dirac_login.py +2 -2
- DIRAC/FrameworkSystem/scripts/dirac_proxy_init.py +7 -8
- DIRAC/Interfaces/API/Dirac.py +27 -15
- DIRAC/Interfaces/API/DiracAdmin.py +45 -17
- DIRAC/Interfaces/API/Job.py +9 -13
- DIRAC/Interfaces/scripts/dirac_admin_allow_site.py +12 -18
- DIRAC/Interfaces/scripts/dirac_admin_ban_site.py +12 -10
- DIRAC/Interfaces/scripts/dirac_admin_get_site_mask.py +4 -13
- DIRAC/Interfaces/scripts/dirac_admin_reset_job.py +3 -6
- DIRAC/Interfaces/scripts/dirac_wms_job_parameters.py +0 -1
- DIRAC/MonitoringSystem/Client/Types/WMSHistory.py +4 -0
- DIRAC/MonitoringSystem/Client/WebAppClient.py +26 -0
- DIRAC/MonitoringSystem/ConfigTemplate.cfg +9 -0
- DIRAC/MonitoringSystem/DB/MonitoringDB.py +6 -25
- DIRAC/MonitoringSystem/Service/MonitoringHandler.py +0 -33
- DIRAC/MonitoringSystem/Service/WebAppHandler.py +599 -0
- DIRAC/MonitoringSystem/private/MainReporter.py +0 -3
- DIRAC/ProductionSystem/DB/ProductionDB.sql +4 -4
- DIRAC/ProductionSystem/scripts/dirac_prod_get.py +2 -2
- DIRAC/ProductionSystem/scripts/dirac_prod_get_all.py +2 -2
- DIRAC/ProductionSystem/scripts/dirac_prod_get_trans.py +2 -3
- DIRAC/RequestManagementSystem/Agent/RequestExecutingAgent.py +8 -6
- DIRAC/RequestManagementSystem/Agent/RequestOperations/ForwardDISET.py +2 -14
- DIRAC/RequestManagementSystem/Client/ReqClient.py +66 -13
- DIRAC/RequestManagementSystem/ConfigTemplate.cfg +6 -6
- DIRAC/RequestManagementSystem/DB/RequestDB.py +10 -5
- DIRAC/RequestManagementSystem/DB/test/RMSTestScenari.py +2 -0
- DIRAC/RequestManagementSystem/private/RequestValidator.py +40 -46
- DIRAC/ResourceStatusSystem/Client/SiteStatus.py +4 -2
- DIRAC/ResourceStatusSystem/Command/FreeDiskSpaceCommand.py +3 -1
- DIRAC/ResourceStatusSystem/DB/ResourceManagementDB.py +8 -8
- DIRAC/ResourceStatusSystem/DB/ResourceStatusDB.py +2 -2
- DIRAC/ResourceStatusSystem/Utilities/CSHelpers.py +2 -31
- DIRAC/ResourceStatusSystem/scripts/dirac_rss_set_status.py +30 -12
- DIRAC/Resources/Catalog/RucioFileCatalogClient.py +195 -1
- DIRAC/Resources/Catalog/test/Test_RucioFileCatalogClient.py +181 -0
- DIRAC/Resources/Computing/AREXComputingElement.py +25 -8
- DIRAC/Resources/Computing/BatchSystems/Condor.py +126 -108
- DIRAC/Resources/Computing/BatchSystems/SLURM.py +5 -1
- DIRAC/Resources/Computing/BatchSystems/test/Test_SLURM.py +46 -0
- DIRAC/Resources/Computing/ComputingElement.py +1 -1
- DIRAC/Resources/Computing/HTCondorCEComputingElement.py +44 -44
- DIRAC/Resources/Computing/InProcessComputingElement.py +4 -2
- DIRAC/Resources/Computing/LocalComputingElement.py +1 -18
- DIRAC/Resources/Computing/SSHBatchComputingElement.py +1 -17
- DIRAC/Resources/Computing/SSHComputingElement.py +1 -18
- DIRAC/Resources/Computing/SingularityComputingElement.py +19 -5
- DIRAC/Resources/Computing/test/Test_HTCondorCEComputingElement.py +67 -49
- DIRAC/Resources/Computing/test/Test_PoolComputingElement.py +2 -1
- DIRAC/Resources/IdProvider/CheckInIdProvider.py +13 -0
- DIRAC/Resources/IdProvider/IdProviderFactory.py +11 -3
- DIRAC/Resources/MessageQueue/StompMQConnector.py +1 -1
- DIRAC/Resources/Storage/GFAL2_StorageBase.py +24 -15
- DIRAC/Resources/Storage/OccupancyPlugins/WLCGAccountingHTTPJson.py +1 -3
- DIRAC/Resources/Storage/StorageBase.py +4 -2
- DIRAC/Resources/Storage/StorageElement.py +6 -7
- DIRAC/StorageManagementSystem/DB/StorageManagementDB.sql +2 -2
- DIRAC/TransformationSystem/Agent/TaskManagerAgentBase.py +10 -16
- DIRAC/TransformationSystem/Agent/TransformationAgent.py +22 -1
- DIRAC/TransformationSystem/Agent/TransformationCleaningAgent.py +16 -16
- DIRAC/TransformationSystem/Client/TaskManager.py +2 -4
- DIRAC/TransformationSystem/Client/Transformation.py +6 -7
- DIRAC/TransformationSystem/Client/TransformationClient.py +21 -11
- DIRAC/TransformationSystem/Client/Utilities.py +9 -0
- DIRAC/TransformationSystem/DB/TransformationDB.py +11 -14
- DIRAC/TransformationSystem/DB/TransformationDB.sql +9 -9
- DIRAC/TransformationSystem/Service/TransformationManagerHandler.py +0 -333
- DIRAC/TransformationSystem/Utilities/ReplicationCLIParameters.py +3 -3
- DIRAC/TransformationSystem/Utilities/TransformationInfo.py +7 -5
- DIRAC/TransformationSystem/scripts/dirac_production_runjoblocal.py +2 -4
- DIRAC/TransformationSystem/test/Test_TransformationInfo.py +22 -15
- DIRAC/TransformationSystem/test/Test_replicationTransformation.py +5 -6
- DIRAC/Workflow/Modules/test/Test_Modules.py +5 -0
- DIRAC/WorkloadManagementSystem/Agent/JobAgent.py +38 -26
- DIRAC/WorkloadManagementSystem/Agent/JobCleaningAgent.py +12 -8
- DIRAC/WorkloadManagementSystem/Agent/PilotSyncAgent.py +4 -3
- DIRAC/WorkloadManagementSystem/Agent/PushJobAgent.py +13 -13
- DIRAC/WorkloadManagementSystem/Agent/SiteDirector.py +18 -14
- DIRAC/WorkloadManagementSystem/Agent/StalledJobAgent.py +18 -51
- DIRAC/WorkloadManagementSystem/Agent/StatesAccountingAgent.py +41 -1
- DIRAC/WorkloadManagementSystem/Agent/test/Test_Agent_JobAgent.py +45 -4
- DIRAC/WorkloadManagementSystem/Agent/test/Test_Agent_JobCleaningAgent.py +7 -9
- DIRAC/WorkloadManagementSystem/Agent/test/Test_Agent_PushJobAgent.py +1 -0
- DIRAC/WorkloadManagementSystem/Agent/test/Test_Agent_SiteDirector.py +9 -2
- DIRAC/WorkloadManagementSystem/Agent/test/Test_Agent_StalledJobAgent.py +4 -5
- DIRAC/WorkloadManagementSystem/Client/DownloadInputData.py +9 -9
- DIRAC/WorkloadManagementSystem/Client/InputDataResolution.py +6 -6
- DIRAC/WorkloadManagementSystem/Client/JobMonitoringClient.py +10 -11
- DIRAC/WorkloadManagementSystem/Client/JobReport.py +1 -1
- DIRAC/WorkloadManagementSystem/Client/JobState/CachedJobState.py +3 -0
- DIRAC/WorkloadManagementSystem/Client/JobState/JobManifest.py +32 -261
- DIRAC/WorkloadManagementSystem/Client/JobState/JobState.py +6 -0
- DIRAC/WorkloadManagementSystem/Client/JobStateUpdateClient.py +3 -0
- DIRAC/WorkloadManagementSystem/Client/JobStatus.py +8 -152
- DIRAC/WorkloadManagementSystem/Client/PoolXMLSlice.py +12 -19
- DIRAC/WorkloadManagementSystem/Client/SandboxStoreClient.py +25 -38
- DIRAC/WorkloadManagementSystem/Client/WMSClient.py +2 -3
- DIRAC/WorkloadManagementSystem/Client/test/Test_Client_DownloadInputData.py +29 -0
- DIRAC/WorkloadManagementSystem/ConfigTemplate.cfg +4 -8
- DIRAC/WorkloadManagementSystem/DB/JobDB.py +89 -132
- DIRAC/WorkloadManagementSystem/DB/JobDB.sql +8 -8
- DIRAC/WorkloadManagementSystem/DB/JobDBUtils.py +18 -147
- DIRAC/WorkloadManagementSystem/DB/JobLoggingDB.py +19 -6
- DIRAC/WorkloadManagementSystem/DB/JobParametersDB.py +9 -9
- DIRAC/WorkloadManagementSystem/DB/PilotAgentsDB.py +16 -5
- DIRAC/WorkloadManagementSystem/DB/PilotAgentsDB.sql +3 -3
- DIRAC/WorkloadManagementSystem/DB/SandboxMetadataDB.py +44 -82
- DIRAC/WorkloadManagementSystem/DB/StatusUtils.py +125 -0
- DIRAC/WorkloadManagementSystem/DB/tests/Test_JobDB.py +1 -1
- DIRAC/WorkloadManagementSystem/DB/tests/Test_StatusUtils.py +28 -0
- DIRAC/WorkloadManagementSystem/Executor/JobSanity.py +5 -4
- DIRAC/WorkloadManagementSystem/Executor/JobScheduling.py +4 -0
- DIRAC/WorkloadManagementSystem/FutureClient/JobStateUpdateClient.py +75 -33
- DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py +22 -11
- DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapperTemplate.py +9 -10
- DIRAC/WorkloadManagementSystem/JobWrapper/test/Test_JobWrapper.py +60 -10
- DIRAC/WorkloadManagementSystem/JobWrapper/test/Test_JobWrapperTemplate.py +4 -0
- DIRAC/WorkloadManagementSystem/Service/JobManagerHandler.py +33 -154
- DIRAC/WorkloadManagementSystem/Service/JobMonitoringHandler.py +5 -323
- DIRAC/WorkloadManagementSystem/Service/JobStateUpdateHandler.py +0 -16
- DIRAC/WorkloadManagementSystem/Service/PilotManagerHandler.py +6 -103
- DIRAC/WorkloadManagementSystem/Service/SandboxStoreHandler.py +7 -53
- DIRAC/WorkloadManagementSystem/Service/WMSAdministratorHandler.py +16 -79
- DIRAC/WorkloadManagementSystem/Service/WMSUtilities.py +4 -18
- DIRAC/WorkloadManagementSystem/Utilities/JobModel.py +28 -209
- DIRAC/WorkloadManagementSystem/Utilities/JobParameters.py +65 -3
- DIRAC/WorkloadManagementSystem/Utilities/JobStatusUtility.py +2 -64
- DIRAC/WorkloadManagementSystem/Utilities/ParametricJob.py +7 -171
- DIRAC/WorkloadManagementSystem/Utilities/PilotCStoJSONSynchronizer.py +73 -7
- DIRAC/WorkloadManagementSystem/Utilities/PilotWrapper.py +41 -11
- DIRAC/WorkloadManagementSystem/Utilities/RemoteRunner.py +16 -0
- DIRAC/WorkloadManagementSystem/Utilities/Utils.py +36 -1
- DIRAC/WorkloadManagementSystem/Utilities/jobAdministration.py +15 -0
- DIRAC/WorkloadManagementSystem/Utilities/test/Test_JobModel.py +1 -15
- DIRAC/WorkloadManagementSystem/Utilities/test/Test_ParametricJob.py +45 -128
- DIRAC/WorkloadManagementSystem/Utilities/test/Test_PilotWrapper.py +16 -0
- DIRAC/WorkloadManagementSystem/scripts/dirac_jobexec.py +7 -2
- DIRAC/WorkloadManagementSystem/scripts/dirac_wms_pilot_job_info.py +1 -1
- DIRAC/__init__.py +62 -60
- DIRAC/tests/Utilities/testJobDefinitions.py +22 -28
- {DIRAC-9.0.0a42.dist-info → dirac-9.0.7.dist-info}/METADATA +8 -5
- {DIRAC-9.0.0a42.dist-info → dirac-9.0.7.dist-info}/RECORD +229 -228
- {DIRAC-9.0.0a42.dist-info → dirac-9.0.7.dist-info}/WHEEL +1 -1
- {DIRAC-9.0.0a42.dist-info → dirac-9.0.7.dist-info}/entry_points.txt +0 -3
- DIRAC/Core/Utilities/test/Test_List.py +0 -150
- DIRAC/Core/Utilities/test/Test_Time.py +0 -88
- DIRAC/Resources/Computing/PilotBundle.py +0 -70
- DIRAC/TransformationSystem/scripts/dirac_transformation_archive.py +0 -30
- DIRAC/TransformationSystem/scripts/dirac_transformation_clean.py +0 -30
- DIRAC/TransformationSystem/scripts/dirac_transformation_remove_output.py +0 -30
- DIRAC/WorkloadManagementSystem/Utilities/test/Test_JobManager.py +0 -58
- {DIRAC-9.0.0a42.dist-info → dirac-9.0.7.dist-info/licenses}/LICENSE +0 -0
- {DIRAC-9.0.0a42.dist-info → dirac-9.0.7.dist-info}/top_level.txt +0 -0
|
@@ -1,7 +1,12 @@
|
|
|
1
1
|
""" Collection of utilities for finding paths in the CS
|
|
2
2
|
"""
|
|
3
|
+
import re
|
|
4
|
+
from copy import deepcopy
|
|
5
|
+
from collections.abc import Iterable
|
|
3
6
|
from urllib import parse
|
|
4
7
|
|
|
8
|
+
from cachetools import cached, TTLCache
|
|
9
|
+
|
|
5
10
|
from DIRAC.Core.Utilities import List
|
|
6
11
|
from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData
|
|
7
12
|
from DIRAC.ConfigurationSystem.Client.Helpers import Path
|
|
@@ -151,6 +156,43 @@ def getSystemURLs(system, failover=False):
|
|
|
151
156
|
return urlDict
|
|
152
157
|
|
|
153
158
|
|
|
159
|
+
def groupURLsByPriority(urls: Iterable[str]) -> list[set[str]]:
|
|
160
|
+
"""Group URLs by priority.
|
|
161
|
+
|
|
162
|
+
:param Iterable[str] preferredURLPatterns: patterns to check in ranked order
|
|
163
|
+
:param set[str] urls: URLs to check
|
|
164
|
+
|
|
165
|
+
:return: list[set[str]] -- list of URL groups, ordered by priority
|
|
166
|
+
"""
|
|
167
|
+
return deepcopy(_groupURLsByPriority(frozenset(urls)))
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
@cached(cache=TTLCache(maxsize=1024, ttl=300))
|
|
171
|
+
def _groupURLsByPriority(urls: frozenset[str]) -> list[set[str]]:
|
|
172
|
+
preferredURLPatterns = []
|
|
173
|
+
if patterns := gConfigurationData.extractOptionFromCFG("/DIRAC/PreferredURLPatterns"):
|
|
174
|
+
preferredURLPatterns = [re.compile(pattern) for pattern in List.fromChar(patterns)]
|
|
175
|
+
|
|
176
|
+
urlGroups = [set() for _ in range(len(preferredURLPatterns) + 1)]
|
|
177
|
+
for url in urls:
|
|
178
|
+
urlGroups[findURLPriority(preferredURLPatterns, url)].add(url)
|
|
179
|
+
return urlGroups
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def findURLPriority(preferredURLPatterns: list[re.Pattern[str]], url: str) -> int:
|
|
183
|
+
"""Find which preferred URL pattern the URL matches.
|
|
184
|
+
|
|
185
|
+
:param str preferredURLPatterns: patterns to check in ranked order
|
|
186
|
+
:param str url: URL to check
|
|
187
|
+
|
|
188
|
+
:return: int -- index of the pattern that matched, smallest is the most preferred
|
|
189
|
+
"""
|
|
190
|
+
for i, pattern in enumerate(preferredURLPatterns):
|
|
191
|
+
if re.match(pattern, url):
|
|
192
|
+
return i
|
|
193
|
+
return len(preferredURLPatterns)
|
|
194
|
+
|
|
195
|
+
|
|
154
196
|
def getServiceURLs(system, service=None, failover=False):
|
|
155
197
|
"""Generate url.
|
|
156
198
|
|
|
@@ -168,8 +210,8 @@ def getServiceURLs(system, service=None, failover=False):
|
|
|
168
210
|
# Add failover URLs at the end of the list
|
|
169
211
|
failover = "Failover" if failover else ""
|
|
170
212
|
for fURLs in ["", "Failover"] if failover else [""]:
|
|
171
|
-
urlList = []
|
|
172
213
|
urls = List.fromChar(gConfigurationData.extractOptionFromCFG(f"{systemSection}/{fURLs}URLs/{service}"))
|
|
214
|
+
urlList = set()
|
|
173
215
|
|
|
174
216
|
# Be sure that urls not None
|
|
175
217
|
for url in urls or []:
|
|
@@ -186,16 +228,13 @@ def getServiceURLs(system, service=None, failover=False):
|
|
|
186
228
|
|
|
187
229
|
for srv in mainServers:
|
|
188
230
|
_url = checkComponentURL(url.replace("$MAINSERVERS$", srv), system, service, pathMandatory=True)
|
|
189
|
-
|
|
190
|
-
urlList.append(_url)
|
|
231
|
+
urlList.add(_url)
|
|
191
232
|
continue
|
|
192
233
|
|
|
193
|
-
|
|
194
|
-
if _url not in urlList:
|
|
195
|
-
urlList.append(_url)
|
|
234
|
+
urlList.add(checkComponentURL(url, system, service, pathMandatory=True))
|
|
196
235
|
|
|
197
|
-
|
|
198
|
-
|
|
236
|
+
for urlGroup in groupURLsByPriority(urlList):
|
|
237
|
+
resList.extend(List.randomize(urlGroup))
|
|
199
238
|
|
|
200
239
|
return resList
|
|
201
240
|
|
|
@@ -32,6 +32,9 @@ class CERNLDAPSyncPlugin:
|
|
|
32
32
|
else:
|
|
33
33
|
userDict["PrimaryCERNAccount"] = self._findOwnerAccountName(username, attributes)
|
|
34
34
|
|
|
35
|
+
if userDict["CERNAccountType"] in ["Primary", "Secondary"]:
|
|
36
|
+
userDict["CERNPersonId"] = attributes.get("employeeId", [None])[0]
|
|
37
|
+
|
|
35
38
|
def _findOwnerAccountName(self, username, attributes):
|
|
36
39
|
"""Find the owner account from a CERN LDAP entry.
|
|
37
40
|
|
|
@@ -64,7 +67,7 @@ class CERNLDAPSyncPlugin:
|
|
|
64
67
|
status, result, response, _ = self._connection.search(
|
|
65
68
|
"OU=Users,OU=Organic Units,DC=cern,DC=ch",
|
|
66
69
|
f"(CN={commonName})",
|
|
67
|
-
attributes=["cernAccountOwner", "cernAccountType"],
|
|
70
|
+
attributes=["cernAccountOwner", "cernAccountType", "employeeId"],
|
|
68
71
|
)
|
|
69
72
|
if not status:
|
|
70
73
|
raise ValueError(f"Bad status from LDAP search: {result}")
|
|
@@ -5,22 +5,23 @@
|
|
|
5
5
|
from collections import defaultdict
|
|
6
6
|
|
|
7
7
|
from diraccfg import CFG
|
|
8
|
-
|
|
9
|
-
from DIRAC
|
|
10
|
-
from DIRAC.Core.Security.IAMService import IAMService
|
|
11
|
-
from DIRAC.Core.Security.VOMSService import VOMSService
|
|
12
|
-
from DIRAC.Core.Utilities.List import fromChar
|
|
13
|
-
from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader
|
|
14
|
-
from DIRAC.Core.Utilities.PrettyPrint import printTable
|
|
8
|
+
|
|
9
|
+
from DIRAC import S_ERROR, S_OK, gConfig, gLogger
|
|
15
10
|
from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI
|
|
16
11
|
from DIRAC.ConfigurationSystem.Client.Helpers.Registry import (
|
|
17
|
-
getVOOption,
|
|
18
|
-
getVOMSRoleGroupMapping,
|
|
19
|
-
getUsersInVO,
|
|
20
12
|
getAllUsers,
|
|
21
|
-
getUserOption,
|
|
22
13
|
getGroupsForUser,
|
|
14
|
+
getUserOption,
|
|
15
|
+
getUsersInVO,
|
|
16
|
+
getVOMSRoleGroupMapping,
|
|
17
|
+
getVOOption,
|
|
23
18
|
)
|
|
19
|
+
from DIRAC.Core.Security.IAMService import IAMService
|
|
20
|
+
from DIRAC.Core.Security.VOMSService import VOMSService
|
|
21
|
+
from DIRAC.Core.Utilities.List import fromChar
|
|
22
|
+
from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader
|
|
23
|
+
from DIRAC.Core.Utilities.PrettyPrint import printTable
|
|
24
|
+
from DIRAC.Core.Utilities.ReturnValues import convertToReturnValue, returnValueOrRaise
|
|
24
25
|
|
|
25
26
|
|
|
26
27
|
def _getUserNameFromMail(mail):
|
|
@@ -86,8 +87,15 @@ def _getUserNameFromDN(dn, vo):
|
|
|
86
87
|
return nname
|
|
87
88
|
else:
|
|
88
89
|
robot = False
|
|
90
|
+
# only pop if the remains are sufficient (i.e. not just digits)
|
|
89
91
|
if names[0].lower().startswith("robot"):
|
|
90
|
-
|
|
92
|
+
nameok = False
|
|
93
|
+
if len(names) > 1:
|
|
94
|
+
for name in names[1:]:
|
|
95
|
+
if not name.isdigit():
|
|
96
|
+
nameok = True
|
|
97
|
+
if nameok:
|
|
98
|
+
names.pop(0)
|
|
91
99
|
robot = True
|
|
92
100
|
for name in list(names):
|
|
93
101
|
if name[0].isdigit() or "@" in name:
|
|
@@ -345,11 +353,6 @@ class VOMS2CSSynchronizer:
|
|
|
345
353
|
|
|
346
354
|
# We have a real new user
|
|
347
355
|
if not diracName:
|
|
348
|
-
# Do not consider users with Suspended status in VOMS
|
|
349
|
-
if self.vomsUserDict[dn]["suspended"] or self.vomsUserDict[dn]["certSuspended"]:
|
|
350
|
-
resultDict["SuspendedUsers"].append(newDiracName)
|
|
351
|
-
continue
|
|
352
|
-
|
|
353
356
|
# if we have a nickname, we use the nickname no
|
|
354
357
|
# matter what so we can have users from different
|
|
355
358
|
# VOs with the same nickname / username
|
|
@@ -366,6 +369,11 @@ class VOMS2CSSynchronizer:
|
|
|
366
369
|
newDiracName = "%s_%d" % (trialName, ind)
|
|
367
370
|
ind += 1
|
|
368
371
|
|
|
372
|
+
# Do not consider users with Suspended status in VOMS
|
|
373
|
+
if self.vomsUserDict[dn]["suspended"] or self.vomsUserDict[dn]["certSuspended"]:
|
|
374
|
+
resultDict["SuspendedUsers"].append(newDiracName)
|
|
375
|
+
continue
|
|
376
|
+
|
|
369
377
|
# We now have everything to add the new user
|
|
370
378
|
userDict = {"DN": dn, "CA": self.vomsUserDict[dn]["CA"], "Email": self.vomsUserDict[dn]["mail"]}
|
|
371
379
|
groupsWithRole = []
|
|
@@ -396,11 +404,16 @@ class VOMS2CSSynchronizer:
|
|
|
396
404
|
self.log.info(f"Adding new user {newDiracName}: {str(userDict)}")
|
|
397
405
|
result = self.csapi.modifyUser(newDiracName, userDict, createIfNonExistant=True)
|
|
398
406
|
if not result["OK"]:
|
|
399
|
-
self.log.warn(f"Failed adding new user {newDiracName}")
|
|
407
|
+
self.log.warn(f"Failed adding new user {newDiracName}", result["Message"])
|
|
400
408
|
resultDict["NewUsers"].append(newDiracName)
|
|
401
409
|
newAddedUserDict[newDiracName] = userDict
|
|
402
410
|
continue
|
|
403
411
|
|
|
412
|
+
# If we have a new user with multiple DN,
|
|
413
|
+
# it's a bit tricky, so first create it with a single one
|
|
414
|
+
# and at the next iteration add more DNs
|
|
415
|
+
if diracName in newAddedUserDict:
|
|
416
|
+
continue
|
|
404
417
|
# We have an already existing user
|
|
405
418
|
modified = False
|
|
406
419
|
suspendedInVOMS = self.vomsUserDict[dn]["suspended"] or self.vomsUserDict[dn]["certSuspended"]
|
|
@@ -581,7 +594,7 @@ class VOMS2CSSynchronizer:
|
|
|
581
594
|
|
|
582
595
|
# Try to fill in the DiracX section
|
|
583
596
|
if self.useIAM:
|
|
584
|
-
iam_subs = self.iamSrv.getUsersSub()
|
|
597
|
+
iam_subs = self.iamSrv.getUsersSub(self.vo)
|
|
585
598
|
diracx_vo_config = {"DiracX": {"CsSync": {"VOs": {self.vo: {"UserSubjects": iam_subs}}}}}
|
|
586
599
|
iam_sub_cfg = CFG()
|
|
587
600
|
iam_sub_cfg.loadFromDict(diracx_vo_config)
|
|
@@ -11,9 +11,26 @@ from DIRAC.ConfigurationSystem.private.ConfigurationData import ConfigurationDat
|
|
|
11
11
|
localCFGData = ConfigurationData(False)
|
|
12
12
|
mergedCFG = CFG()
|
|
13
13
|
mergedCFG.loadFromBuffer(
|
|
14
|
-
"""
|
|
14
|
+
r"""
|
|
15
|
+
DIRAC
|
|
16
|
+
{
|
|
17
|
+
PreferredURLPatterns = dips://.*\.site:.*
|
|
18
|
+
PreferredURLPatterns += dips://.*\.other:.*
|
|
19
|
+
}
|
|
15
20
|
Systems
|
|
16
21
|
{
|
|
22
|
+
Configuration
|
|
23
|
+
{
|
|
24
|
+
URLs
|
|
25
|
+
{
|
|
26
|
+
Server = dips://server1.site:1234/Configuration/Server
|
|
27
|
+
Server += dips://server2.site:1234/Configuration/Server
|
|
28
|
+
Server += dips://server3.site:1234/Configuration/Server
|
|
29
|
+
Server += dips://server4.site:1234/Configuration/Server
|
|
30
|
+
Server += dips://server.other:1234/Configuration/Server
|
|
31
|
+
Server += dips://server.external:1234/Configuration/Server
|
|
32
|
+
}
|
|
33
|
+
}
|
|
17
34
|
WorkloadManagement
|
|
18
35
|
{
|
|
19
36
|
URLs
|
|
@@ -181,6 +198,29 @@ def test_getServiceURLs(pathFinder, serviceName, service, failover, result):
|
|
|
181
198
|
assert set(pathFinder.getServiceURLs(serviceName, service=service, failover=failover)) == result
|
|
182
199
|
|
|
183
200
|
|
|
201
|
+
def test_getServiceURLsOrdering(pathFinder):
|
|
202
|
+
"""Ensure the PreferredURLPattern option is respected"""
|
|
203
|
+
all_results = set()
|
|
204
|
+
for _ in range(10_000):
|
|
205
|
+
urls = pathFinder.getServiceURLs("Configuration", service="Server")
|
|
206
|
+
assert set(urls) == {
|
|
207
|
+
"dips://server1.site:1234/Configuration/Server",
|
|
208
|
+
"dips://server2.site:1234/Configuration/Server",
|
|
209
|
+
"dips://server3.site:1234/Configuration/Server",
|
|
210
|
+
"dips://server4.site:1234/Configuration/Server",
|
|
211
|
+
"dips://server.other:1234/Configuration/Server",
|
|
212
|
+
"dips://server.external:1234/Configuration/Server",
|
|
213
|
+
}
|
|
214
|
+
# The second to last URL should always be "other"
|
|
215
|
+
assert urls[-2] == "dips://server.other:1234/Configuration/Server"
|
|
216
|
+
# The last URL should always be the one which isn't preferred
|
|
217
|
+
assert urls[-1] == "dips://server.external:1234/Configuration/Server"
|
|
218
|
+
all_results.add(tuple(urls))
|
|
219
|
+
# There are 4! = 24 possible orderings of the preferred URLs, we should have seen all
|
|
220
|
+
# of them at least once in 10_000 iterations
|
|
221
|
+
assert len(all_results) >= 24
|
|
222
|
+
|
|
223
|
+
|
|
184
224
|
@pytest.mark.parametrize(
|
|
185
225
|
"system, failover, result",
|
|
186
226
|
[
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import time
|
|
2
2
|
|
|
3
3
|
from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData
|
|
4
|
-
from DIRAC.ConfigurationSystem.Client.PathFinder import getGatewayURLs
|
|
4
|
+
from DIRAC.ConfigurationSystem.Client.PathFinder import getGatewayURLs, groupURLsByPriority
|
|
5
5
|
from DIRAC.Core.Utilities import List
|
|
6
6
|
from DIRAC.Core.Utilities.EventDispatcher import gEventDispatcher
|
|
7
7
|
from DIRAC.Core.Utilities.ReturnValues import S_ERROR, S_OK
|
|
@@ -138,7 +138,9 @@ class RefresherBase:
|
|
|
138
138
|
if not initialServerList:
|
|
139
139
|
return S_OK()
|
|
140
140
|
|
|
141
|
-
randomServerList =
|
|
141
|
+
randomServerList = []
|
|
142
|
+
for urlGroup in groupURLsByPriority(initialServerList):
|
|
143
|
+
randomServerList.extend(List.randomize(urlGroup))
|
|
142
144
|
gLogger.debug(f"Randomized server list is {', '.join(randomServerList)}")
|
|
143
145
|
|
|
144
146
|
for sServer in randomServerList:
|
DIRAC/Core/Base/API.py
CHANGED
|
@@ -3,13 +3,11 @@
|
|
|
3
3
|
import pprint
|
|
4
4
|
import sys
|
|
5
5
|
|
|
6
|
-
from DIRAC import
|
|
7
|
-
from DIRAC.Core.Security.ProxyInfo import getProxyInfo, formatProxyInfoAsString
|
|
8
|
-
from DIRAC.Core.Utilities.Version import getCurrentVersion
|
|
6
|
+
from DIRAC import S_ERROR, S_OK, gLogger
|
|
9
7
|
from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getDNForUsername
|
|
10
8
|
from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getSites
|
|
11
|
-
|
|
12
|
-
|
|
9
|
+
from DIRAC.Core.Security.ProxyInfo import formatProxyInfoAsString, getProxyInfo
|
|
10
|
+
from DIRAC.Core.Utilities.Version import getCurrentVersion
|
|
13
11
|
|
|
14
12
|
|
|
15
13
|
def _printFormattedDictList(dictList, fields, uniqueField, orderBy):
|
|
@@ -60,8 +58,7 @@ class API:
|
|
|
60
58
|
def __init__(self):
|
|
61
59
|
"""c'tor"""
|
|
62
60
|
self._printFormattedDictList = _printFormattedDictList
|
|
63
|
-
self.log = gLogger.getSubLogger(
|
|
64
|
-
self.section = COMPONENT_NAME
|
|
61
|
+
self.log = gLogger.getSubLogger(self.__class__.__name__)
|
|
65
62
|
self.pPrint = pprint.PrettyPrinter()
|
|
66
63
|
# Global error dictionary
|
|
67
64
|
self.errorDict = {}
|
DIRAC/Core/Base/SQLAlchemyDB.py
CHANGED
|
@@ -200,6 +200,7 @@ class ServiceReactor:
|
|
|
200
200
|
services at the same time
|
|
201
201
|
"""
|
|
202
202
|
sel = self.__getListeningSelector(svcName)
|
|
203
|
+
throttleExpires = None
|
|
203
204
|
while self.__alive:
|
|
204
205
|
clientTransport = None
|
|
205
206
|
try:
|
|
@@ -223,12 +224,19 @@ class ServiceReactor:
|
|
|
223
224
|
gLogger.warn(f"Client connected from banned ip {clientIP}")
|
|
224
225
|
clientTransport.close()
|
|
225
226
|
continue
|
|
227
|
+
# Handle throttling
|
|
228
|
+
if self.__services[svcName].wantsThrottle and throttleExpires is None:
|
|
229
|
+
throttleExpires = time.time() + THROTTLE_SERVICE_SLEEP_SECONDS
|
|
230
|
+
if throttleExpires:
|
|
231
|
+
if time.time() > throttleExpires:
|
|
232
|
+
throttleExpires = None
|
|
233
|
+
else:
|
|
234
|
+
gLogger.warn("Rejecting client due to throttling", str(clientTransport.getRemoteAddress()))
|
|
235
|
+
clientTransport.close()
|
|
236
|
+
continue
|
|
226
237
|
# Handle connection
|
|
227
238
|
self.__stats.connectionStablished()
|
|
228
239
|
self.__services[svcName].handleConnection(clientTransport)
|
|
229
|
-
while self.__services[svcName].wantsThrottle:
|
|
230
|
-
gLogger.warn("Sleeping as service requested throttling", svcName)
|
|
231
|
-
time.sleep(THROTTLE_SERVICE_SLEEP_SECONDS)
|
|
232
240
|
# Renew context?
|
|
233
241
|
now = time.time()
|
|
234
242
|
renewed = False
|
|
@@ -323,7 +323,7 @@ class BaseClient:
|
|
|
323
323
|
pass
|
|
324
324
|
|
|
325
325
|
# We randomize the list, and add at the end the failover URLs (System/FailoverURLs/Component)
|
|
326
|
-
urlsList = List.
|
|
326
|
+
urlsList = List.fromChar(urls, ",") + failoverUrls
|
|
327
327
|
self.__nbOfUrls = len(urlsList)
|
|
328
328
|
self.__nbOfRetry = (
|
|
329
329
|
2 if self.__nbOfUrls > 2 else 3
|
|
@@ -445,7 +445,6 @@ and this is thread {cThID}
|
|
|
445
445
|
return self.__initStatus
|
|
446
446
|
if self.__enableThreadCheck:
|
|
447
447
|
self.__checkThreadID()
|
|
448
|
-
|
|
449
448
|
gLogger.debug(f"Trying to connect to: {self.serviceURL}")
|
|
450
449
|
try:
|
|
451
450
|
# Calls the transport method of the apropriate protocol.
|
|
@@ -110,19 +110,18 @@ class SSLTransport(BaseTransport):
|
|
|
110
110
|
if self.serverMode():
|
|
111
111
|
raise RuntimeError("SSLTransport is in server mode.")
|
|
112
112
|
|
|
113
|
-
|
|
113
|
+
errors = []
|
|
114
114
|
host, port = self.stServerAddress
|
|
115
115
|
|
|
116
116
|
# The following piece of code was inspired by the python socket documentation
|
|
117
117
|
# as well as the implementation of M2Crypto.httpslib.HTTPSConnection
|
|
118
118
|
|
|
119
|
-
#
|
|
120
|
-
# a host name.
|
|
119
|
+
# Get all available addresses (IPv6 and IPv4) and try them in order
|
|
121
120
|
try:
|
|
122
121
|
addrInfoList = socket.getaddrinfo(host, port, socket.AF_UNSPEC, socket.SOCK_STREAM)
|
|
123
122
|
except OSError as e:
|
|
124
123
|
return S_ERROR(f"DNS lookup failed {e!r}")
|
|
125
|
-
for family, _socketType, _proto, _canonname,
|
|
124
|
+
for family, _socketType, _proto, _canonname, socketAddress in addrInfoList:
|
|
126
125
|
try:
|
|
127
126
|
self.oSocket = SSL.Connection(self.__ctx, family=family)
|
|
128
127
|
|
|
@@ -138,7 +137,10 @@ class SSLTransport(BaseTransport):
|
|
|
138
137
|
# set SNI server name since we know it at this point
|
|
139
138
|
self.oSocket.set_tlsext_host_name(host)
|
|
140
139
|
|
|
141
|
-
|
|
140
|
+
# tell the connection which host we are connecting to so we can
|
|
141
|
+
# use the address we obtained from DNS
|
|
142
|
+
self.oSocket.set1_host(host)
|
|
143
|
+
self.oSocket.connect(socketAddress)
|
|
142
144
|
|
|
143
145
|
# Once the connection is established, we can use the timeout
|
|
144
146
|
# asked for RPC
|
|
@@ -151,12 +153,12 @@ class SSLTransport(BaseTransport):
|
|
|
151
153
|
# They should be propagated upwards and caught by the BaseClient
|
|
152
154
|
# not to enter the retry loop
|
|
153
155
|
except OSError as e:
|
|
154
|
-
|
|
156
|
+
errors.append(f"{socketAddress} {e}:{repr(e)}")
|
|
155
157
|
|
|
156
158
|
if self.oSocket is not None:
|
|
157
159
|
self.close()
|
|
158
160
|
|
|
159
|
-
return S_ERROR(
|
|
161
|
+
return S_ERROR("; ".join(errors))
|
|
160
162
|
|
|
161
163
|
def initAsServer(self):
|
|
162
164
|
"""Prepare this server socket for use."""
|
|
@@ -116,7 +116,9 @@ def getM2SSLContext(ctx=None, **kwargs):
|
|
|
116
116
|
# CHRIS: I think clientMode was just an internal of pyGSI implementation
|
|
117
117
|
# if kwargs.get('clientMode', False) and not kwargs.get('useCertificates', False):
|
|
118
118
|
# if not kwargs.get('useCertificates', False):
|
|
119
|
-
if kwargs.get("bServerMode", False) or
|
|
119
|
+
if kwargs.get("bServerMode", False) or (
|
|
120
|
+
kwargs.get("useCertificates", False) and not kwargs.get("proxyLocation", False)
|
|
121
|
+
):
|
|
120
122
|
# Server mode always uses hostcert
|
|
121
123
|
__loadM2SSLCTXHostcert(ctx)
|
|
122
124
|
|
DIRAC/Core/LCG/GOCDBClient.py
CHANGED
|
@@ -210,11 +210,11 @@ class GOCDBClient:
|
|
|
210
210
|
if ongoing:
|
|
211
211
|
params += "&ongoing_only=yes"
|
|
212
212
|
|
|
213
|
-
caPath = getCAsLocation()
|
|
214
|
-
|
|
215
213
|
try:
|
|
216
214
|
response = requests.get(
|
|
217
|
-
"https://goc.egi.eu/gocdbpi/public/?method=get_downtime&topentity=" + params,
|
|
215
|
+
"https://goc.egi.eu/gocdbpi/public/?method=get_downtime&topentity=" + params,
|
|
216
|
+
verify=getCAsLocation(),
|
|
217
|
+
timeout=20,
|
|
218
218
|
)
|
|
219
219
|
response.raise_for_status()
|
|
220
220
|
except requests.exceptions.RequestException as e:
|
|
@@ -269,8 +269,7 @@ class GOCDBClient:
|
|
|
269
269
|
gocdb_ep = gocdb_ep + "&topentity=" + entity
|
|
270
270
|
gocdb_ep = gocdb_ep + when + gocdbpi_startDate + "&scope="
|
|
271
271
|
|
|
272
|
-
|
|
273
|
-
dtPage = requests.get(gocdb_ep, verify=caPath, timeout=20)
|
|
272
|
+
dtPage = requests.get(gocdb_ep, verify=getCAsLocation(), timeout=20)
|
|
274
273
|
|
|
275
274
|
dt = dtPage.text
|
|
276
275
|
|
|
@@ -294,8 +293,7 @@ class GOCDBClient:
|
|
|
294
293
|
# GOCDB-PI query
|
|
295
294
|
gocdb_ep = "https://goc.egi.eu/gocdbpi/public/?method=get_service_endpoint&" + granularity + "=" + entity
|
|
296
295
|
|
|
297
|
-
|
|
298
|
-
service_endpoint_page = requests.get(gocdb_ep, verify=caPath, timeout=20)
|
|
296
|
+
service_endpoint_page = requests.get(gocdb_ep, verify=getCAsLocation(), timeout=20)
|
|
299
297
|
|
|
300
298
|
return service_endpoint_page.text
|
|
301
299
|
|
DIRAC/Core/Security/DiracX.py
CHANGED
|
@@ -10,21 +10,28 @@ __all__ = (
|
|
|
10
10
|
|
|
11
11
|
import base64
|
|
12
12
|
import functools
|
|
13
|
+
import hashlib
|
|
13
14
|
import importlib
|
|
14
15
|
import json
|
|
15
16
|
import re
|
|
16
17
|
import textwrap
|
|
18
|
+
from collections.abc import Iterator
|
|
17
19
|
from contextlib import contextmanager
|
|
18
20
|
from pathlib import Path
|
|
19
|
-
from tempfile import NamedTemporaryFile
|
|
21
|
+
from tempfile import NamedTemporaryFile, gettempdir
|
|
20
22
|
from typing import Any
|
|
21
23
|
|
|
22
|
-
|
|
24
|
+
try:
|
|
25
|
+
from diracx.client.sync import SyncDiracClient
|
|
26
|
+
except ImportError:
|
|
27
|
+
# TODO: Remove this once diracx is tagged
|
|
28
|
+
from diracx.client import DiracClient as SyncDiracClient
|
|
23
29
|
from diracx.core.models import TokenResponse
|
|
24
30
|
from diracx.core.preferences import DiracxPreferences
|
|
25
31
|
from diracx.core.utils import serialize_credentials
|
|
26
32
|
|
|
27
33
|
from DIRAC import gConfig, gLogger
|
|
34
|
+
from DIRAC.Core.Utilities.File import secureOpenForWrite
|
|
28
35
|
|
|
29
36
|
from DIRAC.ConfigurationSystem.Client.Helpers import Registry
|
|
30
37
|
from DIRAC.Core.Security.Locations import getDefaultProxyLocation
|
|
@@ -33,14 +40,14 @@ from DIRAC.Core.Utilities.ReturnValues import convertToReturnValue, returnValueO
|
|
|
33
40
|
|
|
34
41
|
PEM_BEGIN = "-----BEGIN DIRACX-----"
|
|
35
42
|
PEM_END = "-----END DIRACX-----"
|
|
36
|
-
RE_DIRACX_PEM = re.compile(rf"{PEM_BEGIN}\n(
|
|
43
|
+
RE_DIRACX_PEM = re.compile(rf"{PEM_BEGIN}\n(.*?)\n{PEM_END}", re.DOTALL)
|
|
37
44
|
|
|
38
45
|
|
|
39
46
|
@convertToReturnValue
|
|
40
47
|
def addTokenToPEM(pemPath, group):
|
|
41
48
|
from DIRAC.Core.Base.Client import Client
|
|
42
49
|
|
|
43
|
-
vo = Registry.
|
|
50
|
+
vo = Registry.getVOForGroup(group)
|
|
44
51
|
if not vo:
|
|
45
52
|
gLogger.error(f"ERROR: Could not find VO for group {group}, DiracX will not work!")
|
|
46
53
|
disabledVOs = gConfig.getValue("/DiracX/DisabledVOs", [])
|
|
@@ -55,21 +62,26 @@ def addTokenToPEM(pemPath, group):
|
|
|
55
62
|
token_type=token_content.get("token_type"),
|
|
56
63
|
refresh_token=token_content.get("refresh_token"),
|
|
57
64
|
)
|
|
58
|
-
|
|
59
65
|
token_pem = f"{PEM_BEGIN}\n"
|
|
60
66
|
data = base64.b64encode(serialize_credentials(token).encode("utf-8")).decode()
|
|
61
67
|
token_pem += textwrap.fill(data, width=64)
|
|
62
68
|
token_pem += f"\n{PEM_END}\n"
|
|
63
69
|
|
|
64
|
-
|
|
65
|
-
|
|
70
|
+
pem = Path(pemPath).read_text()
|
|
71
|
+
# Remove any existing DiracX token there would be
|
|
72
|
+
new_pem = re.sub(RE_DIRACX_PEM, "", pem)
|
|
73
|
+
new_pem += token_pem
|
|
74
|
+
|
|
75
|
+
Path(pemPath).write_text(new_pem)
|
|
66
76
|
|
|
67
77
|
|
|
68
78
|
def diracxTokenFromPEM(pemPath) -> dict[str, Any] | None:
|
|
69
79
|
"""Extract the DiracX token from the proxy PEM file"""
|
|
70
80
|
pem = Path(pemPath).read_text()
|
|
71
|
-
if match := RE_DIRACX_PEM.
|
|
72
|
-
match
|
|
81
|
+
if match := RE_DIRACX_PEM.findall(pem):
|
|
82
|
+
if len(match) > 1:
|
|
83
|
+
raise ValueError("Found multiple DiracX tokens, this should never happen")
|
|
84
|
+
match = match[0]
|
|
73
85
|
return json.loads(base64.b64decode(match).decode("utf-8"))
|
|
74
86
|
|
|
75
87
|
|
|
@@ -82,7 +94,7 @@ class FutureClient:
|
|
|
82
94
|
|
|
83
95
|
|
|
84
96
|
@contextmanager
|
|
85
|
-
def DiracXClient() ->
|
|
97
|
+
def DiracXClient() -> Iterator[SyncDiracClient]:
|
|
86
98
|
"""Get a DiracX client instance with the current user's credentials"""
|
|
87
99
|
diracxUrl = gConfig.getValue("/DiracX/URL")
|
|
88
100
|
if not diracxUrl:
|
|
@@ -93,14 +105,16 @@ def DiracXClient() -> _DiracClient:
|
|
|
93
105
|
if not diracxToken:
|
|
94
106
|
raise ValueError(f"No diracx token in the proxy file {proxyLocation}")
|
|
95
107
|
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
token_file.
|
|
108
|
+
hash = hashlib.sha256(diracxToken["refresh_token"].split(".")[1].encode())
|
|
109
|
+
token_file = Path(gettempdir()) / f"dx_{hash.hexdigest()}"
|
|
110
|
+
if not token_file.exists():
|
|
111
|
+
token_file.parent.mkdir(parents=True, exist_ok=True)
|
|
112
|
+
with secureOpenForWrite(token_file) as (fd, _):
|
|
113
|
+
fd.write(json.dumps(diracxToken))
|
|
100
114
|
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
115
|
+
pref = DiracxPreferences(url=diracxUrl, credentials_path=token_file)
|
|
116
|
+
with SyncDiracClient(diracx_preferences=pref) as api:
|
|
117
|
+
yield api
|
|
104
118
|
|
|
105
119
|
|
|
106
120
|
def addRPCStub(meth):
|
|
@@ -3,11 +3,7 @@
|
|
|
3
3
|
|
|
4
4
|
import requests
|
|
5
5
|
|
|
6
|
-
from DIRAC import gConfig, gLogger
|
|
7
|
-
from DIRAC.Core.Utilities import DErrno
|
|
8
|
-
from DIRAC.Core.Security.Locations import getProxyLocation, getCAsLocation
|
|
9
|
-
from DIRAC.Core.Utilities.Decorators import deprecated
|
|
10
|
-
from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getVOOption
|
|
6
|
+
from DIRAC import S_OK, gConfig, gLogger
|
|
11
7
|
from DIRAC.ConfigurationSystem.Client.Helpers.CSGlobals import getVO
|
|
12
8
|
|
|
13
9
|
|
|
@@ -148,15 +144,15 @@ class IAMService:
|
|
|
148
144
|
result = S_OK({"Users": users, "Errors": errors})
|
|
149
145
|
return result
|
|
150
146
|
|
|
151
|
-
def getUsersSub(self) -> dict[str, str]:
|
|
147
|
+
def getUsersSub(self, vo=None) -> dict[str, str]:
|
|
152
148
|
"""
|
|
153
149
|
Return the mapping based on IAM sub:
|
|
154
150
|
{nickname : sub}
|
|
155
|
-
|
|
156
151
|
"""
|
|
157
152
|
iam_users_raw = self._getIamUserDump()
|
|
158
153
|
diracx_user_section = {}
|
|
159
154
|
for user_info in iam_users_raw:
|
|
155
|
+
userGroups = [grp["display"] for grp in user_info.get("groups", [])]
|
|
160
156
|
# The nickname is available in the list of attributes
|
|
161
157
|
# (if configured so)
|
|
162
158
|
# in the form {'name': 'nickname', 'value': 'chaen'}
|
|
@@ -170,9 +166,8 @@ class IAMService:
|
|
|
170
166
|
except (KeyError, IndexError):
|
|
171
167
|
nickname = user_info["userName"]
|
|
172
168
|
sub = user_info["id"]
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
169
|
+
if not vo or vo in userGroups:
|
|
170
|
+
diracx_user_section[nickname] = sub
|
|
176
171
|
# reorder it
|
|
177
172
|
diracx_user_section = dict(sorted(diracx_user_section.items()))
|
|
178
173
|
|