DIRAC 9.0.0a60__py3-none-any.whl → 9.0.0a62__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- DIRAC/AccountingSystem/Client/AccountingCLI.py +0 -140
- DIRAC/AccountingSystem/Client/DataStoreClient.py +0 -13
- DIRAC/AccountingSystem/Client/Types/BaseAccountingType.py +0 -7
- DIRAC/AccountingSystem/ConfigTemplate.cfg +0 -5
- DIRAC/AccountingSystem/Service/DataStoreHandler.py +0 -72
- DIRAC/ConfigurationSystem/Client/Helpers/Registry.py +34 -6
- DIRAC/ConfigurationSystem/Client/LocalConfiguration.py +3 -0
- DIRAC/ConfigurationSystem/Client/VOMS2CSSynchronizer.py +8 -1
- DIRAC/Core/DISET/private/BaseClient.py +1 -2
- DIRAC/Core/Security/ProxyInfo.py +9 -5
- DIRAC/Core/Utilities/Os.py +32 -1
- DIRAC/Core/scripts/dirac_apptainer_exec.py +12 -4
- DIRAC/DataManagementSystem/Utilities/DMSHelpers.py +5 -1
- DIRAC/Interfaces/API/DiracAdmin.py +17 -5
- DIRAC/Interfaces/scripts/dirac_admin_allow_site.py +7 -1
- DIRAC/Interfaces/scripts/dirac_admin_ban_site.py +7 -1
- DIRAC/MonitoringSystem/Client/Types/WMSHistory.py +4 -0
- DIRAC/MonitoringSystem/DB/MonitoringDB.py +0 -20
- DIRAC/MonitoringSystem/Service/MonitoringHandler.py +0 -33
- DIRAC/MonitoringSystem/Service/WebAppHandler.py +68 -1
- DIRAC/ResourceStatusSystem/Client/SiteStatus.py +4 -2
- DIRAC/ResourceStatusSystem/Utilities/CSHelpers.py +2 -31
- DIRAC/ResourceStatusSystem/scripts/dirac_rss_set_status.py +18 -4
- DIRAC/Resources/Computing/BatchSystems/Condor.py +126 -108
- DIRAC/Resources/Computing/HTCondorCEComputingElement.py +33 -15
- DIRAC/Resources/Computing/test/Test_HTCondorCEComputingElement.py +67 -49
- DIRAC/Resources/Storage/StorageBase.py +4 -2
- DIRAC/TransformationSystem/Agent/TaskManagerAgentBase.py +10 -13
- DIRAC/TransformationSystem/Agent/TransformationAgent.py +22 -1
- DIRAC/TransformationSystem/Agent/TransformationCleaningAgent.py +6 -3
- DIRAC/TransformationSystem/Client/Utilities.py +6 -0
- DIRAC/WorkloadManagementSystem/Agent/JobCleaningAgent.py +4 -3
- DIRAC/WorkloadManagementSystem/Agent/StatesAccountingAgent.py +34 -1
- DIRAC/WorkloadManagementSystem/Client/JobMonitoringClient.py +0 -9
- DIRAC/WorkloadManagementSystem/Client/SandboxStoreClient.py +0 -37
- DIRAC/WorkloadManagementSystem/DB/JobDB.py +0 -58
- DIRAC/WorkloadManagementSystem/DB/SandboxMetadataDB.py +25 -37
- DIRAC/WorkloadManagementSystem/Executor/JobSanity.py +3 -3
- DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py +9 -6
- DIRAC/WorkloadManagementSystem/Service/JobManagerHandler.py +0 -10
- DIRAC/WorkloadManagementSystem/Service/JobMonitoringHandler.py +0 -126
- DIRAC/WorkloadManagementSystem/Service/PilotManagerHandler.py +6 -3
- DIRAC/WorkloadManagementSystem/Service/SandboxStoreHandler.py +5 -51
- {dirac-9.0.0a60.dist-info → dirac-9.0.0a62.dist-info}/METADATA +1 -1
- {dirac-9.0.0a60.dist-info → dirac-9.0.0a62.dist-info}/RECORD +49 -49
- {dirac-9.0.0a60.dist-info → dirac-9.0.0a62.dist-info}/WHEEL +0 -0
- {dirac-9.0.0a60.dist-info → dirac-9.0.0a62.dist-info}/entry_points.txt +0 -0
- {dirac-9.0.0a60.dist-info → dirac-9.0.0a62.dist-info}/licenses/LICENSE +0 -0
- {dirac-9.0.0a60.dist-info → dirac-9.0.0a62.dist-info}/top_level.txt +0 -0
|
@@ -57,99 +57,6 @@ class AccountingCLI(CLI):
|
|
|
57
57
|
traceback.print_tb(sys.exc_info()[2])
|
|
58
58
|
print("________________________\n")
|
|
59
59
|
|
|
60
|
-
def do_registerType(self, args):
|
|
61
|
-
"""
|
|
62
|
-
Registers a new accounting type
|
|
63
|
-
Usage : registerType <typeName>
|
|
64
|
-
<DIRACRoot>/DIRAC/AccountingSystem/Client/Types/<typeName>
|
|
65
|
-
should exist and inherit the base type
|
|
66
|
-
"""
|
|
67
|
-
try:
|
|
68
|
-
argList = args.split()
|
|
69
|
-
if argList:
|
|
70
|
-
typeName = argList[0].strip()
|
|
71
|
-
else:
|
|
72
|
-
gLogger.error("No type name specified")
|
|
73
|
-
return
|
|
74
|
-
# Try to import the type
|
|
75
|
-
result = self.objectLoader.loadObject(f"DIRAC.AccountingSystem.Client.Types.{typeName}")
|
|
76
|
-
if not result["OK"]:
|
|
77
|
-
return result
|
|
78
|
-
typeClass = result["Value"]
|
|
79
|
-
|
|
80
|
-
gLogger.info(f"Loaded type {typeClass.__name__}")
|
|
81
|
-
typeDef = typeClass().getDefinition()
|
|
82
|
-
acClient = DataStoreClient()
|
|
83
|
-
retVal = acClient.registerType(*typeDef)
|
|
84
|
-
if retVal["OK"]:
|
|
85
|
-
gLogger.info("Type registered successfully")
|
|
86
|
-
else:
|
|
87
|
-
gLogger.error(f"Error: {retVal['Message']}")
|
|
88
|
-
except Exception:
|
|
89
|
-
self.showTraceback()
|
|
90
|
-
|
|
91
|
-
def do_resetBucketLength(self, args):
|
|
92
|
-
"""
|
|
93
|
-
Set the bucket Length. Will trigger a recalculation of buckets. Can take a while.
|
|
94
|
-
Usage : resetBucketLength <typeName>
|
|
95
|
-
<DIRACRoot>/DIRAC/AccountingSystem/Client/Types/<typeName>
|
|
96
|
-
should exist and inherit the base type
|
|
97
|
-
"""
|
|
98
|
-
try:
|
|
99
|
-
argList = args.split()
|
|
100
|
-
if argList:
|
|
101
|
-
typeName = argList[0].strip()
|
|
102
|
-
else:
|
|
103
|
-
gLogger.error("No type name specified")
|
|
104
|
-
return
|
|
105
|
-
|
|
106
|
-
# Try to import the type
|
|
107
|
-
result = self.objectLoader.loadObject(f"DIRAC.AccountingSystem.Client.Types.{typeName}")
|
|
108
|
-
if not result["OK"]:
|
|
109
|
-
return result
|
|
110
|
-
typeClass = result["Value"]
|
|
111
|
-
gLogger.info(f"Loaded type {typeClass.__name__}")
|
|
112
|
-
typeDef = typeClass().getDefinition()
|
|
113
|
-
acClient = DataStoreClient()
|
|
114
|
-
retVal = acClient.setBucketsLength(typeDef[0], typeDef[3])
|
|
115
|
-
if retVal["OK"]:
|
|
116
|
-
gLogger.info("Type registered successfully")
|
|
117
|
-
else:
|
|
118
|
-
gLogger.error(f"Error: {retVal['Message']}")
|
|
119
|
-
except Exception:
|
|
120
|
-
self.showTraceback()
|
|
121
|
-
|
|
122
|
-
def do_regenerateBuckets(self, args):
|
|
123
|
-
"""
|
|
124
|
-
Regenerate buckets for type. Can take a while.
|
|
125
|
-
Usage : regenerateBuckets <typeName>
|
|
126
|
-
<DIRACRoot>/DIRAC/AccountingSystem/Client/Types/<typeName>
|
|
127
|
-
should exist and inherit the base type
|
|
128
|
-
"""
|
|
129
|
-
try:
|
|
130
|
-
argList = args.split()
|
|
131
|
-
if argList:
|
|
132
|
-
typeName = argList[0].strip()
|
|
133
|
-
else:
|
|
134
|
-
gLogger.error("No type name specified")
|
|
135
|
-
return
|
|
136
|
-
|
|
137
|
-
# Try to import the type
|
|
138
|
-
result = self.objectLoader.loadObject(f"DIRAC.AccountingSystem.Client.Types.{typeName}")
|
|
139
|
-
if not result["OK"]:
|
|
140
|
-
return result
|
|
141
|
-
typeClass = result["Value"]
|
|
142
|
-
gLogger.info(f"Loaded type {typeClass.__name__}")
|
|
143
|
-
typeDef = typeClass().getDefinition()
|
|
144
|
-
acClient = DataStoreClient()
|
|
145
|
-
retVal = acClient.regenerateBuckets(typeDef[0])
|
|
146
|
-
if retVal["OK"]:
|
|
147
|
-
gLogger.info("Buckets recalculated!")
|
|
148
|
-
else:
|
|
149
|
-
gLogger.error(f"Error: {retVal['Message']}")
|
|
150
|
-
except Exception:
|
|
151
|
-
self.showTraceback()
|
|
152
|
-
|
|
153
60
|
def do_showRegisteredTypes(self, args):
|
|
154
61
|
"""
|
|
155
62
|
Get a list of registered types
|
|
@@ -170,50 +77,3 @@ class AccountingCLI(CLI):
|
|
|
170
77
|
print(" Value fields:\n %s" % "\n ".join(typeList[2]))
|
|
171
78
|
except Exception:
|
|
172
79
|
self.showTraceback()
|
|
173
|
-
|
|
174
|
-
def do_deleteType(self, args):
|
|
175
|
-
"""
|
|
176
|
-
Delete a registered accounting type.
|
|
177
|
-
Usage : deleteType <typeName>
|
|
178
|
-
WARN! It will delete all data associated to that type! VERY DANGEROUS!
|
|
179
|
-
If you screw it, you'll discover a new dimension of pain and doom! :)
|
|
180
|
-
"""
|
|
181
|
-
try:
|
|
182
|
-
argList = args.split()
|
|
183
|
-
if argList:
|
|
184
|
-
typeName = argList[0].strip()
|
|
185
|
-
else:
|
|
186
|
-
gLogger.error("No type name specified")
|
|
187
|
-
return
|
|
188
|
-
|
|
189
|
-
choice = input(
|
|
190
|
-
f"Are you completely sure you want to delete type {typeName} and all it's data? yes/no [no]: "
|
|
191
|
-
)
|
|
192
|
-
choice = choice.lower()
|
|
193
|
-
if choice not in ("yes", "y"):
|
|
194
|
-
print("Delete aborted")
|
|
195
|
-
return
|
|
196
|
-
|
|
197
|
-
acClient = DataStoreClient()
|
|
198
|
-
retVal = acClient.deleteType(typeName)
|
|
199
|
-
if not retVal["OK"]:
|
|
200
|
-
gLogger.error(f"Error: {retVal['Message']}")
|
|
201
|
-
return
|
|
202
|
-
print("Hope you meant it, because it's done")
|
|
203
|
-
except Exception:
|
|
204
|
-
self.showTraceback()
|
|
205
|
-
|
|
206
|
-
def do_compactBuckets(self, args):
|
|
207
|
-
"""
|
|
208
|
-
Compact buckets table
|
|
209
|
-
Usage : compactBuckets
|
|
210
|
-
"""
|
|
211
|
-
try:
|
|
212
|
-
acClient = DataStoreClient()
|
|
213
|
-
retVal = acClient.compactDB()
|
|
214
|
-
if not retVal["OK"]:
|
|
215
|
-
gLogger.error(f"Error: {retVal['Message']}")
|
|
216
|
-
return
|
|
217
|
-
gLogger.info("Done")
|
|
218
|
-
except Exception:
|
|
219
|
-
self.showTraceback()
|
|
@@ -122,19 +122,6 @@ class DataStoreClient(Client):
|
|
|
122
122
|
|
|
123
123
|
return S_OK()
|
|
124
124
|
|
|
125
|
-
def remove(self, register):
|
|
126
|
-
"""
|
|
127
|
-
Remove a Register from the Accounting DataStore
|
|
128
|
-
"""
|
|
129
|
-
if not self.__checkBaseType(register.__class__):
|
|
130
|
-
return S_ERROR("register is not a valid type (has to inherit from BaseAccountingType")
|
|
131
|
-
retVal = register.checkValues()
|
|
132
|
-
if not retVal["OK"]:
|
|
133
|
-
return retVal
|
|
134
|
-
if gConfig.getValue("/LocalSite/DisableAccounting", False):
|
|
135
|
-
return S_OK()
|
|
136
|
-
return self._getRPC().remove(*register.getValues())
|
|
137
|
-
|
|
138
125
|
|
|
139
126
|
def _sendToFailover(rpcStub):
|
|
140
127
|
"""Create a ForwardDISET operation for failover"""
|
|
@@ -161,13 +161,6 @@ class BaseAccountingType:
|
|
|
161
161
|
cD[self.fieldsList[iPos]] = self.valuesList[iPos]
|
|
162
162
|
return cD
|
|
163
163
|
|
|
164
|
-
def registerToServer(self):
|
|
165
|
-
"""
|
|
166
|
-
Register type in server
|
|
167
|
-
"""
|
|
168
|
-
rpcClient = Client(url="Accounting/DataStore")
|
|
169
|
-
return rpcClient.registerType(*self.getDefinition())
|
|
170
|
-
|
|
171
164
|
def commit(self):
|
|
172
165
|
"""
|
|
173
166
|
Commit register to server
|
|
@@ -9,11 +9,6 @@ Services
|
|
|
9
9
|
Authorization
|
|
10
10
|
{
|
|
11
11
|
Default = authenticated
|
|
12
|
-
compactDB = ServiceAdministrator
|
|
13
|
-
deleteType = ServiceAdministrator
|
|
14
|
-
registerType = ServiceAdministrator
|
|
15
|
-
setBucketsLength = ServiceAdministrator
|
|
16
|
-
regenerateBuckets = ServiceAdministrator
|
|
17
12
|
}
|
|
18
13
|
}
|
|
19
14
|
##END
|
|
@@ -14,7 +14,6 @@ import datetime
|
|
|
14
14
|
from DIRAC import S_ERROR, S_OK
|
|
15
15
|
from DIRAC.AccountingSystem.DB.MultiAccountingDB import MultiAccountingDB
|
|
16
16
|
from DIRAC.ConfigurationSystem.Client import PathFinder
|
|
17
|
-
from DIRAC.Core.Base.Client import Client
|
|
18
17
|
from DIRAC.Core.DISET.RequestHandler import RequestHandler, getServiceOption
|
|
19
18
|
from DIRAC.Core.Utilities import TimeUtilities
|
|
20
19
|
from DIRAC.Core.Utilities.ThreadScheduler import gThreadScheduler
|
|
@@ -39,30 +38,6 @@ class DataStoreHandler(RequestHandler):
|
|
|
39
38
|
gThreadScheduler.addPeriodicTask(60, cls.__acDB.loadPendingRecords)
|
|
40
39
|
return S_OK()
|
|
41
40
|
|
|
42
|
-
types_registerType = [str, list, list, list]
|
|
43
|
-
|
|
44
|
-
def export_registerType(self, typeName, definitionKeyFields, definitionAccountingFields, bucketsLength):
|
|
45
|
-
"""
|
|
46
|
-
Register a new type. (Only for all powerful admins)
|
|
47
|
-
"""
|
|
48
|
-
return self.__acDB.registerType(typeName, definitionKeyFields, definitionAccountingFields, bucketsLength)
|
|
49
|
-
|
|
50
|
-
types_setBucketsLength = [str, list]
|
|
51
|
-
|
|
52
|
-
def export_setBucketsLength(self, typeName, bucketsLength):
|
|
53
|
-
"""
|
|
54
|
-
Change the buckets Length. (Only for all powerful admins)
|
|
55
|
-
"""
|
|
56
|
-
return self.__acDB.changeBucketsLength(typeName, bucketsLength)
|
|
57
|
-
|
|
58
|
-
types_regenerateBuckets = [str]
|
|
59
|
-
|
|
60
|
-
def export_regenerateBuckets(self, typeName):
|
|
61
|
-
"""
|
|
62
|
-
Recalculate buckets. (Only for all powerful admins)
|
|
63
|
-
"""
|
|
64
|
-
return self.__acDB.regenerateBuckets(typeName)
|
|
65
|
-
|
|
66
41
|
types_getRegisteredTypes = []
|
|
67
42
|
|
|
68
43
|
def export_getRegisteredTypes(self):
|
|
@@ -106,51 +81,4 @@ class DataStoreHandler(RequestHandler):
|
|
|
106
81
|
records.append((entry[0], startTime, endTime, entry[3]))
|
|
107
82
|
return self.__acDB.insertRecordBundleThroughQueue(records)
|
|
108
83
|
|
|
109
|
-
types_compactDB = []
|
|
110
|
-
|
|
111
|
-
def export_compactDB(self):
|
|
112
|
-
"""
|
|
113
|
-
Compact the db by grouping buckets
|
|
114
|
-
"""
|
|
115
|
-
# if we are running workers (not only one service) we can redirect the request to the master
|
|
116
|
-
# For more information please read the Administrative guide Accounting part!
|
|
117
|
-
# ADVICE: If you want to trigger the bucketing, please make sure the bucketing is not running!!!!
|
|
118
|
-
if self.runBucketing:
|
|
119
|
-
return self.__acDB.compactBuckets()
|
|
120
|
-
|
|
121
|
-
return Client(url="Accounting/DataStoreMaster").compactDB()
|
|
122
|
-
|
|
123
84
|
types_remove = [str, datetime.datetime, datetime.datetime, list]
|
|
124
|
-
|
|
125
|
-
def export_remove(self, typeName, startTime, endTime, valuesList):
|
|
126
|
-
"""
|
|
127
|
-
Remove a record for a type
|
|
128
|
-
"""
|
|
129
|
-
startTime = int(TimeUtilities.toEpoch(startTime))
|
|
130
|
-
endTime = int(TimeUtilities.toEpoch(endTime))
|
|
131
|
-
return self.__acDB.deleteRecord(typeName, startTime, endTime, valuesList)
|
|
132
|
-
|
|
133
|
-
types_removeRegisters = [list]
|
|
134
|
-
|
|
135
|
-
def export_removeRegisters(self, entriesList):
|
|
136
|
-
"""
|
|
137
|
-
Remove a record for a type
|
|
138
|
-
"""
|
|
139
|
-
expectedTypes = [str, datetime.datetime, datetime.datetime, list]
|
|
140
|
-
for entry in entriesList:
|
|
141
|
-
if len(entry) != 4:
|
|
142
|
-
return S_ERROR("Invalid records")
|
|
143
|
-
for i, en in enumerate(entry):
|
|
144
|
-
if not isinstance(en, expectedTypes[i]):
|
|
145
|
-
return S_ERROR(f"{i} field in the records should be {expectedTypes[i]}")
|
|
146
|
-
ok = 0
|
|
147
|
-
for entry in entriesList:
|
|
148
|
-
startTime = int(TimeUtilities.toEpoch(entry[1]))
|
|
149
|
-
endTime = int(TimeUtilities.toEpoch(entry[2]))
|
|
150
|
-
record = entry[3]
|
|
151
|
-
result = self.__acDB.deleteRecord(entry[0], startTime, endTime, record)
|
|
152
|
-
if not result["OK"]:
|
|
153
|
-
return S_OK(ok)
|
|
154
|
-
ok += 1
|
|
155
|
-
|
|
156
|
-
return S_OK(ok)
|
|
@@ -1,24 +1,53 @@
|
|
|
1
|
-
"""
|
|
2
|
-
"""
|
|
1
|
+
"""Helper for /Registry section"""
|
|
3
2
|
|
|
4
3
|
import errno
|
|
4
|
+
import inspect
|
|
5
|
+
import sys
|
|
5
6
|
|
|
6
7
|
from threading import Lock
|
|
8
|
+
from collections.abc import Iterable
|
|
7
9
|
|
|
8
10
|
from cachetools import TTLCache, cached
|
|
11
|
+
from cachetools.keys import hashkey
|
|
9
12
|
|
|
10
13
|
|
|
14
|
+
from typing import Optional
|
|
15
|
+
from collections.abc import Iterable
|
|
16
|
+
|
|
11
17
|
from DIRAC import S_OK, S_ERROR
|
|
12
18
|
from DIRAC.ConfigurationSystem.Client.Config import gConfig
|
|
13
19
|
from DIRAC.ConfigurationSystem.Client.Helpers.CSGlobals import getVO
|
|
14
20
|
|
|
15
21
|
ID_DN_PREFIX = "/O=DIRAC/CN="
|
|
16
22
|
|
|
23
|
+
# 300 is the default CS refresh time
|
|
24
|
+
|
|
25
|
+
CACHE_REFRESH_TIME = 300
|
|
17
26
|
# pylint: disable=missing-docstring
|
|
18
27
|
|
|
19
28
|
gBaseRegistrySection = "/Registry"
|
|
20
29
|
|
|
21
30
|
|
|
31
|
+
def reset_all_caches():
|
|
32
|
+
"""This method is called to clear all caches.
|
|
33
|
+
It is necessary to reinitialize them after the central CS
|
|
34
|
+
has been loaded
|
|
35
|
+
"""
|
|
36
|
+
for cache in [
|
|
37
|
+
obj
|
|
38
|
+
for name, obj in inspect.getmembers(sys.modules[__name__])
|
|
39
|
+
if (inspect.isfunction(obj) and hasattr(obj, "cache_clear"))
|
|
40
|
+
]:
|
|
41
|
+
cache.cache_clear()
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def get_username_for_dn_key(dn: str, userList: Optional[Iterable[str]] = None):
|
|
45
|
+
if userList:
|
|
46
|
+
return hashkey(dn, *sorted(userList))
|
|
47
|
+
return hashkey(dn)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
@cached(TTLCache(maxsize=1000, ttl=CACHE_REFRESH_TIME), lock=Lock(), key=get_username_for_dn_key)
|
|
22
51
|
def getUsernameForDN(dn, usersList=None):
|
|
23
52
|
"""Find DIRAC user for DN
|
|
24
53
|
|
|
@@ -39,6 +68,7 @@ def getUsernameForDN(dn, usersList=None):
|
|
|
39
68
|
return S_ERROR(f"No username found for dn {dn}")
|
|
40
69
|
|
|
41
70
|
|
|
71
|
+
@cached(TTLCache(maxsize=1000, ttl=CACHE_REFRESH_TIME), lock=Lock())
|
|
42
72
|
def getDNForUsername(username):
|
|
43
73
|
"""Get user DN for user
|
|
44
74
|
|
|
@@ -419,6 +449,7 @@ def getBannedIPs():
|
|
|
419
449
|
return gConfig.getValue(f"{gBaseRegistrySection}/BannedIPs", [])
|
|
420
450
|
|
|
421
451
|
|
|
452
|
+
@cached(TTLCache(maxsize=1000, ttl=CACHE_REFRESH_TIME), lock=Lock())
|
|
422
453
|
def getVOForGroup(group):
|
|
423
454
|
"""Search VO name for group
|
|
424
455
|
|
|
@@ -634,10 +665,7 @@ def getDNProperty(userDN, value, defaultValue=None):
|
|
|
634
665
|
return S_OK(defaultValue)
|
|
635
666
|
|
|
636
667
|
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
@cached(_cache_getProxyProvidersForDN, lock=Lock())
|
|
668
|
+
@cached(TTLCache(maxsize=1000, ttl=CACHE_REFRESH_TIME), lock=Lock())
|
|
641
669
|
def getProxyProvidersForDN(userDN):
|
|
642
670
|
"""Get proxy providers by user DN
|
|
643
671
|
|
|
@@ -568,6 +568,9 @@ class LocalConfiguration:
|
|
|
568
568
|
objLoader = ObjectLoader()
|
|
569
569
|
objLoader.reloadRootModules()
|
|
570
570
|
self.__initLogger(self.componentName, self.loggingSection, forceInit=True)
|
|
571
|
+
from DIRAC.ConfigurationSystem.Client.Helpers.Registry import reset_all_caches
|
|
572
|
+
|
|
573
|
+
reset_all_caches()
|
|
571
574
|
return res
|
|
572
575
|
|
|
573
576
|
def isCSEnabled(self):
|
|
@@ -87,8 +87,15 @@ def _getUserNameFromDN(dn, vo):
|
|
|
87
87
|
return nname
|
|
88
88
|
else:
|
|
89
89
|
robot = False
|
|
90
|
+
# only pop if the remains are sufficient (i.e. not just digits)
|
|
90
91
|
if names[0].lower().startswith("robot"):
|
|
91
|
-
|
|
92
|
+
nameok = False
|
|
93
|
+
if len(names) > 1:
|
|
94
|
+
for name in names[1:]:
|
|
95
|
+
if not name.isdigit():
|
|
96
|
+
nameok = True
|
|
97
|
+
if nameok:
|
|
98
|
+
names.pop(0)
|
|
92
99
|
robot = True
|
|
93
100
|
for name in list(names):
|
|
94
101
|
if name[0].isdigit() or "@" in name:
|
|
@@ -323,7 +323,7 @@ class BaseClient:
|
|
|
323
323
|
pass
|
|
324
324
|
|
|
325
325
|
# We randomize the list, and add at the end the failover URLs (System/FailoverURLs/Component)
|
|
326
|
-
urlsList = List.
|
|
326
|
+
urlsList = List.fromChar(urls, ",") + failoverUrls
|
|
327
327
|
self.__nbOfUrls = len(urlsList)
|
|
328
328
|
self.__nbOfRetry = (
|
|
329
329
|
2 if self.__nbOfUrls > 2 else 3
|
|
@@ -445,7 +445,6 @@ and this is thread {cThID}
|
|
|
445
445
|
return self.__initStatus
|
|
446
446
|
if self.__enableThreadCheck:
|
|
447
447
|
self.__checkThreadID()
|
|
448
|
-
|
|
449
448
|
gLogger.debug(f"Trying to connect to: {self.serviceURL}")
|
|
450
449
|
try:
|
|
451
450
|
# Calls the transport method of the apropriate protocol.
|
DIRAC/Core/Security/ProxyInfo.py
CHANGED
|
@@ -1,10 +1,13 @@
|
|
|
1
1
|
"""
|
|
2
|
-
|
|
2
|
+
Set of utilities to retrieve Information from proxy
|
|
3
3
|
"""
|
|
4
|
+
|
|
4
5
|
import base64
|
|
5
6
|
|
|
6
7
|
from DIRAC import S_ERROR, S_OK, gLogger
|
|
7
8
|
from DIRAC.ConfigurationSystem.Client.Helpers import Registry
|
|
9
|
+
from DIRAC.ConfigurationSystem.Client.Helpers.CSGlobals import getVO
|
|
10
|
+
|
|
8
11
|
from DIRAC.Core.Security import Locations
|
|
9
12
|
from DIRAC.Core.Security.DiracX import diracxTokenFromPEM
|
|
10
13
|
from DIRAC.Core.Security.VOMS import VOMS
|
|
@@ -207,10 +210,11 @@ def getVOfromProxyGroup():
|
|
|
207
210
|
"""
|
|
208
211
|
Return the VO associated to the group in the proxy
|
|
209
212
|
"""
|
|
210
|
-
|
|
213
|
+
|
|
211
214
|
ret = getProxyInfo(disableVOMS=True)
|
|
212
|
-
if not ret["OK"]:
|
|
213
|
-
|
|
214
|
-
|
|
215
|
+
if not ret["OK"] or "group" not in ret["Value"]:
|
|
216
|
+
voName = getVO()
|
|
217
|
+
else:
|
|
215
218
|
voName = Registry.getVOForGroup(ret["Value"]["group"])
|
|
219
|
+
|
|
216
220
|
return S_OK(voName)
|
DIRAC/Core/Utilities/Os.py
CHANGED
|
@@ -3,10 +3,11 @@
|
|
|
3
3
|
by default on Error they return None
|
|
4
4
|
"""
|
|
5
5
|
import os
|
|
6
|
+
import threading
|
|
6
7
|
|
|
7
8
|
import DIRAC
|
|
8
|
-
from DIRAC.Core.Utilities.Subprocess import shellCall, systemCall
|
|
9
9
|
from DIRAC.Core.Utilities import List
|
|
10
|
+
from DIRAC.Core.Utilities.Subprocess import shellCall, systemCall
|
|
10
11
|
|
|
11
12
|
DEBUG = 0
|
|
12
13
|
|
|
@@ -128,3 +129,33 @@ def sourceEnv(timeout, cmdTuple, inputEnv=None):
|
|
|
128
129
|
result["stderr"] = stderr
|
|
129
130
|
|
|
130
131
|
return result
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def safe_listdir(directory, timeout=60):
|
|
135
|
+
"""This is a "safe" list directory,
|
|
136
|
+
for lazily-loaded File Systems like CVMFS.
|
|
137
|
+
There's by default a 60 seconds timeout.
|
|
138
|
+
|
|
139
|
+
.. warning::
|
|
140
|
+
There is no distinction between an empty directory, and a non existent one.
|
|
141
|
+
It will return `[]` in both cases.
|
|
142
|
+
|
|
143
|
+
:param str directory: directory to list
|
|
144
|
+
:param int timeout: optional timeout, in seconds. Defaults to 60.
|
|
145
|
+
"""
|
|
146
|
+
|
|
147
|
+
def listdir(directory):
|
|
148
|
+
try:
|
|
149
|
+
return os.listdir(directory)
|
|
150
|
+
except FileNotFoundError:
|
|
151
|
+
print(f"{directory} not found")
|
|
152
|
+
return []
|
|
153
|
+
|
|
154
|
+
contents = []
|
|
155
|
+
t = threading.Thread(target=lambda: contents.extend(listdir(directory)))
|
|
156
|
+
t.daemon = True # don't delay program's exit
|
|
157
|
+
t.start()
|
|
158
|
+
t.join(timeout)
|
|
159
|
+
if t.is_alive():
|
|
160
|
+
return None # timeout
|
|
161
|
+
return contents
|
|
@@ -10,6 +10,7 @@ import DIRAC
|
|
|
10
10
|
from DIRAC import S_ERROR, gConfig, gLogger
|
|
11
11
|
from DIRAC.Core.Base.Script import Script
|
|
12
12
|
from DIRAC.Core.Security.Locations import getCAsLocation, getProxyLocation, getVOMSLocation
|
|
13
|
+
from DIRAC.Core.Utilities.Os import safe_listdir
|
|
13
14
|
from DIRAC.Core.Utilities.Subprocess import systemCall
|
|
14
15
|
|
|
15
16
|
|
|
@@ -73,7 +74,7 @@ def main():
|
|
|
73
74
|
cmd.extend(["--contain"]) # use minimal /dev and empty other directories (e.g. /tmp and $HOME)
|
|
74
75
|
cmd.extend(["--ipc"]) # run container in a new IPC namespace
|
|
75
76
|
cmd.extend(["--pid"]) # run container in a new PID namespace
|
|
76
|
-
cmd.extend(["--bind", f"{os.getcwd()}
|
|
77
|
+
cmd.extend(["--bind", f"{os.getcwd()}"]) # bind current directory for dirac_container.sh
|
|
77
78
|
if proxy_location:
|
|
78
79
|
cmd.extend(["--bind", f"{proxy_location}:/etc/proxy"]) # bind proxy file
|
|
79
80
|
cmd.extend(["--bind", f"{getCAsLocation()}:/etc/grid-security/certificates"]) # X509_CERT_DIR
|
|
@@ -83,12 +84,18 @@ def main():
|
|
|
83
84
|
cmd.extend(["--bind", f"{vomses_location}:/etc/grid-security/vomses"]) # X509_VOMSES
|
|
84
85
|
cmd.extend(["--bind", "{0}:{0}:ro".format(etc_dir)]) # etc dir for dirac.cfg
|
|
85
86
|
cmd.extend(["--bind", "{0}:{0}:ro".format(os.path.join(os.path.realpath(sys.base_prefix)))]) # code dir
|
|
86
|
-
|
|
87
|
+
# here bind optional paths
|
|
88
|
+
for bind_path in gConfig.getValue("/Resources/Computing/Singularity/BindPaths", []):
|
|
89
|
+
if safe_listdir(bind_path):
|
|
90
|
+
cmd.extend(["--bind", f"{bind_path}:{bind_path}"])
|
|
91
|
+
else:
|
|
92
|
+
gLogger.warning(f"Bind path {bind_path} does not exist, skipping")
|
|
93
|
+
cmd.extend(["--cwd", f"{os.getcwd()}"]) # set working directory
|
|
87
94
|
|
|
88
95
|
rootImage = user_image or gConfig.getValue("/Resources/Computing/Singularity/ContainerRoot") or CONTAINER_DEFROOT
|
|
89
96
|
|
|
90
97
|
if os.path.isdir(rootImage) or os.path.isfile(rootImage):
|
|
91
|
-
cmd.extend([rootImage, "/
|
|
98
|
+
cmd.extend([rootImage, f"{os.getcwd()}/dirac_container.sh"])
|
|
92
99
|
else:
|
|
93
100
|
# if we are here is because there's no image, or it is not accessible (e.g. not on CVMFS)
|
|
94
101
|
gLogger.error("Apptainer image to exec not found: ", rootImage)
|
|
@@ -100,7 +107,8 @@ def main():
|
|
|
100
107
|
gLogger.error(result["Message"])
|
|
101
108
|
DIRAC.exit(1)
|
|
102
109
|
if result["Value"][0] != 0:
|
|
103
|
-
gLogger.error(result["Value"][
|
|
110
|
+
gLogger.error("Apptainer command failed with exit code", result["Value"][0])
|
|
111
|
+
gLogger.error("Command output:", result["Value"])
|
|
104
112
|
DIRAC.exit(2)
|
|
105
113
|
gLogger.notice(result["Value"][1])
|
|
106
114
|
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
"""
|
|
2
|
-
|
|
2
|
+
This module contains helper methods for accessing operational attributes or parameters of DMS objects
|
|
3
3
|
|
|
4
4
|
"""
|
|
5
|
+
|
|
5
6
|
from collections import defaultdict
|
|
6
7
|
from DIRAC import gConfig, gLogger, S_OK, S_ERROR
|
|
7
8
|
from DIRAC.ConfigurationSystem.Client.Helpers.Path import cfgPath
|
|
@@ -17,6 +18,9 @@ sLog = gLogger.getSubLogger(__name__)
|
|
|
17
18
|
def resolveSEGroup(seGroupList, allSEs=None):
|
|
18
19
|
"""
|
|
19
20
|
Resolves recursively a (list of) SEs that can be groupSEs
|
|
21
|
+
For modules, JobWrapper or whatever runs at a given site,
|
|
22
|
+
prefer using :py:func:`~DIRAC.DataManagementSystem.Utilities.ResolveSE.getDestinationSEList`
|
|
23
|
+
|
|
20
24
|
|
|
21
25
|
:param seGroupList: list of SEs to resolve or comma-separated SEs
|
|
22
26
|
:type seGroupList: list or string
|
|
@@ -4,7 +4,9 @@ All administrative functionality is exposed through the DIRAC Admin API. Exampl
|
|
|
4
4
|
site banning and unbanning, WMS proxy uploading etc.
|
|
5
5
|
|
|
6
6
|
"""
|
|
7
|
+
|
|
7
8
|
import os
|
|
9
|
+
from datetime import datetime, timedelta
|
|
8
10
|
|
|
9
11
|
from DIRAC import S_ERROR, S_OK, gConfig, gLogger
|
|
10
12
|
from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI
|
|
@@ -150,7 +152,7 @@ class DiracAdmin(API):
|
|
|
150
152
|
return result
|
|
151
153
|
|
|
152
154
|
#############################################################################
|
|
153
|
-
def allowSite(self, site, comment, printOutput=False):
|
|
155
|
+
def allowSite(self, site, comment, printOutput=False, days=1):
|
|
154
156
|
"""Adds the site to the site mask. The site must be a valid DIRAC site name
|
|
155
157
|
|
|
156
158
|
Example usage:
|
|
@@ -174,7 +176,13 @@ class DiracAdmin(API):
|
|
|
174
176
|
gLogger.notice(f"Site {site} is already Active")
|
|
175
177
|
return S_OK(f"Site {site} is already Active")
|
|
176
178
|
|
|
177
|
-
|
|
179
|
+
tokenLifetime = int(days)
|
|
180
|
+
if tokenLifetime <= 0:
|
|
181
|
+
tokenExpiration = datetime.max
|
|
182
|
+
else:
|
|
183
|
+
tokenExpiration = datetime.utcnow().replace(microsecond=0) + timedelta(days=tokenLifetime)
|
|
184
|
+
|
|
185
|
+
if not (result := self.sitestatus.setSiteStatus(site, "Active", comment, expiry=tokenExpiration))["OK"]:
|
|
178
186
|
return result
|
|
179
187
|
|
|
180
188
|
if printOutput:
|
|
@@ -223,7 +231,7 @@ class DiracAdmin(API):
|
|
|
223
231
|
return S_OK()
|
|
224
232
|
|
|
225
233
|
#############################################################################
|
|
226
|
-
def banSite(self, site, comment, printOutput=False):
|
|
234
|
+
def banSite(self, site, comment, printOutput=False, days=1):
|
|
227
235
|
"""Removes the site from the site mask.
|
|
228
236
|
|
|
229
237
|
Example usage:
|
|
@@ -236,7 +244,6 @@ class DiracAdmin(API):
|
|
|
236
244
|
"""
|
|
237
245
|
if not (result := self._checkSiteIsValid(site))["OK"]:
|
|
238
246
|
return result
|
|
239
|
-
|
|
240
247
|
mask = self.getSiteMask(status="Banned")
|
|
241
248
|
if not mask["OK"]:
|
|
242
249
|
return mask
|
|
@@ -246,7 +253,12 @@ class DiracAdmin(API):
|
|
|
246
253
|
gLogger.notice(f"Site {site} is already Banned")
|
|
247
254
|
return S_OK(f"Site {site} is already Banned")
|
|
248
255
|
|
|
249
|
-
|
|
256
|
+
tokenLifetime = int(days)
|
|
257
|
+
if tokenLifetime <= 0:
|
|
258
|
+
tokenExpiration = datetime.max
|
|
259
|
+
else:
|
|
260
|
+
tokenExpiration = datetime.utcnow().replace(microsecond=0) + timedelta(days=tokenLifetime)
|
|
261
|
+
if not (result := self.sitestatus.setSiteStatus(site, "Banned", comment, expiry=tokenExpiration))["OK"]:
|
|
250
262
|
return result
|
|
251
263
|
|
|
252
264
|
if printOutput:
|
|
@@ -13,6 +13,9 @@ from DIRAC.Core.Base.Script import Script
|
|
|
13
13
|
@Script()
|
|
14
14
|
def main():
|
|
15
15
|
Script.registerSwitch("E:", "email=", "Boolean True/False (True by default)")
|
|
16
|
+
Script.registerSwitch(
|
|
17
|
+
"", "days=", "Number of days the token is valid for. Default is 1 day. 0 or less days denotes forever."
|
|
18
|
+
)
|
|
16
19
|
# Registering arguments will automatically add their description to the help menu
|
|
17
20
|
Script.registerArgument("Site: Name of the Site")
|
|
18
21
|
Script.registerArgument("Comment: Reason of the action")
|
|
@@ -32,9 +35,12 @@ def main():
|
|
|
32
35
|
Script.showHelp()
|
|
33
36
|
|
|
34
37
|
email = True
|
|
38
|
+
days = 1
|
|
35
39
|
for switch in Script.getUnprocessedSwitches():
|
|
36
40
|
if switch[0] == "email":
|
|
37
41
|
email = getBoolean(switch[1])
|
|
42
|
+
if switch[0] == "days":
|
|
43
|
+
days = int(switch[1])
|
|
38
44
|
|
|
39
45
|
diracAdmin = DiracAdmin()
|
|
40
46
|
exitCode = 0
|
|
@@ -42,7 +48,7 @@ def main():
|
|
|
42
48
|
|
|
43
49
|
# parseCommandLine show help when mandatory arguments are not specified or incorrect argument
|
|
44
50
|
site, comment = Script.getPositionalArgs(group=True)
|
|
45
|
-
result = diracAdmin.allowSite(site, comment, printOutput=True)
|
|
51
|
+
result = diracAdmin.allowSite(site, comment, printOutput=True, days=days)
|
|
46
52
|
if not result["OK"]:
|
|
47
53
|
errorList.append((site, result["Message"]))
|
|
48
54
|
exitCode = 2
|