DIRAC 9.0.0a42__py3-none-any.whl → 9.0.7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- DIRAC/AccountingSystem/Client/AccountingCLI.py +0 -140
- DIRAC/AccountingSystem/Client/DataStoreClient.py +0 -13
- DIRAC/AccountingSystem/Client/Types/BaseAccountingType.py +0 -7
- DIRAC/AccountingSystem/ConfigTemplate.cfg +0 -5
- DIRAC/AccountingSystem/Service/DataStoreHandler.py +0 -72
- DIRAC/ConfigurationSystem/Client/Helpers/CSGlobals.py +0 -9
- DIRAC/ConfigurationSystem/Client/Helpers/Registry.py +38 -26
- DIRAC/ConfigurationSystem/Client/Helpers/Resources.py +11 -43
- DIRAC/ConfigurationSystem/Client/Helpers/test/Test_Helpers.py +0 -16
- DIRAC/ConfigurationSystem/Client/LocalConfiguration.py +14 -8
- DIRAC/ConfigurationSystem/Client/PathFinder.py +47 -8
- DIRAC/ConfigurationSystem/Client/SyncPlugins/CERNLDAPSyncPlugin.py +4 -1
- DIRAC/ConfigurationSystem/Client/VOMS2CSSynchronizer.py +32 -19
- DIRAC/ConfigurationSystem/Client/test/Test_PathFinder.py +41 -1
- DIRAC/ConfigurationSystem/private/RefresherBase.py +4 -2
- DIRAC/Core/Base/API.py +4 -7
- DIRAC/Core/Base/SQLAlchemyDB.py +1 -0
- DIRAC/Core/DISET/ServiceReactor.py +11 -3
- DIRAC/Core/DISET/private/BaseClient.py +1 -2
- DIRAC/Core/DISET/private/Transports/M2SSLTransport.py +9 -7
- DIRAC/Core/DISET/private/Transports/SSL/M2Utils.py +3 -1
- DIRAC/Core/LCG/GOCDBClient.py +5 -7
- DIRAC/Core/Security/DiracX.py +31 -17
- DIRAC/Core/Security/IAMService.py +5 -10
- DIRAC/Core/Security/Locations.py +27 -18
- DIRAC/Core/Security/ProxyInfo.py +9 -5
- DIRAC/Core/Security/VOMSService.py +2 -4
- DIRAC/Core/Security/m2crypto/X509Certificate.py +4 -6
- DIRAC/Core/Security/m2crypto/asn1_utils.py +17 -5
- DIRAC/Core/Security/test/test_diracx_token_from_pem.py +161 -0
- DIRAC/Core/Tornado/Client/ClientSelector.py +4 -1
- DIRAC/Core/Tornado/Server/TornadoService.py +1 -1
- DIRAC/Core/Utilities/CGroups2.py +328 -0
- DIRAC/Core/Utilities/ClassAd/ClassAdLight.py +4 -290
- DIRAC/Core/Utilities/DErrno.py +5 -309
- DIRAC/Core/Utilities/Extensions.py +10 -1
- DIRAC/Core/Utilities/File.py +1 -1
- DIRAC/Core/Utilities/Graphs/GraphData.py +1 -1
- DIRAC/Core/Utilities/Graphs/GraphUtilities.py +6 -1
- DIRAC/Core/Utilities/JDL.py +1 -195
- DIRAC/Core/Utilities/List.py +1 -124
- DIRAC/Core/Utilities/MySQL.py +103 -99
- DIRAC/Core/Utilities/Os.py +32 -1
- DIRAC/Core/Utilities/Platform.py +2 -107
- DIRAC/Core/Utilities/Proxy.py +0 -4
- DIRAC/Core/Utilities/ReturnValues.py +7 -252
- DIRAC/Core/Utilities/StateMachine.py +12 -178
- DIRAC/Core/Utilities/Subprocess.py +35 -14
- DIRAC/Core/Utilities/TimeUtilities.py +10 -253
- DIRAC/Core/Utilities/test/Test_JDL.py +0 -3
- DIRAC/Core/Utilities/test/Test_Profiler.py +20 -20
- DIRAC/Core/scripts/dirac_agent.py +1 -1
- DIRAC/Core/scripts/dirac_apptainer_exec.py +72 -46
- DIRAC/Core/scripts/dirac_configure.py +1 -3
- DIRAC/Core/scripts/dirac_install_db.py +24 -6
- DIRAC/Core/scripts/dirac_platform.py +1 -92
- DIRAC/DataManagementSystem/Agent/FTS3Agent.py +8 -7
- DIRAC/DataManagementSystem/Agent/RequestOperations/RemoveFile.py +7 -6
- DIRAC/DataManagementSystem/Client/FTS3Job.py +71 -34
- DIRAC/DataManagementSystem/DB/FTS3DB.py +7 -3
- DIRAC/DataManagementSystem/DB/FileCatalogComponents/DatasetManager/DatasetManager.py +1 -1
- DIRAC/DataManagementSystem/DB/FileCatalogDB.sql +9 -9
- DIRAC/DataManagementSystem/DB/FileCatalogWithFkAndPsDB.sql +9 -9
- DIRAC/DataManagementSystem/Utilities/DMSHelpers.py +6 -2
- DIRAC/DataManagementSystem/scripts/dirac_admin_allow_se.py +13 -8
- DIRAC/DataManagementSystem/scripts/dirac_admin_ban_se.py +13 -8
- DIRAC/DataManagementSystem/scripts/dirac_dms_create_moving_request.py +2 -0
- DIRAC/DataManagementSystem/scripts/dirac_dms_protocol_matrix.py +0 -1
- DIRAC/FrameworkSystem/Client/BundleDeliveryClient.py +2 -7
- DIRAC/FrameworkSystem/Client/ComponentInstaller.py +9 -4
- DIRAC/FrameworkSystem/Client/ProxyManagerClient.py +5 -2
- DIRAC/FrameworkSystem/Client/SystemAdministratorClientCLI.py +11 -6
- DIRAC/FrameworkSystem/ConfigTemplate.cfg +2 -0
- DIRAC/FrameworkSystem/DB/AuthDB.py +3 -3
- DIRAC/FrameworkSystem/DB/InstalledComponentsDB.py +4 -4
- DIRAC/FrameworkSystem/DB/ProxyDB.py +11 -3
- DIRAC/FrameworkSystem/DB/TokenDB.py +1 -1
- DIRAC/FrameworkSystem/Service/ProxyManagerHandler.py +8 -6
- DIRAC/FrameworkSystem/Utilities/MonitoringUtilities.py +2 -19
- DIRAC/FrameworkSystem/Utilities/TokenManagementUtilities.py +3 -2
- DIRAC/FrameworkSystem/Utilities/diracx.py +36 -14
- DIRAC/FrameworkSystem/private/authorization/AuthServer.py +2 -2
- DIRAC/FrameworkSystem/scripts/dirac_admin_update_pilot.py +18 -11
- DIRAC/FrameworkSystem/scripts/dirac_login.py +2 -2
- DIRAC/FrameworkSystem/scripts/dirac_proxy_init.py +7 -8
- DIRAC/Interfaces/API/Dirac.py +27 -15
- DIRAC/Interfaces/API/DiracAdmin.py +45 -17
- DIRAC/Interfaces/API/Job.py +9 -13
- DIRAC/Interfaces/scripts/dirac_admin_allow_site.py +12 -18
- DIRAC/Interfaces/scripts/dirac_admin_ban_site.py +12 -10
- DIRAC/Interfaces/scripts/dirac_admin_get_site_mask.py +4 -13
- DIRAC/Interfaces/scripts/dirac_admin_reset_job.py +3 -6
- DIRAC/Interfaces/scripts/dirac_wms_job_parameters.py +0 -1
- DIRAC/MonitoringSystem/Client/Types/WMSHistory.py +4 -0
- DIRAC/MonitoringSystem/Client/WebAppClient.py +26 -0
- DIRAC/MonitoringSystem/ConfigTemplate.cfg +9 -0
- DIRAC/MonitoringSystem/DB/MonitoringDB.py +6 -25
- DIRAC/MonitoringSystem/Service/MonitoringHandler.py +0 -33
- DIRAC/MonitoringSystem/Service/WebAppHandler.py +599 -0
- DIRAC/MonitoringSystem/private/MainReporter.py +0 -3
- DIRAC/ProductionSystem/DB/ProductionDB.sql +4 -4
- DIRAC/ProductionSystem/scripts/dirac_prod_get.py +2 -2
- DIRAC/ProductionSystem/scripts/dirac_prod_get_all.py +2 -2
- DIRAC/ProductionSystem/scripts/dirac_prod_get_trans.py +2 -3
- DIRAC/RequestManagementSystem/Agent/RequestExecutingAgent.py +8 -6
- DIRAC/RequestManagementSystem/Agent/RequestOperations/ForwardDISET.py +2 -14
- DIRAC/RequestManagementSystem/Client/ReqClient.py +66 -13
- DIRAC/RequestManagementSystem/ConfigTemplate.cfg +6 -6
- DIRAC/RequestManagementSystem/DB/RequestDB.py +10 -5
- DIRAC/RequestManagementSystem/DB/test/RMSTestScenari.py +2 -0
- DIRAC/RequestManagementSystem/private/RequestValidator.py +40 -46
- DIRAC/ResourceStatusSystem/Client/SiteStatus.py +4 -2
- DIRAC/ResourceStatusSystem/Command/FreeDiskSpaceCommand.py +3 -1
- DIRAC/ResourceStatusSystem/DB/ResourceManagementDB.py +8 -8
- DIRAC/ResourceStatusSystem/DB/ResourceStatusDB.py +2 -2
- DIRAC/ResourceStatusSystem/Utilities/CSHelpers.py +2 -31
- DIRAC/ResourceStatusSystem/scripts/dirac_rss_set_status.py +30 -12
- DIRAC/Resources/Catalog/RucioFileCatalogClient.py +195 -1
- DIRAC/Resources/Catalog/test/Test_RucioFileCatalogClient.py +181 -0
- DIRAC/Resources/Computing/AREXComputingElement.py +25 -8
- DIRAC/Resources/Computing/BatchSystems/Condor.py +126 -108
- DIRAC/Resources/Computing/BatchSystems/SLURM.py +5 -1
- DIRAC/Resources/Computing/BatchSystems/test/Test_SLURM.py +46 -0
- DIRAC/Resources/Computing/ComputingElement.py +1 -1
- DIRAC/Resources/Computing/HTCondorCEComputingElement.py +44 -44
- DIRAC/Resources/Computing/InProcessComputingElement.py +4 -2
- DIRAC/Resources/Computing/LocalComputingElement.py +1 -18
- DIRAC/Resources/Computing/SSHBatchComputingElement.py +1 -17
- DIRAC/Resources/Computing/SSHComputingElement.py +1 -18
- DIRAC/Resources/Computing/SingularityComputingElement.py +19 -5
- DIRAC/Resources/Computing/test/Test_HTCondorCEComputingElement.py +67 -49
- DIRAC/Resources/Computing/test/Test_PoolComputingElement.py +2 -1
- DIRAC/Resources/IdProvider/CheckInIdProvider.py +13 -0
- DIRAC/Resources/IdProvider/IdProviderFactory.py +11 -3
- DIRAC/Resources/MessageQueue/StompMQConnector.py +1 -1
- DIRAC/Resources/Storage/GFAL2_StorageBase.py +24 -15
- DIRAC/Resources/Storage/OccupancyPlugins/WLCGAccountingHTTPJson.py +1 -3
- DIRAC/Resources/Storage/StorageBase.py +4 -2
- DIRAC/Resources/Storage/StorageElement.py +6 -7
- DIRAC/StorageManagementSystem/DB/StorageManagementDB.sql +2 -2
- DIRAC/TransformationSystem/Agent/TaskManagerAgentBase.py +10 -16
- DIRAC/TransformationSystem/Agent/TransformationAgent.py +22 -1
- DIRAC/TransformationSystem/Agent/TransformationCleaningAgent.py +16 -16
- DIRAC/TransformationSystem/Client/TaskManager.py +2 -4
- DIRAC/TransformationSystem/Client/Transformation.py +6 -7
- DIRAC/TransformationSystem/Client/TransformationClient.py +21 -11
- DIRAC/TransformationSystem/Client/Utilities.py +9 -0
- DIRAC/TransformationSystem/DB/TransformationDB.py +11 -14
- DIRAC/TransformationSystem/DB/TransformationDB.sql +9 -9
- DIRAC/TransformationSystem/Service/TransformationManagerHandler.py +0 -333
- DIRAC/TransformationSystem/Utilities/ReplicationCLIParameters.py +3 -3
- DIRAC/TransformationSystem/Utilities/TransformationInfo.py +7 -5
- DIRAC/TransformationSystem/scripts/dirac_production_runjoblocal.py +2 -4
- DIRAC/TransformationSystem/test/Test_TransformationInfo.py +22 -15
- DIRAC/TransformationSystem/test/Test_replicationTransformation.py +5 -6
- DIRAC/Workflow/Modules/test/Test_Modules.py +5 -0
- DIRAC/WorkloadManagementSystem/Agent/JobAgent.py +38 -26
- DIRAC/WorkloadManagementSystem/Agent/JobCleaningAgent.py +12 -8
- DIRAC/WorkloadManagementSystem/Agent/PilotSyncAgent.py +4 -3
- DIRAC/WorkloadManagementSystem/Agent/PushJobAgent.py +13 -13
- DIRAC/WorkloadManagementSystem/Agent/SiteDirector.py +18 -14
- DIRAC/WorkloadManagementSystem/Agent/StalledJobAgent.py +18 -51
- DIRAC/WorkloadManagementSystem/Agent/StatesAccountingAgent.py +41 -1
- DIRAC/WorkloadManagementSystem/Agent/test/Test_Agent_JobAgent.py +45 -4
- DIRAC/WorkloadManagementSystem/Agent/test/Test_Agent_JobCleaningAgent.py +7 -9
- DIRAC/WorkloadManagementSystem/Agent/test/Test_Agent_PushJobAgent.py +1 -0
- DIRAC/WorkloadManagementSystem/Agent/test/Test_Agent_SiteDirector.py +9 -2
- DIRAC/WorkloadManagementSystem/Agent/test/Test_Agent_StalledJobAgent.py +4 -5
- DIRAC/WorkloadManagementSystem/Client/DownloadInputData.py +9 -9
- DIRAC/WorkloadManagementSystem/Client/InputDataResolution.py +6 -6
- DIRAC/WorkloadManagementSystem/Client/JobMonitoringClient.py +10 -11
- DIRAC/WorkloadManagementSystem/Client/JobReport.py +1 -1
- DIRAC/WorkloadManagementSystem/Client/JobState/CachedJobState.py +3 -0
- DIRAC/WorkloadManagementSystem/Client/JobState/JobManifest.py +32 -261
- DIRAC/WorkloadManagementSystem/Client/JobState/JobState.py +6 -0
- DIRAC/WorkloadManagementSystem/Client/JobStateUpdateClient.py +3 -0
- DIRAC/WorkloadManagementSystem/Client/JobStatus.py +8 -152
- DIRAC/WorkloadManagementSystem/Client/PoolXMLSlice.py +12 -19
- DIRAC/WorkloadManagementSystem/Client/SandboxStoreClient.py +25 -38
- DIRAC/WorkloadManagementSystem/Client/WMSClient.py +2 -3
- DIRAC/WorkloadManagementSystem/Client/test/Test_Client_DownloadInputData.py +29 -0
- DIRAC/WorkloadManagementSystem/ConfigTemplate.cfg +4 -8
- DIRAC/WorkloadManagementSystem/DB/JobDB.py +89 -132
- DIRAC/WorkloadManagementSystem/DB/JobDB.sql +8 -8
- DIRAC/WorkloadManagementSystem/DB/JobDBUtils.py +18 -147
- DIRAC/WorkloadManagementSystem/DB/JobLoggingDB.py +19 -6
- DIRAC/WorkloadManagementSystem/DB/JobParametersDB.py +9 -9
- DIRAC/WorkloadManagementSystem/DB/PilotAgentsDB.py +16 -5
- DIRAC/WorkloadManagementSystem/DB/PilotAgentsDB.sql +3 -3
- DIRAC/WorkloadManagementSystem/DB/SandboxMetadataDB.py +44 -82
- DIRAC/WorkloadManagementSystem/DB/StatusUtils.py +125 -0
- DIRAC/WorkloadManagementSystem/DB/tests/Test_JobDB.py +1 -1
- DIRAC/WorkloadManagementSystem/DB/tests/Test_StatusUtils.py +28 -0
- DIRAC/WorkloadManagementSystem/Executor/JobSanity.py +5 -4
- DIRAC/WorkloadManagementSystem/Executor/JobScheduling.py +4 -0
- DIRAC/WorkloadManagementSystem/FutureClient/JobStateUpdateClient.py +75 -33
- DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py +22 -11
- DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapperTemplate.py +9 -10
- DIRAC/WorkloadManagementSystem/JobWrapper/test/Test_JobWrapper.py +60 -10
- DIRAC/WorkloadManagementSystem/JobWrapper/test/Test_JobWrapperTemplate.py +4 -0
- DIRAC/WorkloadManagementSystem/Service/JobManagerHandler.py +33 -154
- DIRAC/WorkloadManagementSystem/Service/JobMonitoringHandler.py +5 -323
- DIRAC/WorkloadManagementSystem/Service/JobStateUpdateHandler.py +0 -16
- DIRAC/WorkloadManagementSystem/Service/PilotManagerHandler.py +6 -103
- DIRAC/WorkloadManagementSystem/Service/SandboxStoreHandler.py +7 -53
- DIRAC/WorkloadManagementSystem/Service/WMSAdministratorHandler.py +16 -79
- DIRAC/WorkloadManagementSystem/Service/WMSUtilities.py +4 -18
- DIRAC/WorkloadManagementSystem/Utilities/JobModel.py +28 -209
- DIRAC/WorkloadManagementSystem/Utilities/JobParameters.py +65 -3
- DIRAC/WorkloadManagementSystem/Utilities/JobStatusUtility.py +2 -64
- DIRAC/WorkloadManagementSystem/Utilities/ParametricJob.py +7 -171
- DIRAC/WorkloadManagementSystem/Utilities/PilotCStoJSONSynchronizer.py +73 -7
- DIRAC/WorkloadManagementSystem/Utilities/PilotWrapper.py +41 -11
- DIRAC/WorkloadManagementSystem/Utilities/RemoteRunner.py +16 -0
- DIRAC/WorkloadManagementSystem/Utilities/Utils.py +36 -1
- DIRAC/WorkloadManagementSystem/Utilities/jobAdministration.py +15 -0
- DIRAC/WorkloadManagementSystem/Utilities/test/Test_JobModel.py +1 -15
- DIRAC/WorkloadManagementSystem/Utilities/test/Test_ParametricJob.py +45 -128
- DIRAC/WorkloadManagementSystem/Utilities/test/Test_PilotWrapper.py +16 -0
- DIRAC/WorkloadManagementSystem/scripts/dirac_jobexec.py +7 -2
- DIRAC/WorkloadManagementSystem/scripts/dirac_wms_pilot_job_info.py +1 -1
- DIRAC/__init__.py +62 -60
- DIRAC/tests/Utilities/testJobDefinitions.py +22 -28
- {DIRAC-9.0.0a42.dist-info → dirac-9.0.7.dist-info}/METADATA +8 -5
- {DIRAC-9.0.0a42.dist-info → dirac-9.0.7.dist-info}/RECORD +229 -228
- {DIRAC-9.0.0a42.dist-info → dirac-9.0.7.dist-info}/WHEEL +1 -1
- {DIRAC-9.0.0a42.dist-info → dirac-9.0.7.dist-info}/entry_points.txt +0 -3
- DIRAC/Core/Utilities/test/Test_List.py +0 -150
- DIRAC/Core/Utilities/test/Test_Time.py +0 -88
- DIRAC/Resources/Computing/PilotBundle.py +0 -70
- DIRAC/TransformationSystem/scripts/dirac_transformation_archive.py +0 -30
- DIRAC/TransformationSystem/scripts/dirac_transformation_clean.py +0 -30
- DIRAC/TransformationSystem/scripts/dirac_transformation_remove_output.py +0 -30
- DIRAC/WorkloadManagementSystem/Utilities/test/Test_JobManager.py +0 -58
- {DIRAC-9.0.0a42.dist-info → dirac-9.0.7.dist-info/licenses}/LICENSE +0 -0
- {DIRAC-9.0.0a42.dist-info → dirac-9.0.7.dist-info}/top_level.txt +0 -0
|
@@ -3,22 +3,12 @@
|
|
|
3
3
|
import datetime
|
|
4
4
|
|
|
5
5
|
from DIRAC import S_ERROR, S_OK
|
|
6
|
-
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
|
|
7
6
|
from DIRAC.Core.DISET.RequestHandler import RequestHandler
|
|
8
7
|
from DIRAC.Core.Security.Properties import SecurityProperty
|
|
9
8
|
from DIRAC.Core.Utilities.Decorators import deprecated
|
|
10
9
|
from DIRAC.Core.Utilities.DEncode import ignoreEncodeWarning
|
|
11
10
|
from DIRAC.Core.Utilities.JEncode import encode as jencode
|
|
12
11
|
from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader
|
|
13
|
-
from DIRAC.RequestManagementSystem.Client.Operation import Operation
|
|
14
|
-
from DIRAC.RequestManagementSystem.Client.Request import Request
|
|
15
|
-
from DIRAC.TransformationSystem.Client import TransformationFilesStatus
|
|
16
|
-
from DIRAC.WorkloadManagementSystem.Client import JobStatus
|
|
17
|
-
|
|
18
|
-
TASKS_STATE_NAMES = ["TotalCreated", "Created"] + sorted(
|
|
19
|
-
set(JobStatus.JOB_STATES) | set(Request.ALL_STATES) | set(Operation.ALL_STATES)
|
|
20
|
-
)
|
|
21
|
-
FILES_STATE_NAMES = ["PercentProcessed", "Total"] + TransformationFilesStatus.TRANSFORMATION_FILES_STATES
|
|
22
12
|
|
|
23
13
|
|
|
24
14
|
class TransformationManagerHandlerMixin:
|
|
@@ -530,39 +520,12 @@ class TransformationManagerHandlerMixin:
|
|
|
530
520
|
"""Set metadata to a file or to a directory (path)"""
|
|
531
521
|
return cls.transformationDB.setMetadata(path, querydict)
|
|
532
522
|
|
|
533
|
-
####################################################################
|
|
534
|
-
#
|
|
535
|
-
# These are the methods used for web monitoring
|
|
536
|
-
#
|
|
537
|
-
|
|
538
|
-
# TODO Get rid of this (talk to Matvey)
|
|
539
|
-
types_getDistinctAttributeValues = [str, dict]
|
|
540
|
-
|
|
541
|
-
@classmethod
|
|
542
|
-
def export_getDistinctAttributeValues(cls, attribute, selectDict):
|
|
543
|
-
res = cls.transformationDB.getTableDistinctAttributeValues("Transformations", [attribute], selectDict)
|
|
544
|
-
if not res["OK"]:
|
|
545
|
-
return res
|
|
546
|
-
return S_OK(res["Value"][attribute])
|
|
547
|
-
|
|
548
523
|
types_getTableDistinctAttributeValues = [str, list, dict]
|
|
549
524
|
|
|
550
525
|
@classmethod
|
|
551
526
|
def export_getTableDistinctAttributeValues(cls, table, attributes, selectDict):
|
|
552
527
|
return cls.transformationDB.getTableDistinctAttributeValues(table, attributes, selectDict)
|
|
553
528
|
|
|
554
|
-
types_getTransformationStatusCounters = []
|
|
555
|
-
|
|
556
|
-
@classmethod
|
|
557
|
-
def export_getTransformationStatusCounters(cls):
|
|
558
|
-
res = cls.transformationDB.getCounters("Transformations", ["Status"], {})
|
|
559
|
-
if not res["OK"]:
|
|
560
|
-
return res
|
|
561
|
-
statDict = {}
|
|
562
|
-
for attrDict, count in res["Value"]:
|
|
563
|
-
statDict[attrDict["Status"]] = count
|
|
564
|
-
return S_OK(statDict)
|
|
565
|
-
|
|
566
529
|
types_getTransformationSummary = []
|
|
567
530
|
|
|
568
531
|
def export_getTransformationSummary(self):
|
|
@@ -587,302 +550,6 @@ class TransformationManagerHandlerMixin:
|
|
|
587
550
|
resultDict[transID] = transDict
|
|
588
551
|
return S_OK(resultDict)
|
|
589
552
|
|
|
590
|
-
types_getTabbedSummaryWeb = [str, dict, dict, list, int, int]
|
|
591
|
-
|
|
592
|
-
def export_getTabbedSummaryWeb(self, table, requestedTables, selectDict, sortList, startItem, maxItems):
|
|
593
|
-
tableDestinations = {
|
|
594
|
-
"Transformations": {
|
|
595
|
-
"TransformationFiles": ["TransformationID"],
|
|
596
|
-
"TransformationTasks": ["TransformationID"],
|
|
597
|
-
},
|
|
598
|
-
"TransformationFiles": {
|
|
599
|
-
"Transformations": ["TransformationID"],
|
|
600
|
-
"TransformationTasks": ["TransformationID", "TaskID"],
|
|
601
|
-
},
|
|
602
|
-
"TransformationTasks": {
|
|
603
|
-
"Transformations": ["TransformationID"],
|
|
604
|
-
"TransformationFiles": ["TransformationID", "TaskID"],
|
|
605
|
-
},
|
|
606
|
-
}
|
|
607
|
-
|
|
608
|
-
tableSelections = {
|
|
609
|
-
"Transformations": ["TransformationID", "AgentType", "Type", "TransformationGroup", "Plugin"],
|
|
610
|
-
"TransformationFiles": ["TransformationID", "TaskID", "Status", "UsedSE", "TargetSE"],
|
|
611
|
-
"TransformationTasks": ["TransformationID", "TaskID", "ExternalStatus", "TargetSE"],
|
|
612
|
-
}
|
|
613
|
-
|
|
614
|
-
tableTimeStamps = {
|
|
615
|
-
"Transformations": "CreationDate",
|
|
616
|
-
"TransformationFiles": "LastUpdate",
|
|
617
|
-
"TransformationTasks": "CreationTime",
|
|
618
|
-
}
|
|
619
|
-
|
|
620
|
-
tableStatusColumn = {
|
|
621
|
-
"Transformations": "Status",
|
|
622
|
-
"TransformationFiles": "Status",
|
|
623
|
-
"TransformationTasks": "ExternalStatus",
|
|
624
|
-
}
|
|
625
|
-
|
|
626
|
-
resDict = {}
|
|
627
|
-
res = self.__getTableSummaryWeb(
|
|
628
|
-
table,
|
|
629
|
-
selectDict,
|
|
630
|
-
sortList,
|
|
631
|
-
startItem,
|
|
632
|
-
maxItems,
|
|
633
|
-
selectColumns=tableSelections[table],
|
|
634
|
-
timeStamp=tableTimeStamps[table],
|
|
635
|
-
statusColumn=tableStatusColumn[table],
|
|
636
|
-
)
|
|
637
|
-
if not res["OK"]:
|
|
638
|
-
self.log.error("Failed to get Summary for table", f"{table} {res['Message']}")
|
|
639
|
-
return res
|
|
640
|
-
resDict[table] = res["Value"]
|
|
641
|
-
selections = res["Value"]["Selections"]
|
|
642
|
-
tableSelection = {}
|
|
643
|
-
for destination in tableDestinations[table].keys():
|
|
644
|
-
tableSelection[destination] = {}
|
|
645
|
-
for parameter in tableDestinations[table][destination]:
|
|
646
|
-
tableSelection[destination][parameter] = selections.get(parameter, [])
|
|
647
|
-
|
|
648
|
-
for table, paramDict in requestedTables.items():
|
|
649
|
-
sortList = paramDict.get("SortList", [])
|
|
650
|
-
startItem = paramDict.get("StartItem", 0)
|
|
651
|
-
maxItems = paramDict.get("MaxItems", 50)
|
|
652
|
-
res = self.__getTableSummaryWeb(
|
|
653
|
-
table,
|
|
654
|
-
tableSelection[table],
|
|
655
|
-
sortList,
|
|
656
|
-
startItem,
|
|
657
|
-
maxItems,
|
|
658
|
-
selectColumns=tableSelections[table],
|
|
659
|
-
timeStamp=tableTimeStamps[table],
|
|
660
|
-
statusColumn=tableStatusColumn[table],
|
|
661
|
-
)
|
|
662
|
-
if not res["OK"]:
|
|
663
|
-
self.log.error("Failed to get Summary for table", f"{table} {res['Message']}")
|
|
664
|
-
return res
|
|
665
|
-
resDict[table] = res["Value"]
|
|
666
|
-
return S_OK(resDict)
|
|
667
|
-
|
|
668
|
-
types_getTransformationsSummaryWeb = [dict, list, int, int]
|
|
669
|
-
|
|
670
|
-
def export_getTransformationsSummaryWeb(self, selectDict, sortList, startItem, maxItems):
|
|
671
|
-
return self.__getTableSummaryWeb(
|
|
672
|
-
"Transformations",
|
|
673
|
-
selectDict,
|
|
674
|
-
sortList,
|
|
675
|
-
startItem,
|
|
676
|
-
maxItems,
|
|
677
|
-
selectColumns=["TransformationID", "AgentType", "Type", "Group", "Plugin"],
|
|
678
|
-
timeStamp="CreationDate",
|
|
679
|
-
statusColumn="Status",
|
|
680
|
-
)
|
|
681
|
-
|
|
682
|
-
types_getTransformationTasksSummaryWeb = [dict, list, int, int]
|
|
683
|
-
|
|
684
|
-
def export_getTransformationTasksSummaryWeb(self, selectDict, sortList, startItem, maxItems):
|
|
685
|
-
return self.__getTableSummaryWeb(
|
|
686
|
-
"TransformationTasks",
|
|
687
|
-
selectDict,
|
|
688
|
-
sortList,
|
|
689
|
-
startItem,
|
|
690
|
-
maxItems,
|
|
691
|
-
selectColumns=["TransformationID", "ExternalStatus", "TargetSE"],
|
|
692
|
-
timeStamp="CreationTime",
|
|
693
|
-
statusColumn="ExternalStatus",
|
|
694
|
-
)
|
|
695
|
-
|
|
696
|
-
types_getTransformationFilesSummaryWeb = [dict, list, int, int]
|
|
697
|
-
|
|
698
|
-
def export_getTransformationFilesSummaryWeb(self, selectDict, sortList, startItem, maxItems):
|
|
699
|
-
return self.__getTableSummaryWeb(
|
|
700
|
-
"TransformationFiles",
|
|
701
|
-
selectDict,
|
|
702
|
-
sortList,
|
|
703
|
-
startItem,
|
|
704
|
-
maxItems,
|
|
705
|
-
selectColumns=["TransformationID", "Status", "UsedSE", "TargetSE"],
|
|
706
|
-
timeStamp="LastUpdate",
|
|
707
|
-
statusColumn="Status",
|
|
708
|
-
)
|
|
709
|
-
|
|
710
|
-
def __getTableSummaryWeb(
|
|
711
|
-
self, table, selectDict, sortList, startItem, maxItems, selectColumns=[], timeStamp=None, statusColumn="Status"
|
|
712
|
-
):
|
|
713
|
-
fromDate = selectDict.get("FromDate", None)
|
|
714
|
-
if fromDate:
|
|
715
|
-
del selectDict["FromDate"]
|
|
716
|
-
# if not fromDate:
|
|
717
|
-
# fromDate = last_update
|
|
718
|
-
toDate = selectDict.get("ToDate", None)
|
|
719
|
-
if toDate:
|
|
720
|
-
del selectDict["ToDate"]
|
|
721
|
-
# Sorting instructions. Only one for the moment.
|
|
722
|
-
if sortList:
|
|
723
|
-
orderAttribute = sortList[0][0] + ":" + sortList[0][1]
|
|
724
|
-
else:
|
|
725
|
-
orderAttribute = None
|
|
726
|
-
# Get the columns that match the selection
|
|
727
|
-
fcn = None
|
|
728
|
-
fcnName = f"get{table}"
|
|
729
|
-
if hasattr(self.transformationDB, fcnName) and callable(getattr(self.transformationDB, fcnName)):
|
|
730
|
-
fcn = getattr(self.transformationDB, fcnName)
|
|
731
|
-
if not fcn:
|
|
732
|
-
return S_ERROR(f"Unable to invoke gTransformationDB.{fcnName}, it isn't a member function")
|
|
733
|
-
res = fcn(condDict=selectDict, older=toDate, newer=fromDate, timeStamp=timeStamp, orderAttribute=orderAttribute)
|
|
734
|
-
if not res["OK"]:
|
|
735
|
-
return res
|
|
736
|
-
|
|
737
|
-
# The full list of columns in contained here
|
|
738
|
-
allRows = res["Records"]
|
|
739
|
-
# Prepare the standard structure now within the resultDict dictionary
|
|
740
|
-
resultDict = {}
|
|
741
|
-
# Create the total records entry
|
|
742
|
-
resultDict["TotalRecords"] = len(allRows)
|
|
743
|
-
# Create the ParameterNames entry
|
|
744
|
-
resultDict["ParameterNames"] = res["ParameterNames"]
|
|
745
|
-
# Find which element in the tuple contains the requested status
|
|
746
|
-
if statusColumn not in resultDict["ParameterNames"]:
|
|
747
|
-
return S_ERROR("Provided status column not present")
|
|
748
|
-
statusColumnIndex = resultDict["ParameterNames"].index(statusColumn)
|
|
749
|
-
|
|
750
|
-
# Get the rows which are within the selected window
|
|
751
|
-
if resultDict["TotalRecords"] == 0:
|
|
752
|
-
return S_OK(resultDict)
|
|
753
|
-
ini = startItem
|
|
754
|
-
last = ini + maxItems
|
|
755
|
-
if ini >= resultDict["TotalRecords"]:
|
|
756
|
-
return S_ERROR("Item number out of range")
|
|
757
|
-
if last > resultDict["TotalRecords"]:
|
|
758
|
-
last = resultDict["TotalRecords"]
|
|
759
|
-
selectedRows = allRows[ini:last]
|
|
760
|
-
resultDict["Records"] = selectedRows
|
|
761
|
-
|
|
762
|
-
# Generate the status dictionary
|
|
763
|
-
statusDict = {}
|
|
764
|
-
for row in selectedRows:
|
|
765
|
-
status = row[statusColumnIndex]
|
|
766
|
-
statusDict[status] = statusDict.setdefault(status, 0) + 1
|
|
767
|
-
resultDict["Extras"] = statusDict
|
|
768
|
-
|
|
769
|
-
# Obtain the distinct values of the selection parameters
|
|
770
|
-
res = self.transformationDB.getTableDistinctAttributeValues(
|
|
771
|
-
table, selectColumns, selectDict, older=toDate, newer=fromDate
|
|
772
|
-
)
|
|
773
|
-
distinctSelections = zip(selectColumns, [])
|
|
774
|
-
if res["OK"]:
|
|
775
|
-
distinctSelections = res["Value"]
|
|
776
|
-
resultDict["Selections"] = distinctSelections
|
|
777
|
-
|
|
778
|
-
return S_OK(resultDict)
|
|
779
|
-
|
|
780
|
-
types_getTransformationSummaryWeb = [dict, list, int, int]
|
|
781
|
-
|
|
782
|
-
def export_getTransformationSummaryWeb(self, selectDict, sortList, startItem, maxItems):
|
|
783
|
-
"""Get the summary of the transformation information for a given page in the generic format"""
|
|
784
|
-
|
|
785
|
-
# Obtain the timing information from the selectDict
|
|
786
|
-
last_update = selectDict.get("CreationDate", None)
|
|
787
|
-
if last_update:
|
|
788
|
-
del selectDict["CreationDate"]
|
|
789
|
-
fromDate = selectDict.get("FromDate", None)
|
|
790
|
-
if fromDate:
|
|
791
|
-
del selectDict["FromDate"]
|
|
792
|
-
if not fromDate:
|
|
793
|
-
fromDate = last_update
|
|
794
|
-
toDate = selectDict.get("ToDate", None)
|
|
795
|
-
if toDate:
|
|
796
|
-
del selectDict["ToDate"]
|
|
797
|
-
# Sorting instructions. Only one for the moment.
|
|
798
|
-
if sortList:
|
|
799
|
-
orderAttribute = []
|
|
800
|
-
for i in sortList:
|
|
801
|
-
orderAttribute += [i[0] + ":" + i[1]]
|
|
802
|
-
else:
|
|
803
|
-
orderAttribute = None
|
|
804
|
-
|
|
805
|
-
# Get the transformations that match the selection
|
|
806
|
-
res = self.transformationDB.getTransformations(
|
|
807
|
-
condDict=selectDict, older=toDate, newer=fromDate, orderAttribute=orderAttribute
|
|
808
|
-
)
|
|
809
|
-
if not res["OK"]:
|
|
810
|
-
return res
|
|
811
|
-
|
|
812
|
-
ops = Operations()
|
|
813
|
-
# Prepare the standard structure now within the resultDict dictionary
|
|
814
|
-
resultDict = {}
|
|
815
|
-
trList = res["Records"]
|
|
816
|
-
# Create the total records entry
|
|
817
|
-
nTrans = len(trList)
|
|
818
|
-
resultDict["TotalRecords"] = nTrans
|
|
819
|
-
# Create the ParameterNames entry
|
|
820
|
-
# As this list is a reference to the list in the DB, we cannot extend it, therefore copy it
|
|
821
|
-
resultDict["ParameterNames"] = list(res["ParameterNames"])
|
|
822
|
-
# Add the job states to the ParameterNames entry
|
|
823
|
-
taskStateNames = TASKS_STATE_NAMES + ops.getValue("Transformations/AdditionalTaskStates", [])
|
|
824
|
-
resultDict["ParameterNames"] += ["Jobs_" + x for x in taskStateNames]
|
|
825
|
-
# Add the file states to the ParameterNames entry
|
|
826
|
-
fileStateNames = FILES_STATE_NAMES + ops.getValue("Transformations/AdditionalFileStates", [])
|
|
827
|
-
resultDict["ParameterNames"] += ["Files_" + x for x in fileStateNames]
|
|
828
|
-
|
|
829
|
-
# Get the transformations which are within the selected window
|
|
830
|
-
if nTrans == 0:
|
|
831
|
-
return S_OK(resultDict)
|
|
832
|
-
ini = startItem
|
|
833
|
-
last = ini + maxItems
|
|
834
|
-
if ini >= nTrans:
|
|
835
|
-
return S_ERROR("Item number out of range")
|
|
836
|
-
if last > nTrans:
|
|
837
|
-
last = nTrans
|
|
838
|
-
transList = trList[ini:last]
|
|
839
|
-
|
|
840
|
-
statusDict = {}
|
|
841
|
-
extendableTranfs = ops.getValue("Transformations/ExtendableTransfTypes", ["Simulation", "MCsimulation"])
|
|
842
|
-
givenUpFileStatus = ops.getValue("Transformations/GivenUpFileStatus", ["MissingInFC"])
|
|
843
|
-
problematicStatuses = ops.getValue("Transformations/ProblematicStatuses", ["Problematic"])
|
|
844
|
-
# Add specific information for each selected transformation
|
|
845
|
-
for trans in transList:
|
|
846
|
-
transDict = dict(zip(resultDict["ParameterNames"], trans))
|
|
847
|
-
|
|
848
|
-
# Update the status counters
|
|
849
|
-
status = transDict["Status"]
|
|
850
|
-
statusDict[status] = statusDict.setdefault(status, 0) + 1
|
|
851
|
-
|
|
852
|
-
# Get the statistics on the number of jobs for the transformation
|
|
853
|
-
transID = transDict["TransformationID"]
|
|
854
|
-
res = self.transformationDB.getTransformationTaskStats(transID)
|
|
855
|
-
taskDict = {}
|
|
856
|
-
if res["OK"] and res["Value"]:
|
|
857
|
-
taskDict = res["Value"]
|
|
858
|
-
for state in taskStateNames:
|
|
859
|
-
trans.append(taskDict.get(state, 0))
|
|
860
|
-
|
|
861
|
-
# Get the statistics for the number of files for the transformation
|
|
862
|
-
fileDict = {}
|
|
863
|
-
transType = transDict["Type"]
|
|
864
|
-
if transType.lower() in extendableTranfs:
|
|
865
|
-
fileDict["PercentProcessed"] = "-"
|
|
866
|
-
else:
|
|
867
|
-
res = self.transformationDB.getTransformationStats(transID)
|
|
868
|
-
if res["OK"]:
|
|
869
|
-
fileDict = res["Value"]
|
|
870
|
-
total = fileDict["Total"]
|
|
871
|
-
for stat in givenUpFileStatus:
|
|
872
|
-
total -= fileDict.get(stat, 0)
|
|
873
|
-
processed = fileDict.get(TransformationFilesStatus.PROCESSED, 0)
|
|
874
|
-
fileDict["PercentProcessed"] = f"{int(processed * 1000.0 / total) / 10.0:.1f}" if total else 0.0
|
|
875
|
-
problematic = 0
|
|
876
|
-
for stat in problematicStatuses:
|
|
877
|
-
problematic += fileDict.get(stat, 0)
|
|
878
|
-
fileDict["Problematic"] = problematic
|
|
879
|
-
for state in fileStateNames:
|
|
880
|
-
trans.append(fileDict.get(state, 0))
|
|
881
|
-
|
|
882
|
-
resultDict["Records"] = transList
|
|
883
|
-
resultDict["Extras"] = statusDict
|
|
884
|
-
return S_OK(resultDict)
|
|
885
|
-
|
|
886
553
|
|
|
887
554
|
class TransformationManagerHandler(TransformationManagerHandlerMixin, RequestHandler):
|
|
888
555
|
pass
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
"""
|
|
2
2
|
Command Line Parameters for creating the Replication transformations Script
|
|
3
3
|
"""
|
|
4
|
-
from DIRAC import
|
|
4
|
+
from DIRAC import S_ERROR, S_OK, gLogger
|
|
5
|
+
from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getVOForGroup
|
|
5
6
|
from DIRAC.Core.Security.Properties import SecurityProperty
|
|
6
7
|
from DIRAC.Core.Security.ProxyInfo import getProxyInfo
|
|
7
|
-
from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getVOMSVOForGroup
|
|
8
8
|
|
|
9
9
|
|
|
10
10
|
class Params:
|
|
@@ -144,7 +144,7 @@ class Params:
|
|
|
144
144
|
return False
|
|
145
145
|
proxyValues = proxyInfo.get("Value", {})
|
|
146
146
|
group = proxyValues.get("group", "")
|
|
147
|
-
vomsvo =
|
|
147
|
+
vomsvo = getVOForGroup(group)
|
|
148
148
|
if not vomsvo:
|
|
149
149
|
self.errorMessages.append("ERROR: ProxyGroup not associated to VOMS VO, get a different proxy")
|
|
150
150
|
return False
|
|
@@ -2,19 +2,19 @@
|
|
|
2
2
|
from collections import OrderedDict, defaultdict
|
|
3
3
|
from itertools import zip_longest
|
|
4
4
|
|
|
5
|
-
from DIRAC import
|
|
5
|
+
from DIRAC import S_OK, gLogger
|
|
6
6
|
from DIRAC.Core.Utilities.List import breakListIntoChunks
|
|
7
7
|
from DIRAC.Core.Utilities.Proxy import UserProxy
|
|
8
8
|
from DIRAC.DataManagementSystem.Client.DataManager import DataManager
|
|
9
9
|
from DIRAC.TransformationSystem.Utilities.JobInfo import JobInfo
|
|
10
10
|
from DIRAC.WorkloadManagementSystem.Client import JobStatus
|
|
11
|
-
from DIRAC.WorkloadManagementSystem.
|
|
11
|
+
from DIRAC.WorkloadManagementSystem.Utilities.JobStatusUtility import JobStatusUtility
|
|
12
12
|
|
|
13
13
|
|
|
14
14
|
class TransformationInfo:
|
|
15
15
|
"""Hold information about a transformation."""
|
|
16
16
|
|
|
17
|
-
def __init__(self, transformationID, transInfoDict, enabled, tClient, fcClient, jobMon):
|
|
17
|
+
def __init__(self, transformationID, transInfoDict, enabled, tClient, fcClient, jobMon, jobStatusUtility=None):
|
|
18
18
|
"""Store clients etc."""
|
|
19
19
|
self.log = gLogger.getSubLogger(__name__ + f"[{transformationID}]")
|
|
20
20
|
self.enabled = enabled
|
|
@@ -26,7 +26,7 @@ class TransformationInfo:
|
|
|
26
26
|
self.transType = transInfoDict["Type"]
|
|
27
27
|
self.author = transInfoDict["Author"]
|
|
28
28
|
self.authorGroup = transInfoDict["AuthorGroup"]
|
|
29
|
-
self.
|
|
29
|
+
self.jobStatusUtility = jobStatusUtility or JobStatusUtility()
|
|
30
30
|
|
|
31
31
|
def checkTasksStatus(self):
|
|
32
32
|
"""Check the status for the task of given transformation and taskID"""
|
|
@@ -97,7 +97,9 @@ class TransformationInfo:
|
|
|
97
97
|
"""Update the job status."""
|
|
98
98
|
if self.enabled:
|
|
99
99
|
source = "DataRecoveryAgent"
|
|
100
|
-
result = self.
|
|
100
|
+
result = self.jobStatusUtility.setJobStatus(
|
|
101
|
+
int(jobID), status=status, minorStatus=minorstatus, source=source, dateTime=None, force=True
|
|
102
|
+
)
|
|
101
103
|
else:
|
|
102
104
|
return S_OK("DisabledMode")
|
|
103
105
|
if not result["OK"]:
|
|
@@ -16,7 +16,7 @@ from urllib.request import urlopen
|
|
|
16
16
|
from DIRAC.Interfaces.API.Dirac import Dirac
|
|
17
17
|
from DIRAC.Core.Utilities.File import mkDir
|
|
18
18
|
from DIRAC.Core.Base.Script import Script
|
|
19
|
-
from DIRAC.ConfigurationSystem.Client.Helpers.CSGlobals import getVO
|
|
19
|
+
from DIRAC.ConfigurationSystem.Client.Helpers.CSGlobals import getVO
|
|
20
20
|
from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData
|
|
21
21
|
|
|
22
22
|
|
|
@@ -79,14 +79,12 @@ def __configurePilot(basepath, vo):
|
|
|
79
79
|
This method was created specifically for LHCb pilots, more info
|
|
80
80
|
about othe VOs is needed to make it more general.
|
|
81
81
|
"""
|
|
82
|
-
currentSetup = getSetup()
|
|
83
82
|
masterCS = gConfigurationData.getMasterServer()
|
|
84
83
|
|
|
85
84
|
os.system(
|
|
86
85
|
"python "
|
|
87
86
|
+ basepath
|
|
88
|
-
+ "dirac-pilot.py -
|
|
89
|
-
% (currentSetup, vo, masterCS)
|
|
87
|
+
+ f"dirac-pilot.py -l {vo} -C {masterCS} -N ce.debug.ch -Q default -n DIRAC.JobDebugger.ch -dd"
|
|
90
88
|
)
|
|
91
89
|
|
|
92
90
|
diracdir = os.path.expanduser("~") + os.path.sep
|
|
@@ -1,20 +1,20 @@
|
|
|
1
1
|
"""Test the Transformationinfo."""
|
|
2
2
|
from collections import OrderedDict
|
|
3
3
|
from contextlib import contextmanager
|
|
4
|
+
from unittest.mock import MagicMock as Mock
|
|
5
|
+
from unittest.mock import patch
|
|
4
6
|
|
|
5
7
|
import pytest
|
|
6
|
-
from unittest.mock import MagicMock as Mock, patch
|
|
7
8
|
|
|
8
|
-
from DIRAC import S_OK, S_ERROR
|
|
9
9
|
import DIRAC
|
|
10
|
-
import DIRAC.TransformationSystem.Client.TransformationClient
|
|
11
10
|
import DIRAC.Resources.Catalog.FileCatalogClient
|
|
11
|
+
import DIRAC.TransformationSystem.Client.TransformationClient
|
|
12
12
|
import DIRAC.WorkloadManagementSystem.Client.JobMonitoringClient
|
|
13
|
+
from DIRAC import S_ERROR, S_OK
|
|
14
|
+
from DIRAC.tests.Utilities.utils import MatchStringWith
|
|
13
15
|
from DIRAC.TransformationSystem.Utilities.JobInfo import JobInfo
|
|
14
16
|
from DIRAC.TransformationSystem.Utilities.TransformationInfo import TransformationInfo
|
|
15
17
|
|
|
16
|
-
from DIRAC.tests.Utilities.utils import MatchStringWith
|
|
17
|
-
|
|
18
18
|
# pylint: disable=W0212, redefined-outer-name
|
|
19
19
|
|
|
20
20
|
|
|
@@ -53,7 +53,7 @@ def tiFixture():
|
|
|
53
53
|
tMock.setFileStatusForTransformation = Mock(name="setFileStat")
|
|
54
54
|
fcMock = Mock(name="fcMock", spec=DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient)
|
|
55
55
|
jmMock = Mock(name="jobMonMock", spec=DIRAC.WorkloadManagementSystem.Client.JobMonitoringClient.JobMonitoringClient)
|
|
56
|
-
jsucMock = Mock(name="jsuc", spec=DIRAC.WorkloadManagementSystem.
|
|
56
|
+
jsucMock = Mock(name="jsuc", spec=DIRAC.WorkloadManagementSystem.Utilities.JobStatusUtility.JobStatusUtility)
|
|
57
57
|
transInfoDict = dict(
|
|
58
58
|
TransformationID=1234,
|
|
59
59
|
TransformationName="TestProd12",
|
|
@@ -63,10 +63,15 @@ def tiFixture():
|
|
|
63
63
|
)
|
|
64
64
|
|
|
65
65
|
tri = TransformationInfo(
|
|
66
|
-
transformationID=1234,
|
|
66
|
+
transformationID=1234,
|
|
67
|
+
transInfoDict=transInfoDict,
|
|
68
|
+
enabled=False,
|
|
69
|
+
tClient=tMock,
|
|
70
|
+
fcClient=fcMock,
|
|
71
|
+
jobMon=jmMock,
|
|
72
|
+
jobStatusUtility=jsucMock,
|
|
67
73
|
)
|
|
68
74
|
tri.log = Mock(name="LogMock")
|
|
69
|
-
tri.jobStateClient = jsucMock
|
|
70
75
|
return tri
|
|
71
76
|
|
|
72
77
|
|
|
@@ -216,23 +221,25 @@ def test_setTaskStatus(tiFixture):
|
|
|
216
221
|
def test_updateJobStatus(tiFixture):
|
|
217
222
|
"""DIRAC.TransformationSystem.Utilities.TransformationInfo updateJobStatus................"""
|
|
218
223
|
|
|
219
|
-
tiFixture.
|
|
220
|
-
tiFixture.
|
|
224
|
+
tiFixture.jobStatusUtility.setJobStatus = Mock()
|
|
225
|
+
tiFixture.jobStatusUtility.setJobStatus.return_value = S_OK()
|
|
221
226
|
tiFixture.enabled = False
|
|
222
227
|
res = tiFixture._TransformationInfo__updateJobStatus(1234, "Failed", minorstatus=None)
|
|
223
228
|
assert res["OK"]
|
|
224
229
|
assert res["Value"] == "DisabledMode"
|
|
225
|
-
tiFixture.
|
|
230
|
+
tiFixture.jobStatusUtility.setJobStatus.assert_not_called()
|
|
226
231
|
|
|
227
|
-
tiFixture.
|
|
228
|
-
tiFixture.
|
|
232
|
+
tiFixture.jobStatusUtility.setJobStatus.reset()
|
|
233
|
+
tiFixture.jobStatusUtility.setJobStatus.return_value = S_OK("added record")
|
|
229
234
|
tiFixture.enabled = True
|
|
230
235
|
res = tiFixture._TransformationInfo__updateJobStatus(1234, "Failed", minorstatus=None)
|
|
231
236
|
assert res["OK"]
|
|
232
237
|
assert res["Value"] == "added record"
|
|
233
|
-
tiFixture.
|
|
238
|
+
tiFixture.jobStatusUtility.setJobStatus.assert_called_once_with(
|
|
239
|
+
1234, status="Failed", minorStatus=None, source="DataRecoveryAgent", dateTime=None, force=True
|
|
240
|
+
)
|
|
234
241
|
|
|
235
|
-
tiFixture.
|
|
242
|
+
tiFixture.jobStatusUtility.setJobStatus.return_value = S_ERROR("Error setting job status")
|
|
236
243
|
tiFixture.enabled = True
|
|
237
244
|
with pytest.raises(RuntimeError) as re:
|
|
238
245
|
tiFixture._TransformationInfo__updateJobStatus(1234, "Failed", minorstatus=None)
|
|
@@ -1,14 +1,13 @@
|
|
|
1
1
|
"""Test the dirac-transformation-replication script and helper"""
|
|
2
2
|
import unittest
|
|
3
|
+
from unittest.mock import MagicMock as Mock
|
|
4
|
+
from unittest.mock import patch
|
|
3
5
|
|
|
4
|
-
from
|
|
5
|
-
|
|
6
|
-
from DIRAC import S_OK, S_ERROR
|
|
7
|
-
|
|
8
|
-
from DIRAC.TransformationSystem.Utilities.ReplicationTransformation import createDataTransformation
|
|
6
|
+
from DIRAC import S_ERROR, S_OK
|
|
9
7
|
from DIRAC.TransformationSystem.Utilities.ReplicationCLIParameters import Params
|
|
8
|
+
from DIRAC.TransformationSystem.Utilities.ReplicationTransformation import createDataTransformation
|
|
10
9
|
|
|
11
|
-
GET_VOMS = "DIRAC.TransformationSystem.Utilities.ReplicationCLIParameters.
|
|
10
|
+
GET_VOMS = "DIRAC.TransformationSystem.Utilities.ReplicationCLIParameters.getVOForGroup"
|
|
12
11
|
GET_PROXY = "DIRAC.TransformationSystem.Utilities.ReplicationCLIParameters.getProxyInfo"
|
|
13
12
|
|
|
14
13
|
|
|
@@ -5,6 +5,8 @@ import os
|
|
|
5
5
|
import copy
|
|
6
6
|
import shutil
|
|
7
7
|
|
|
8
|
+
import pytest
|
|
9
|
+
|
|
8
10
|
from unittest.mock import MagicMock as Mock
|
|
9
11
|
|
|
10
12
|
from DIRAC import gLogger
|
|
@@ -742,6 +744,7 @@ class FailoverRequestSuccess(ModulesTestCase):
|
|
|
742
744
|
class ScriptSuccess(ModulesTestCase):
|
|
743
745
|
#################################################
|
|
744
746
|
|
|
747
|
+
@pytest.mark.slow
|
|
745
748
|
def test_execute(self):
|
|
746
749
|
self.script.jobType = "merge"
|
|
747
750
|
self.script.stepInputData = ["foo", "bar"]
|
|
@@ -770,6 +773,7 @@ class ScriptSuccess(ModulesTestCase):
|
|
|
770
773
|
class ScriptUnicode(ModulesTestCase):
|
|
771
774
|
#################################################
|
|
772
775
|
|
|
776
|
+
@pytest.mark.slow
|
|
773
777
|
def test_execute(self):
|
|
774
778
|
self.script.jobType = "merge"
|
|
775
779
|
self.script.stepInputData = ["foo", "bar"]
|
|
@@ -799,6 +803,7 @@ class ScriptUnicode(ModulesTestCase):
|
|
|
799
803
|
class ScriptFailure(ModulesTestCase):
|
|
800
804
|
#################################################
|
|
801
805
|
|
|
806
|
+
@pytest.mark.slow
|
|
802
807
|
def test_execute(self):
|
|
803
808
|
self.script.jobType = "merge"
|
|
804
809
|
self.script.stepInputData = ["foo", "bar"]
|