DIRAC 9.0.0a42__py3-none-any.whl → 9.0.7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- DIRAC/AccountingSystem/Client/AccountingCLI.py +0 -140
- DIRAC/AccountingSystem/Client/DataStoreClient.py +0 -13
- DIRAC/AccountingSystem/Client/Types/BaseAccountingType.py +0 -7
- DIRAC/AccountingSystem/ConfigTemplate.cfg +0 -5
- DIRAC/AccountingSystem/Service/DataStoreHandler.py +0 -72
- DIRAC/ConfigurationSystem/Client/Helpers/CSGlobals.py +0 -9
- DIRAC/ConfigurationSystem/Client/Helpers/Registry.py +38 -26
- DIRAC/ConfigurationSystem/Client/Helpers/Resources.py +11 -43
- DIRAC/ConfigurationSystem/Client/Helpers/test/Test_Helpers.py +0 -16
- DIRAC/ConfigurationSystem/Client/LocalConfiguration.py +14 -8
- DIRAC/ConfigurationSystem/Client/PathFinder.py +47 -8
- DIRAC/ConfigurationSystem/Client/SyncPlugins/CERNLDAPSyncPlugin.py +4 -1
- DIRAC/ConfigurationSystem/Client/VOMS2CSSynchronizer.py +32 -19
- DIRAC/ConfigurationSystem/Client/test/Test_PathFinder.py +41 -1
- DIRAC/ConfigurationSystem/private/RefresherBase.py +4 -2
- DIRAC/Core/Base/API.py +4 -7
- DIRAC/Core/Base/SQLAlchemyDB.py +1 -0
- DIRAC/Core/DISET/ServiceReactor.py +11 -3
- DIRAC/Core/DISET/private/BaseClient.py +1 -2
- DIRAC/Core/DISET/private/Transports/M2SSLTransport.py +9 -7
- DIRAC/Core/DISET/private/Transports/SSL/M2Utils.py +3 -1
- DIRAC/Core/LCG/GOCDBClient.py +5 -7
- DIRAC/Core/Security/DiracX.py +31 -17
- DIRAC/Core/Security/IAMService.py +5 -10
- DIRAC/Core/Security/Locations.py +27 -18
- DIRAC/Core/Security/ProxyInfo.py +9 -5
- DIRAC/Core/Security/VOMSService.py +2 -4
- DIRAC/Core/Security/m2crypto/X509Certificate.py +4 -6
- DIRAC/Core/Security/m2crypto/asn1_utils.py +17 -5
- DIRAC/Core/Security/test/test_diracx_token_from_pem.py +161 -0
- DIRAC/Core/Tornado/Client/ClientSelector.py +4 -1
- DIRAC/Core/Tornado/Server/TornadoService.py +1 -1
- DIRAC/Core/Utilities/CGroups2.py +328 -0
- DIRAC/Core/Utilities/ClassAd/ClassAdLight.py +4 -290
- DIRAC/Core/Utilities/DErrno.py +5 -309
- DIRAC/Core/Utilities/Extensions.py +10 -1
- DIRAC/Core/Utilities/File.py +1 -1
- DIRAC/Core/Utilities/Graphs/GraphData.py +1 -1
- DIRAC/Core/Utilities/Graphs/GraphUtilities.py +6 -1
- DIRAC/Core/Utilities/JDL.py +1 -195
- DIRAC/Core/Utilities/List.py +1 -124
- DIRAC/Core/Utilities/MySQL.py +103 -99
- DIRAC/Core/Utilities/Os.py +32 -1
- DIRAC/Core/Utilities/Platform.py +2 -107
- DIRAC/Core/Utilities/Proxy.py +0 -4
- DIRAC/Core/Utilities/ReturnValues.py +7 -252
- DIRAC/Core/Utilities/StateMachine.py +12 -178
- DIRAC/Core/Utilities/Subprocess.py +35 -14
- DIRAC/Core/Utilities/TimeUtilities.py +10 -253
- DIRAC/Core/Utilities/test/Test_JDL.py +0 -3
- DIRAC/Core/Utilities/test/Test_Profiler.py +20 -20
- DIRAC/Core/scripts/dirac_agent.py +1 -1
- DIRAC/Core/scripts/dirac_apptainer_exec.py +72 -46
- DIRAC/Core/scripts/dirac_configure.py +1 -3
- DIRAC/Core/scripts/dirac_install_db.py +24 -6
- DIRAC/Core/scripts/dirac_platform.py +1 -92
- DIRAC/DataManagementSystem/Agent/FTS3Agent.py +8 -7
- DIRAC/DataManagementSystem/Agent/RequestOperations/RemoveFile.py +7 -6
- DIRAC/DataManagementSystem/Client/FTS3Job.py +71 -34
- DIRAC/DataManagementSystem/DB/FTS3DB.py +7 -3
- DIRAC/DataManagementSystem/DB/FileCatalogComponents/DatasetManager/DatasetManager.py +1 -1
- DIRAC/DataManagementSystem/DB/FileCatalogDB.sql +9 -9
- DIRAC/DataManagementSystem/DB/FileCatalogWithFkAndPsDB.sql +9 -9
- DIRAC/DataManagementSystem/Utilities/DMSHelpers.py +6 -2
- DIRAC/DataManagementSystem/scripts/dirac_admin_allow_se.py +13 -8
- DIRAC/DataManagementSystem/scripts/dirac_admin_ban_se.py +13 -8
- DIRAC/DataManagementSystem/scripts/dirac_dms_create_moving_request.py +2 -0
- DIRAC/DataManagementSystem/scripts/dirac_dms_protocol_matrix.py +0 -1
- DIRAC/FrameworkSystem/Client/BundleDeliveryClient.py +2 -7
- DIRAC/FrameworkSystem/Client/ComponentInstaller.py +9 -4
- DIRAC/FrameworkSystem/Client/ProxyManagerClient.py +5 -2
- DIRAC/FrameworkSystem/Client/SystemAdministratorClientCLI.py +11 -6
- DIRAC/FrameworkSystem/ConfigTemplate.cfg +2 -0
- DIRAC/FrameworkSystem/DB/AuthDB.py +3 -3
- DIRAC/FrameworkSystem/DB/InstalledComponentsDB.py +4 -4
- DIRAC/FrameworkSystem/DB/ProxyDB.py +11 -3
- DIRAC/FrameworkSystem/DB/TokenDB.py +1 -1
- DIRAC/FrameworkSystem/Service/ProxyManagerHandler.py +8 -6
- DIRAC/FrameworkSystem/Utilities/MonitoringUtilities.py +2 -19
- DIRAC/FrameworkSystem/Utilities/TokenManagementUtilities.py +3 -2
- DIRAC/FrameworkSystem/Utilities/diracx.py +36 -14
- DIRAC/FrameworkSystem/private/authorization/AuthServer.py +2 -2
- DIRAC/FrameworkSystem/scripts/dirac_admin_update_pilot.py +18 -11
- DIRAC/FrameworkSystem/scripts/dirac_login.py +2 -2
- DIRAC/FrameworkSystem/scripts/dirac_proxy_init.py +7 -8
- DIRAC/Interfaces/API/Dirac.py +27 -15
- DIRAC/Interfaces/API/DiracAdmin.py +45 -17
- DIRAC/Interfaces/API/Job.py +9 -13
- DIRAC/Interfaces/scripts/dirac_admin_allow_site.py +12 -18
- DIRAC/Interfaces/scripts/dirac_admin_ban_site.py +12 -10
- DIRAC/Interfaces/scripts/dirac_admin_get_site_mask.py +4 -13
- DIRAC/Interfaces/scripts/dirac_admin_reset_job.py +3 -6
- DIRAC/Interfaces/scripts/dirac_wms_job_parameters.py +0 -1
- DIRAC/MonitoringSystem/Client/Types/WMSHistory.py +4 -0
- DIRAC/MonitoringSystem/Client/WebAppClient.py +26 -0
- DIRAC/MonitoringSystem/ConfigTemplate.cfg +9 -0
- DIRAC/MonitoringSystem/DB/MonitoringDB.py +6 -25
- DIRAC/MonitoringSystem/Service/MonitoringHandler.py +0 -33
- DIRAC/MonitoringSystem/Service/WebAppHandler.py +599 -0
- DIRAC/MonitoringSystem/private/MainReporter.py +0 -3
- DIRAC/ProductionSystem/DB/ProductionDB.sql +4 -4
- DIRAC/ProductionSystem/scripts/dirac_prod_get.py +2 -2
- DIRAC/ProductionSystem/scripts/dirac_prod_get_all.py +2 -2
- DIRAC/ProductionSystem/scripts/dirac_prod_get_trans.py +2 -3
- DIRAC/RequestManagementSystem/Agent/RequestExecutingAgent.py +8 -6
- DIRAC/RequestManagementSystem/Agent/RequestOperations/ForwardDISET.py +2 -14
- DIRAC/RequestManagementSystem/Client/ReqClient.py +66 -13
- DIRAC/RequestManagementSystem/ConfigTemplate.cfg +6 -6
- DIRAC/RequestManagementSystem/DB/RequestDB.py +10 -5
- DIRAC/RequestManagementSystem/DB/test/RMSTestScenari.py +2 -0
- DIRAC/RequestManagementSystem/private/RequestValidator.py +40 -46
- DIRAC/ResourceStatusSystem/Client/SiteStatus.py +4 -2
- DIRAC/ResourceStatusSystem/Command/FreeDiskSpaceCommand.py +3 -1
- DIRAC/ResourceStatusSystem/DB/ResourceManagementDB.py +8 -8
- DIRAC/ResourceStatusSystem/DB/ResourceStatusDB.py +2 -2
- DIRAC/ResourceStatusSystem/Utilities/CSHelpers.py +2 -31
- DIRAC/ResourceStatusSystem/scripts/dirac_rss_set_status.py +30 -12
- DIRAC/Resources/Catalog/RucioFileCatalogClient.py +195 -1
- DIRAC/Resources/Catalog/test/Test_RucioFileCatalogClient.py +181 -0
- DIRAC/Resources/Computing/AREXComputingElement.py +25 -8
- DIRAC/Resources/Computing/BatchSystems/Condor.py +126 -108
- DIRAC/Resources/Computing/BatchSystems/SLURM.py +5 -1
- DIRAC/Resources/Computing/BatchSystems/test/Test_SLURM.py +46 -0
- DIRAC/Resources/Computing/ComputingElement.py +1 -1
- DIRAC/Resources/Computing/HTCondorCEComputingElement.py +44 -44
- DIRAC/Resources/Computing/InProcessComputingElement.py +4 -2
- DIRAC/Resources/Computing/LocalComputingElement.py +1 -18
- DIRAC/Resources/Computing/SSHBatchComputingElement.py +1 -17
- DIRAC/Resources/Computing/SSHComputingElement.py +1 -18
- DIRAC/Resources/Computing/SingularityComputingElement.py +19 -5
- DIRAC/Resources/Computing/test/Test_HTCondorCEComputingElement.py +67 -49
- DIRAC/Resources/Computing/test/Test_PoolComputingElement.py +2 -1
- DIRAC/Resources/IdProvider/CheckInIdProvider.py +13 -0
- DIRAC/Resources/IdProvider/IdProviderFactory.py +11 -3
- DIRAC/Resources/MessageQueue/StompMQConnector.py +1 -1
- DIRAC/Resources/Storage/GFAL2_StorageBase.py +24 -15
- DIRAC/Resources/Storage/OccupancyPlugins/WLCGAccountingHTTPJson.py +1 -3
- DIRAC/Resources/Storage/StorageBase.py +4 -2
- DIRAC/Resources/Storage/StorageElement.py +6 -7
- DIRAC/StorageManagementSystem/DB/StorageManagementDB.sql +2 -2
- DIRAC/TransformationSystem/Agent/TaskManagerAgentBase.py +10 -16
- DIRAC/TransformationSystem/Agent/TransformationAgent.py +22 -1
- DIRAC/TransformationSystem/Agent/TransformationCleaningAgent.py +16 -16
- DIRAC/TransformationSystem/Client/TaskManager.py +2 -4
- DIRAC/TransformationSystem/Client/Transformation.py +6 -7
- DIRAC/TransformationSystem/Client/TransformationClient.py +21 -11
- DIRAC/TransformationSystem/Client/Utilities.py +9 -0
- DIRAC/TransformationSystem/DB/TransformationDB.py +11 -14
- DIRAC/TransformationSystem/DB/TransformationDB.sql +9 -9
- DIRAC/TransformationSystem/Service/TransformationManagerHandler.py +0 -333
- DIRAC/TransformationSystem/Utilities/ReplicationCLIParameters.py +3 -3
- DIRAC/TransformationSystem/Utilities/TransformationInfo.py +7 -5
- DIRAC/TransformationSystem/scripts/dirac_production_runjoblocal.py +2 -4
- DIRAC/TransformationSystem/test/Test_TransformationInfo.py +22 -15
- DIRAC/TransformationSystem/test/Test_replicationTransformation.py +5 -6
- DIRAC/Workflow/Modules/test/Test_Modules.py +5 -0
- DIRAC/WorkloadManagementSystem/Agent/JobAgent.py +38 -26
- DIRAC/WorkloadManagementSystem/Agent/JobCleaningAgent.py +12 -8
- DIRAC/WorkloadManagementSystem/Agent/PilotSyncAgent.py +4 -3
- DIRAC/WorkloadManagementSystem/Agent/PushJobAgent.py +13 -13
- DIRAC/WorkloadManagementSystem/Agent/SiteDirector.py +18 -14
- DIRAC/WorkloadManagementSystem/Agent/StalledJobAgent.py +18 -51
- DIRAC/WorkloadManagementSystem/Agent/StatesAccountingAgent.py +41 -1
- DIRAC/WorkloadManagementSystem/Agent/test/Test_Agent_JobAgent.py +45 -4
- DIRAC/WorkloadManagementSystem/Agent/test/Test_Agent_JobCleaningAgent.py +7 -9
- DIRAC/WorkloadManagementSystem/Agent/test/Test_Agent_PushJobAgent.py +1 -0
- DIRAC/WorkloadManagementSystem/Agent/test/Test_Agent_SiteDirector.py +9 -2
- DIRAC/WorkloadManagementSystem/Agent/test/Test_Agent_StalledJobAgent.py +4 -5
- DIRAC/WorkloadManagementSystem/Client/DownloadInputData.py +9 -9
- DIRAC/WorkloadManagementSystem/Client/InputDataResolution.py +6 -6
- DIRAC/WorkloadManagementSystem/Client/JobMonitoringClient.py +10 -11
- DIRAC/WorkloadManagementSystem/Client/JobReport.py +1 -1
- DIRAC/WorkloadManagementSystem/Client/JobState/CachedJobState.py +3 -0
- DIRAC/WorkloadManagementSystem/Client/JobState/JobManifest.py +32 -261
- DIRAC/WorkloadManagementSystem/Client/JobState/JobState.py +6 -0
- DIRAC/WorkloadManagementSystem/Client/JobStateUpdateClient.py +3 -0
- DIRAC/WorkloadManagementSystem/Client/JobStatus.py +8 -152
- DIRAC/WorkloadManagementSystem/Client/PoolXMLSlice.py +12 -19
- DIRAC/WorkloadManagementSystem/Client/SandboxStoreClient.py +25 -38
- DIRAC/WorkloadManagementSystem/Client/WMSClient.py +2 -3
- DIRAC/WorkloadManagementSystem/Client/test/Test_Client_DownloadInputData.py +29 -0
- DIRAC/WorkloadManagementSystem/ConfigTemplate.cfg +4 -8
- DIRAC/WorkloadManagementSystem/DB/JobDB.py +89 -132
- DIRAC/WorkloadManagementSystem/DB/JobDB.sql +8 -8
- DIRAC/WorkloadManagementSystem/DB/JobDBUtils.py +18 -147
- DIRAC/WorkloadManagementSystem/DB/JobLoggingDB.py +19 -6
- DIRAC/WorkloadManagementSystem/DB/JobParametersDB.py +9 -9
- DIRAC/WorkloadManagementSystem/DB/PilotAgentsDB.py +16 -5
- DIRAC/WorkloadManagementSystem/DB/PilotAgentsDB.sql +3 -3
- DIRAC/WorkloadManagementSystem/DB/SandboxMetadataDB.py +44 -82
- DIRAC/WorkloadManagementSystem/DB/StatusUtils.py +125 -0
- DIRAC/WorkloadManagementSystem/DB/tests/Test_JobDB.py +1 -1
- DIRAC/WorkloadManagementSystem/DB/tests/Test_StatusUtils.py +28 -0
- DIRAC/WorkloadManagementSystem/Executor/JobSanity.py +5 -4
- DIRAC/WorkloadManagementSystem/Executor/JobScheduling.py +4 -0
- DIRAC/WorkloadManagementSystem/FutureClient/JobStateUpdateClient.py +75 -33
- DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py +22 -11
- DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapperTemplate.py +9 -10
- DIRAC/WorkloadManagementSystem/JobWrapper/test/Test_JobWrapper.py +60 -10
- DIRAC/WorkloadManagementSystem/JobWrapper/test/Test_JobWrapperTemplate.py +4 -0
- DIRAC/WorkloadManagementSystem/Service/JobManagerHandler.py +33 -154
- DIRAC/WorkloadManagementSystem/Service/JobMonitoringHandler.py +5 -323
- DIRAC/WorkloadManagementSystem/Service/JobStateUpdateHandler.py +0 -16
- DIRAC/WorkloadManagementSystem/Service/PilotManagerHandler.py +6 -103
- DIRAC/WorkloadManagementSystem/Service/SandboxStoreHandler.py +7 -53
- DIRAC/WorkloadManagementSystem/Service/WMSAdministratorHandler.py +16 -79
- DIRAC/WorkloadManagementSystem/Service/WMSUtilities.py +4 -18
- DIRAC/WorkloadManagementSystem/Utilities/JobModel.py +28 -209
- DIRAC/WorkloadManagementSystem/Utilities/JobParameters.py +65 -3
- DIRAC/WorkloadManagementSystem/Utilities/JobStatusUtility.py +2 -64
- DIRAC/WorkloadManagementSystem/Utilities/ParametricJob.py +7 -171
- DIRAC/WorkloadManagementSystem/Utilities/PilotCStoJSONSynchronizer.py +73 -7
- DIRAC/WorkloadManagementSystem/Utilities/PilotWrapper.py +41 -11
- DIRAC/WorkloadManagementSystem/Utilities/RemoteRunner.py +16 -0
- DIRAC/WorkloadManagementSystem/Utilities/Utils.py +36 -1
- DIRAC/WorkloadManagementSystem/Utilities/jobAdministration.py +15 -0
- DIRAC/WorkloadManagementSystem/Utilities/test/Test_JobModel.py +1 -15
- DIRAC/WorkloadManagementSystem/Utilities/test/Test_ParametricJob.py +45 -128
- DIRAC/WorkloadManagementSystem/Utilities/test/Test_PilotWrapper.py +16 -0
- DIRAC/WorkloadManagementSystem/scripts/dirac_jobexec.py +7 -2
- DIRAC/WorkloadManagementSystem/scripts/dirac_wms_pilot_job_info.py +1 -1
- DIRAC/__init__.py +62 -60
- DIRAC/tests/Utilities/testJobDefinitions.py +22 -28
- {DIRAC-9.0.0a42.dist-info → dirac-9.0.7.dist-info}/METADATA +8 -5
- {DIRAC-9.0.0a42.dist-info → dirac-9.0.7.dist-info}/RECORD +229 -228
- {DIRAC-9.0.0a42.dist-info → dirac-9.0.7.dist-info}/WHEEL +1 -1
- {DIRAC-9.0.0a42.dist-info → dirac-9.0.7.dist-info}/entry_points.txt +0 -3
- DIRAC/Core/Utilities/test/Test_List.py +0 -150
- DIRAC/Core/Utilities/test/Test_Time.py +0 -88
- DIRAC/Resources/Computing/PilotBundle.py +0 -70
- DIRAC/TransformationSystem/scripts/dirac_transformation_archive.py +0 -30
- DIRAC/TransformationSystem/scripts/dirac_transformation_clean.py +0 -30
- DIRAC/TransformationSystem/scripts/dirac_transformation_remove_output.py +0 -30
- DIRAC/WorkloadManagementSystem/Utilities/test/Test_JobManager.py +0 -58
- {DIRAC-9.0.0a42.dist-info → dirac-9.0.7.dist-info/licenses}/LICENSE +0 -0
- {DIRAC-9.0.0a42.dist-info → dirac-9.0.7.dist-info}/top_level.txt +0 -0
|
@@ -260,6 +260,7 @@ def test__checkMatcherInfo(mocker, matcherInfo, matcherParams, expectedResult):
|
|
|
260
260
|
#############################################################################
|
|
261
261
|
|
|
262
262
|
|
|
263
|
+
@pytest.mark.slow
|
|
263
264
|
@pytest.mark.parametrize(
|
|
264
265
|
"mockGCReply, mockPMReply, expected",
|
|
265
266
|
[
|
|
@@ -308,6 +309,7 @@ def test__setupProxy(mocker, mockGCReply, mockPMReply, expected):
|
|
|
308
309
|
assert result["Message"] == expected["Message"]
|
|
309
310
|
|
|
310
311
|
|
|
312
|
+
@pytest.mark.slow
|
|
311
313
|
@pytest.mark.parametrize(
|
|
312
314
|
"mockGCReply, mockPMReply, expected",
|
|
313
315
|
[
|
|
@@ -611,7 +613,7 @@ def test_submitJob(mocker, mockJWInput, expected):
|
|
|
611
613
|
("Pool/Singularity", jobScript % "1", (["Failed to find singularity"], []), ([], [])),
|
|
612
614
|
],
|
|
613
615
|
)
|
|
614
|
-
def test_submitAndCheckJob(mocker, manageJobFiles, localCE, job, expectedResult1, expectedResult2):
|
|
616
|
+
def test_submitAndCheckJob(mocker, manageJobFiles, localCE, job, expectedResult1, expectedResult2, tmp_path):
|
|
615
617
|
"""Test the submission and the management of the job status."""
|
|
616
618
|
jobID = "123"
|
|
617
619
|
jobExecutablePath, jobWrapperPath, jobWrapperConfigPath = manageJobFiles
|
|
@@ -635,7 +637,12 @@ def test_submitAndCheckJob(mocker, manageJobFiles, localCE, job, expectedResult1
|
|
|
635
637
|
),
|
|
636
638
|
)
|
|
637
639
|
mocker.patch("DIRAC.WorkloadManagementSystem.Agent.JobAgent.JobAgent._sendFailoverRequest", return_value=S_OK())
|
|
638
|
-
|
|
640
|
+
empty_file_path = tmp_path / "empty_file"
|
|
641
|
+
empty_file_path.touch()
|
|
642
|
+
mocker.patch(
|
|
643
|
+
"DIRAC.WorkloadManagementSystem.Agent.JobAgent.writeChainToTemporaryFile",
|
|
644
|
+
return_value=S_OK(str(empty_file_path)),
|
|
645
|
+
)
|
|
639
646
|
mocker.patch(
|
|
640
647
|
"DIRAC.Resources.Computing.SingularityComputingElement.SingularityComputingElement.submitJob",
|
|
641
648
|
return_value=S_ERROR("Failed to find singularity"),
|
|
@@ -712,7 +719,7 @@ def test_submitAndCheckJob(mocker, manageJobFiles, localCE, job, expectedResult1
|
|
|
712
719
|
assert len(jobAgent.computingElement.taskResults) == 0
|
|
713
720
|
|
|
714
721
|
|
|
715
|
-
def test_submitAndCheck2Jobs(mocker):
|
|
722
|
+
def test_submitAndCheck2Jobs(mocker, tmp_path):
|
|
716
723
|
"""Test the submission and the management of the job status.
|
|
717
724
|
|
|
718
725
|
This time, a first job is successfully submitted, but the second submission fails.
|
|
@@ -728,7 +735,17 @@ def test_submitAndCheck2Jobs(mocker):
|
|
|
728
735
|
),
|
|
729
736
|
)
|
|
730
737
|
mocker.patch("DIRAC.WorkloadManagementSystem.Agent.JobAgent.JobAgent._sendFailoverRequest", return_value=S_OK())
|
|
731
|
-
|
|
738
|
+
|
|
739
|
+
def make_empty_file(*args, **kwargs):
|
|
740
|
+
"""Create an empty file and return its path."""
|
|
741
|
+
empty_file_path = tmp_path / "empty_file"
|
|
742
|
+
empty_file_path.touch()
|
|
743
|
+
return S_OK(str(empty_file_path))
|
|
744
|
+
|
|
745
|
+
mocker.patch(
|
|
746
|
+
"DIRAC.WorkloadManagementSystem.Agent.JobAgent.writeChainToTemporaryFile",
|
|
747
|
+
make_empty_file,
|
|
748
|
+
)
|
|
732
749
|
mocker.patch(
|
|
733
750
|
"DIRAC.Resources.Computing.InProcessComputingElement.InProcessComputingElement.submitJob",
|
|
734
751
|
side_effect=[S_OK(), S_ERROR("ComputingElement error")],
|
|
@@ -788,3 +805,27 @@ def test_submitAndCheck2Jobs(mocker):
|
|
|
788
805
|
|
|
789
806
|
# From here, taskResults should be empty
|
|
790
807
|
assert len(jobAgent.computingElement.taskResults) == 0
|
|
808
|
+
|
|
809
|
+
|
|
810
|
+
def test_failureBeforeSubmission(mocker):
|
|
811
|
+
"""Test that a failure before job submission is handled correctly.
|
|
812
|
+
|
|
813
|
+
We want to make sure that there is no endless loop in the finalize method.
|
|
814
|
+
"""
|
|
815
|
+
# Mock the JobAgent
|
|
816
|
+
mocker.patch("DIRAC.WorkloadManagementSystem.Agent.JobAgent.AgentModule.__init__")
|
|
817
|
+
|
|
818
|
+
jobAgent = JobAgent("JobAgent", "Test")
|
|
819
|
+
jobAgent.log = gLogger.getSubLogger("JobAgent")
|
|
820
|
+
|
|
821
|
+
# Here we simulate a failure before the job submission: no TaskID
|
|
822
|
+
jobID = "123"
|
|
823
|
+
jobAgent.jobs[jobID] = {}
|
|
824
|
+
jobAgent.jobs[jobID]["JobReport"] = JobReport(jobID)
|
|
825
|
+
|
|
826
|
+
# Make sure that the job is removed from jobAgent.jobs
|
|
827
|
+
result = jobAgent._checkSubmittedJobs()
|
|
828
|
+
assert result["OK"]
|
|
829
|
+
assert result["Value"] == ([], [])
|
|
830
|
+
|
|
831
|
+
assert jobAgent.jobs == {}
|
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
""" Test class for Job Cleaning Agent
|
|
2
2
|
"""
|
|
3
|
-
import pytest
|
|
4
3
|
from unittest.mock import MagicMock
|
|
5
4
|
|
|
5
|
+
import pytest
|
|
6
|
+
|
|
6
7
|
# DIRAC Components
|
|
7
|
-
from DIRAC import
|
|
8
|
+
from DIRAC import S_OK, gLogger
|
|
8
9
|
from DIRAC.WorkloadManagementSystem.Agent.JobCleaningAgent import JobCleaningAgent
|
|
9
10
|
|
|
10
11
|
gLogger.setLevel("DEBUG")
|
|
@@ -32,7 +33,6 @@ def jca(mocker):
|
|
|
32
33
|
mocker.patch("DIRAC.WorkloadManagementSystem.Agent.JobCleaningAgent.JobDB.selectJobs", side_effect=mockReply)
|
|
33
34
|
mocker.patch("DIRAC.WorkloadManagementSystem.Agent.JobCleaningAgent.JobDB.__init__", side_effect=mockNone)
|
|
34
35
|
mocker.patch("DIRAC.WorkloadManagementSystem.Agent.JobCleaningAgent.ReqClient", return_value=mockNone)
|
|
35
|
-
mocker.patch("DIRAC.WorkloadManagementSystem.Agent.JobCleaningAgent.JobMonitoringClient", return_value=mockJMC)
|
|
36
36
|
|
|
37
37
|
jca = JobCleaningAgent()
|
|
38
38
|
jca.log = gLogger
|
|
@@ -98,7 +98,7 @@ def test_deleteJobsByStatus(jca, conditions, mockReplyInput, expected):
|
|
|
98
98
|
"inputs, params, expected",
|
|
99
99
|
[
|
|
100
100
|
([], {"OK": True, "Value": {}}, {"OK": True, "Value": {"Failed": {}, "Successful": {}}}),
|
|
101
|
-
(["
|
|
101
|
+
(["123", "456"], {"OK": True, "Value": {}}, {"OK": True, "Value": {"Failed": {}, "Successful": {}}}),
|
|
102
102
|
(
|
|
103
103
|
[],
|
|
104
104
|
{"OK": True, "Value": {1: {"OutputSandboxLFN": "/some/lfn/1.txt"}}},
|
|
@@ -113,11 +113,11 @@ def test_deleteJobsByStatus(jca, conditions, mockReplyInput, expected):
|
|
|
113
113
|
{"OK": True, "Value": {"Failed": {}, "Successful": {1: "/some/lfn/1.txt", 2: "/some/other/lfn/2.txt"}}},
|
|
114
114
|
),
|
|
115
115
|
(
|
|
116
|
-
["
|
|
116
|
+
["123", "456"],
|
|
117
117
|
{"OK": True, "Value": {1: {"OutputSandboxLFN": "/some/lfn/1.txt"}}},
|
|
118
118
|
{"OK": True, "Value": {"Failed": {}, "Successful": {1: "/some/lfn/1.txt"}}},
|
|
119
119
|
),
|
|
120
|
-
(["
|
|
120
|
+
(["123", "456"], {"OK": False}, {"OK": False}),
|
|
121
121
|
],
|
|
122
122
|
)
|
|
123
123
|
def test_deleteJobOversizedSandbox(mocker, inputs, params, expected):
|
|
@@ -127,10 +127,10 @@ def test_deleteJobOversizedSandbox(mocker, inputs, params, expected):
|
|
|
127
127
|
mocker.patch("DIRAC.WorkloadManagementSystem.Agent.JobCleaningAgent.AgentModule.am_getOption", return_value=mockAM)
|
|
128
128
|
mocker.patch("DIRAC.WorkloadManagementSystem.Agent.JobCleaningAgent.JobDB", return_value=mockNone)
|
|
129
129
|
mocker.patch("DIRAC.WorkloadManagementSystem.Agent.JobCleaningAgent.ReqClient", return_value=mockNone)
|
|
130
|
-
mocker.patch("DIRAC.WorkloadManagementSystem.Agent.JobCleaningAgent.JobMonitoringClient", return_value=mockJMC)
|
|
131
130
|
mocker.patch(
|
|
132
131
|
"DIRAC.WorkloadManagementSystem.Agent.JobCleaningAgent.getDNForUsername", return_value=S_OK(["/bih/boh/DN"])
|
|
133
132
|
)
|
|
133
|
+
mocker.patch("DIRAC.WorkloadManagementSystem.Agent.JobCleaningAgent.getJobParameters", return_value=params)
|
|
134
134
|
|
|
135
135
|
jobCleaningAgent = JobCleaningAgent()
|
|
136
136
|
jobCleaningAgent.log = gLogger
|
|
@@ -138,8 +138,6 @@ def test_deleteJobOversizedSandbox(mocker, inputs, params, expected):
|
|
|
138
138
|
jobCleaningAgent._AgentModule__configDefaults = mockAM
|
|
139
139
|
jobCleaningAgent.initialize()
|
|
140
140
|
|
|
141
|
-
mockJMC.getJobParameters.return_value = params
|
|
142
|
-
|
|
143
141
|
result = jobCleaningAgent.deleteJobOversizedSandbox(inputs)
|
|
144
142
|
|
|
145
143
|
assert result == expected
|
|
@@ -169,10 +169,16 @@ def sd(mocker, config):
|
|
|
169
169
|
gConfig.getSections("Resources/Sites/LCG")["Value"] + gConfig.getSections("Resources/Sites/DIRAC")["Value"]
|
|
170
170
|
)
|
|
171
171
|
mocker.patch(
|
|
172
|
-
"DIRAC.WorkloadManagementSystem.Agent.SiteDirector.SiteStatus.getUsableSites",
|
|
172
|
+
"DIRAC.WorkloadManagementSystem.Agent.SiteDirector.SiteStatus.getUsableSites", return_value=S_OK(usableSites)
|
|
173
173
|
)
|
|
174
|
+
|
|
175
|
+
# Mock getElementStatus to return a properly formatted dictionary
|
|
176
|
+
def mock_getElementStatus(ceNamesList, *args, **kwargs):
|
|
177
|
+
return S_OK({ceName: {"all": "Active"} for ceName in ceNamesList})
|
|
178
|
+
|
|
174
179
|
mocker.patch(
|
|
175
|
-
"DIRAC.WorkloadManagementSystem.Agent.SiteDirector.ResourceStatus.getElementStatus",
|
|
180
|
+
"DIRAC.WorkloadManagementSystem.Agent.SiteDirector.ResourceStatus.getElementStatus",
|
|
181
|
+
side_effect=mock_getElementStatus,
|
|
176
182
|
)
|
|
177
183
|
mocker.patch(
|
|
178
184
|
"DIRAC.WorkloadManagementSystem.Agent.SiteDirector.gProxyManager.downloadProxy", side_effect=mockPMProxyReply
|
|
@@ -261,6 +267,7 @@ def test_getPilotWrapper(mocker, sd, pilotWrapperDirectory):
|
|
|
261
267
|
# Get pilot options
|
|
262
268
|
pilotOptions = sd._getPilotOptions("ce1.site1.com_condor")
|
|
263
269
|
assert {
|
|
270
|
+
"--architectureScript=123",
|
|
264
271
|
"--preinstalledEnv=123",
|
|
265
272
|
"--wnVO=dteam",
|
|
266
273
|
"-n LCG.Site1.com",
|
|
@@ -25,11 +25,10 @@ def sja(mocker):
|
|
|
25
25
|
)
|
|
26
26
|
mocker.patch("DIRAC.WorkloadManagementSystem.Agent.StalledJobAgent.JobDB")
|
|
27
27
|
mocker.patch("DIRAC.WorkloadManagementSystem.Agent.StalledJobAgent.JobLoggingDB")
|
|
28
|
-
mocker.patch("DIRAC.WorkloadManagementSystem.Agent.StalledJobAgent.
|
|
29
|
-
mocker.patch("DIRAC.WorkloadManagementSystem.Agent.StalledJobAgent.
|
|
30
|
-
mocker.patch("DIRAC.WorkloadManagementSystem.Agent.StalledJobAgent.
|
|
31
|
-
mocker.patch("DIRAC.WorkloadManagementSystem.Agent.StalledJobAgent.
|
|
32
|
-
mocker.patch("DIRAC.WorkloadManagementSystem.Agent.StalledJobAgent.getDNForUsername", return_value=MagicMock())
|
|
28
|
+
mocker.patch("DIRAC.WorkloadManagementSystem.Agent.StalledJobAgent.rescheduleJobs", return_value=MagicMock())
|
|
29
|
+
mocker.patch("DIRAC.WorkloadManagementSystem.Agent.StalledJobAgent.PilotAgentsDB", return_value=MagicMock())
|
|
30
|
+
mocker.patch("DIRAC.WorkloadManagementSystem.Agent.StalledJobAgent.getJobParameters", return_value=MagicMock())
|
|
31
|
+
mocker.patch("DIRAC.WorkloadManagementSystem.Agent.StalledJobAgent.kill_delete_jobs", return_value=MagicMock())
|
|
33
32
|
|
|
34
33
|
stalledJobAgent = StalledJobAgent()
|
|
35
34
|
stalledJobAgent._AgentModule__configDefaults = mockAM
|
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
"""
|
|
2
|
-
|
|
1
|
+
"""The Download Input Data module wraps around the Replica Management
|
|
2
|
+
components to provide access to datasets by downloading locally
|
|
3
3
|
"""
|
|
4
|
+
|
|
4
5
|
import os
|
|
5
6
|
import random
|
|
6
7
|
import tempfile
|
|
@@ -12,8 +13,6 @@ from DIRAC.DataManagementSystem.Utilities.DMSHelpers import DMSHelpers
|
|
|
12
13
|
from DIRAC.Resources.Storage.StorageElement import StorageElement
|
|
13
14
|
from DIRAC.WorkloadManagementSystem.Client.JobStateUpdateClient import JobStateUpdateClient
|
|
14
15
|
|
|
15
|
-
COMPONENT_NAME = "DownloadInputData"
|
|
16
|
-
|
|
17
16
|
|
|
18
17
|
def _isCached(lfn, seName):
|
|
19
18
|
result = returnSingleResult(StorageElement(seName).getFileMetadata(lfn))
|
|
@@ -137,7 +136,7 @@ class DownloadInputData:
|
|
|
137
136
|
if not result["Value"]:
|
|
138
137
|
self.log.warn("Not enough disk space available for download", f"{result['Value']} / {totalSize} bytes")
|
|
139
138
|
self.__setJobParam(
|
|
140
|
-
|
|
139
|
+
self.__class__.__name__,
|
|
141
140
|
f"Not enough disk space available for download: {result['Value']} / {totalSize} bytes",
|
|
142
141
|
)
|
|
143
142
|
return S_OK({"Failed": self.inputData, "Successful": {}})
|
|
@@ -216,7 +215,7 @@ class DownloadInputData:
|
|
|
216
215
|
report += "\n".join(failedReplicas)
|
|
217
216
|
|
|
218
217
|
if report:
|
|
219
|
-
self.__setJobParam(
|
|
218
|
+
self.__setJobParam(self.__class__.__name__, report)
|
|
220
219
|
|
|
221
220
|
return S_OK({"Successful": resolvedData, "Failed": failedReplicas})
|
|
222
221
|
|
|
@@ -228,7 +227,7 @@ class DownloadInputData:
|
|
|
228
227
|
diskSpace = getDiskSpace(self.__getDownloadDir(False)) # MB
|
|
229
228
|
availableBytes = diskSpace * 1024 * 1024 # bytes
|
|
230
229
|
bufferGBs = gConfig.getValue(
|
|
231
|
-
os.path.join("/Systems/WorkloadManagement/JobWrapper", "
|
|
230
|
+
os.path.join("/Systems/WorkloadManagement/JobWrapper", "MinOutputDataBufferGB"), 5.0
|
|
232
231
|
)
|
|
233
232
|
data = bufferGBs * 1024 * 1024 * 1024 # bufferGBs in bytes
|
|
234
233
|
if (data + totalSize) < availableBytes:
|
|
@@ -245,12 +244,13 @@ class DownloadInputData:
|
|
|
245
244
|
return S_ERROR(msg)
|
|
246
245
|
|
|
247
246
|
def __getDownloadDir(self, incrementCounter=True):
|
|
247
|
+
jobIDPath = str(self.configuration.get("JobIDPath", os.getcwd()))
|
|
248
248
|
if self.inputDataDirectory == "PerFile":
|
|
249
249
|
if incrementCounter:
|
|
250
250
|
self.counter += 1
|
|
251
|
-
return tempfile.mkdtemp(prefix=f"InputData_{self.counter}", dir=
|
|
251
|
+
return tempfile.mkdtemp(prefix=f"InputData_{self.counter}", dir=jobIDPath)
|
|
252
252
|
elif self.inputDataDirectory == "CWD":
|
|
253
|
-
return
|
|
253
|
+
return jobIDPath
|
|
254
254
|
else:
|
|
255
255
|
return self.inputDataDirectory
|
|
256
256
|
|
|
@@ -6,13 +6,14 @@
|
|
|
6
6
|
result and in principle has all the necessary information to resolve input data
|
|
7
7
|
for applications.
|
|
8
8
|
"""
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
|
|
9
11
|
import DIRAC
|
|
10
|
-
from DIRAC import S_OK,
|
|
12
|
+
from DIRAC import S_ERROR, S_OK, gConfig, gLogger
|
|
13
|
+
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
|
|
11
14
|
from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader
|
|
12
15
|
from DIRAC.WorkloadManagementSystem.Client.PoolXMLSlice import PoolXMLSlice
|
|
13
|
-
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
|
|
14
16
|
|
|
15
|
-
COMPONENT_NAME = "InputDataResolution"
|
|
16
17
|
CREATE_CATALOG = False
|
|
17
18
|
|
|
18
19
|
|
|
@@ -22,8 +23,7 @@ class InputDataResolution:
|
|
|
22
23
|
def __init__(self, argumentsDict):
|
|
23
24
|
"""Standard constructor"""
|
|
24
25
|
self.arguments = argumentsDict
|
|
25
|
-
self.
|
|
26
|
-
self.log = gLogger.getSubLogger(self.name)
|
|
26
|
+
self.log = gLogger.getSubLogger(self.__class__.__name__)
|
|
27
27
|
op = Operations()
|
|
28
28
|
self.arguments.setdefault("Configuration", {})["AllReplicas"] = op.getValue(
|
|
29
29
|
"InputDataPolicy/AllReplicas", False
|
|
@@ -78,7 +78,7 @@ class InputDataResolution:
|
|
|
78
78
|
self.log.verbose(f"Catalog name will be: {catalogName}")
|
|
79
79
|
|
|
80
80
|
resolvedData = tmpDict
|
|
81
|
-
appCatalog = PoolXMLSlice(catalogName)
|
|
81
|
+
appCatalog = PoolXMLSlice(catalogName, Path(self.arguments["Configuration"].get("JobIDPath", "")))
|
|
82
82
|
return appCatalog.execute(resolvedData)
|
|
83
83
|
|
|
84
84
|
def __resolveInputData(self):
|
|
@@ -14,12 +14,14 @@ except ImportError:
|
|
|
14
14
|
|
|
15
15
|
@createClient("WorkloadManagement/JobMonitoring")
|
|
16
16
|
class JobMonitoringClient(Client):
|
|
17
|
+
# Set to None to raise an error if this service is set as "legacy adapted"
|
|
18
|
+
# See ClientSelector
|
|
19
|
+
diracxClient = None
|
|
20
|
+
|
|
17
21
|
def __init__(self, **kwargs):
|
|
18
22
|
super().__init__(**kwargs)
|
|
19
23
|
self.setServer("WorkloadManagement/JobMonitoring")
|
|
20
24
|
|
|
21
|
-
diracxClient = futureJobMonitoringClient
|
|
22
|
-
|
|
23
25
|
@ignoreEncodeWarning
|
|
24
26
|
def getJobsStatus(self, jobIDs):
|
|
25
27
|
res = self._getRPC().getJobsStatus(jobIDs)
|
|
@@ -38,15 +40,6 @@ class JobMonitoringClient(Client):
|
|
|
38
40
|
res["Value"] = strToIntDict(res["Value"])
|
|
39
41
|
return res
|
|
40
42
|
|
|
41
|
-
@ignoreEncodeWarning
|
|
42
|
-
def getJobsParameters(self, jobIDs, parameters):
|
|
43
|
-
res = self._getRPC().getJobsParameters(jobIDs, parameters)
|
|
44
|
-
|
|
45
|
-
# Cast the str keys to int
|
|
46
|
-
if res["OK"]:
|
|
47
|
-
res["Value"] = strToIntDict(res["Value"])
|
|
48
|
-
return res
|
|
49
|
-
|
|
50
43
|
@ignoreEncodeWarning
|
|
51
44
|
def getJobsMinorStatus(self, jobIDs):
|
|
52
45
|
res = self._getRPC().getJobsMinorStatus(jobIDs)
|
|
@@ -79,3 +72,9 @@ class JobMonitoringClient(Client):
|
|
|
79
72
|
if res["OK"]:
|
|
80
73
|
res["Value"] = strToIntDict(res["Value"])
|
|
81
74
|
return res
|
|
75
|
+
|
|
76
|
+
def getInputData(self, jobIDs):
|
|
77
|
+
res = self._getRPC().getInputData(jobIDs)
|
|
78
|
+
if res["OK"] and isinstance(res["Value"], dict):
|
|
79
|
+
res["Value"] = strToIntDict(res["Value"])
|
|
80
|
+
return res
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
import datetime
|
|
5
5
|
from collections import defaultdict
|
|
6
6
|
|
|
7
|
-
from DIRAC import
|
|
7
|
+
from DIRAC import S_ERROR, S_OK, gLogger
|
|
8
8
|
from DIRAC.Core.Utilities import DEncode
|
|
9
9
|
from DIRAC.RequestManagementSystem.Client.Operation import Operation
|
|
10
10
|
from DIRAC.WorkloadManagementSystem.Client.JobStateUpdateClient import JobStateUpdateClient
|
|
@@ -364,6 +364,9 @@ class CachedJobState:
|
|
|
364
364
|
def getInputData(self):
|
|
365
365
|
return self.__cacheResult("inputData", self.__jobState.getInputData)
|
|
366
366
|
|
|
367
|
+
def setInputData(self, inputData):
|
|
368
|
+
return self.__jobState.setInputData(inputData)
|
|
369
|
+
|
|
367
370
|
def insertIntoTQ(self):
|
|
368
371
|
if self.valid:
|
|
369
372
|
self.__insertIntoTQ = True
|
|
@@ -1,266 +1,37 @@
|
|
|
1
|
-
from
|
|
1
|
+
from __future__ import annotations
|
|
2
2
|
|
|
3
|
-
from
|
|
4
|
-
from DIRAC.Core.Utilities import List
|
|
5
|
-
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
|
|
6
|
-
from DIRAC.Core.Utilities.JDL import loadJDLAsCFG, dumpCFGAsJDL
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
class JobManifest:
|
|
10
|
-
def __init__(self, manifest=""):
|
|
11
|
-
self.__manifest = CFG()
|
|
12
|
-
self.__dirty = False
|
|
13
|
-
self.__ops = False
|
|
14
|
-
if manifest:
|
|
15
|
-
result = self.load(manifest)
|
|
16
|
-
if not result["OK"]:
|
|
17
|
-
raise Exception(result["Message"])
|
|
18
|
-
|
|
19
|
-
def isDirty(self):
|
|
20
|
-
return self.__dirty
|
|
21
|
-
|
|
22
|
-
def setDirty(self):
|
|
23
|
-
self.__dirty = True
|
|
24
|
-
|
|
25
|
-
def clearDirty(self):
|
|
26
|
-
self.__dirty = False
|
|
27
|
-
|
|
28
|
-
def load(self, dataString):
|
|
29
|
-
"""
|
|
30
|
-
Auto discover format type based on [ .. ] of JDL
|
|
31
|
-
"""
|
|
32
|
-
dataString = dataString.strip()
|
|
33
|
-
if dataString[0] == "[" and dataString[-1] == "]":
|
|
34
|
-
return self.loadJDL(dataString)
|
|
35
|
-
else:
|
|
36
|
-
return self.loadCFG(dataString)
|
|
37
|
-
|
|
38
|
-
def loadJDL(self, jdlString):
|
|
39
|
-
"""
|
|
40
|
-
Load job manifest from JDL format
|
|
41
|
-
"""
|
|
42
|
-
result = loadJDLAsCFG(jdlString.strip())
|
|
43
|
-
if not result["OK"]:
|
|
44
|
-
self.__manifest = CFG()
|
|
45
|
-
return result
|
|
46
|
-
self.__manifest = result["Value"][0]
|
|
47
|
-
return S_OK()
|
|
48
|
-
|
|
49
|
-
def loadCFG(self, cfgString):
|
|
50
|
-
"""
|
|
51
|
-
Load job manifest from CFG format
|
|
52
|
-
"""
|
|
53
|
-
try:
|
|
54
|
-
self.__manifest.loadFromBuffer(cfgString)
|
|
55
|
-
except Exception as e:
|
|
56
|
-
return S_ERROR(f"Can't load manifest from cfg: {str(e)}")
|
|
57
|
-
return S_OK()
|
|
58
|
-
|
|
59
|
-
def dumpAsCFG(self):
|
|
60
|
-
return str(self.__manifest)
|
|
61
|
-
|
|
62
|
-
def getAsCFG(self):
|
|
63
|
-
return self.__manifest.clone()
|
|
64
|
-
|
|
65
|
-
def dumpAsJDL(self):
|
|
66
|
-
return dumpCFGAsJDL(self.__manifest)
|
|
67
|
-
|
|
68
|
-
def __getCSValue(self, varName, defaultVal=None):
|
|
69
|
-
if not self.__ops:
|
|
70
|
-
self.__ops = Operations(group=self.__manifest["OwnerGroup"])
|
|
71
|
-
if varName[0] != "/":
|
|
72
|
-
varName = f"JobDescription/{varName}"
|
|
73
|
-
return self.__ops.getValue(varName, defaultVal)
|
|
74
|
-
|
|
75
|
-
def __checkNumericalVar(self, varName, defaultVal, minVal, maxVal):
|
|
76
|
-
"""
|
|
77
|
-
Check a numerical var
|
|
78
|
-
"""
|
|
79
|
-
initialVal = False
|
|
80
|
-
if varName not in self.__manifest:
|
|
81
|
-
varValue = self.__getCSValue(f"Default{varName}", defaultVal)
|
|
82
|
-
else:
|
|
83
|
-
varValue = self.__manifest[varName]
|
|
84
|
-
initialVal = varValue
|
|
85
|
-
try:
|
|
86
|
-
varValue = int(varValue)
|
|
87
|
-
except ValueError:
|
|
88
|
-
return S_ERROR(f"{varName} must be a number")
|
|
89
|
-
minVal = self.__getCSValue(f"Min{varName}", minVal)
|
|
90
|
-
maxVal = self.__getCSValue(f"Max{varName}", maxVal)
|
|
91
|
-
varValue = max(minVal, min(varValue, maxVal))
|
|
92
|
-
if initialVal != varValue:
|
|
93
|
-
self.__manifest.setOption(varName, varValue)
|
|
94
|
-
return S_OK(varValue)
|
|
95
|
-
|
|
96
|
-
def __checkChoiceVar(self, varName, defaultVal, choices):
|
|
97
|
-
"""
|
|
98
|
-
Check a choice var
|
|
99
|
-
"""
|
|
100
|
-
initialVal = False
|
|
101
|
-
if varName not in self.__manifest:
|
|
102
|
-
varValue = self.__getCSValue(f"Default{varName}", defaultVal)
|
|
103
|
-
else:
|
|
104
|
-
varValue = self.__manifest[varName]
|
|
105
|
-
initialVal = varValue
|
|
106
|
-
if varValue not in self.__getCSValue(f"Choices{varName}", choices):
|
|
107
|
-
return S_ERROR(f"{varValue} is not a valid value for {varName}")
|
|
108
|
-
if initialVal != varValue:
|
|
109
|
-
self.__manifest.setOption(varName, varValue)
|
|
110
|
-
return S_OK(varValue)
|
|
111
|
-
|
|
112
|
-
def __checkMultiChoice(self, varName, choices):
|
|
113
|
-
"""
|
|
114
|
-
Check a multi choice var
|
|
115
|
-
"""
|
|
116
|
-
initialVal = False
|
|
117
|
-
if varName not in self.__manifest:
|
|
118
|
-
return S_OK()
|
|
119
|
-
else:
|
|
120
|
-
varValue = self.__manifest[varName]
|
|
121
|
-
initialVal = varValue
|
|
122
|
-
choices = self.__getCSValue(f"Choices{varName}", choices)
|
|
123
|
-
for v in List.fromChar(varValue):
|
|
124
|
-
if v not in choices:
|
|
125
|
-
return S_ERROR(f"{v} is not a valid value for {varName}")
|
|
126
|
-
if initialVal != varValue:
|
|
127
|
-
self.__manifest.setOption(varName, varValue)
|
|
128
|
-
return S_OK(varValue)
|
|
3
|
+
from DIRACCommon.WorkloadManagementSystem.Client.JobState.JobManifest import * # noqa: F401, F403
|
|
129
4
|
|
|
130
|
-
|
|
131
|
-
"""
|
|
132
|
-
Check Maximum Number of Input Data files allowed
|
|
133
|
-
"""
|
|
134
|
-
varName = "InputData"
|
|
135
|
-
if varName not in self.__manifest:
|
|
136
|
-
return S_OK()
|
|
137
|
-
varValue = self.__manifest[varName]
|
|
138
|
-
if len(List.fromChar(varValue)) > maxNumber:
|
|
139
|
-
return S_ERROR(
|
|
140
|
-
"Number of Input Data Files (%s) greater than current limit: %s"
|
|
141
|
-
% (len(List.fromChar(varValue)), maxNumber)
|
|
142
|
-
)
|
|
143
|
-
return S_OK()
|
|
144
|
-
|
|
145
|
-
def __contains__(self, key):
|
|
146
|
-
"""Check if the manifest has the required key"""
|
|
147
|
-
return key in self.__manifest
|
|
5
|
+
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
|
|
148
6
|
|
|
149
|
-
def setOptionsFromDict(self, varDict):
|
|
150
|
-
for k in sorted(varDict):
|
|
151
|
-
self.setOption(k, varDict[k])
|
|
152
7
|
|
|
8
|
+
def makeJobManifestConfig(ownerGroup: str) -> JobManifestConfig:
|
|
9
|
+
ops = Operations(group=ownerGroup)
|
|
10
|
+
|
|
11
|
+
allowedJobTypesForGroup = ops.getValue(
|
|
12
|
+
"JobDescription/ChoicesJobType",
|
|
13
|
+
ops.getValue("JobDescription/AllowedJobTypes", ["User", "Test", "Hospital"])
|
|
14
|
+
+ ops.getValue("Transformations/DataProcessing", []),
|
|
15
|
+
)
|
|
16
|
+
|
|
17
|
+
return {
|
|
18
|
+
"defaultForGroup": {
|
|
19
|
+
"CPUTime": ops.getValue("JobDescription/DefaultCPUTime", 86400),
|
|
20
|
+
"Priority": ops.getValue("JobDescription/DefaultPriority", 1),
|
|
21
|
+
},
|
|
22
|
+
"minForGroup": {
|
|
23
|
+
"CPUTime": ops.getValue("JobDescription/MinCPUTime", 100),
|
|
24
|
+
"Priority": ops.getValue("JobDescription/MinPriority", 0),
|
|
25
|
+
},
|
|
26
|
+
"maxForGroup": {
|
|
27
|
+
"CPUTime": ops.getValue("JobDescription/MaxCPUTime", 500000),
|
|
28
|
+
"Priority": ops.getValue("JobDescription/MaxPriority", 10),
|
|
29
|
+
},
|
|
30
|
+
"allowedJobTypesForGroup": allowedJobTypesForGroup,
|
|
31
|
+
"maxInputData": Operations().getValue("JobDescription/MaxInputData", 500),
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class JobManifest(JobManifest): # noqa: F405 pylint: disable=function-redefined
|
|
153
36
|
def check(self):
|
|
154
|
-
""
|
|
155
|
-
Check that the manifest is OK
|
|
156
|
-
"""
|
|
157
|
-
for k in ["Owner", "OwnerGroup"]:
|
|
158
|
-
if k not in self.__manifest:
|
|
159
|
-
return S_ERROR(f"Missing var {k} in manifest")
|
|
160
|
-
|
|
161
|
-
# Check CPUTime
|
|
162
|
-
result = self.__checkNumericalVar("CPUTime", 86400, 100, 500000)
|
|
163
|
-
if not result["OK"]:
|
|
164
|
-
return result
|
|
165
|
-
|
|
166
|
-
result = self.__checkNumericalVar("Priority", 1, 0, 10)
|
|
167
|
-
if not result["OK"]:
|
|
168
|
-
return result
|
|
169
|
-
|
|
170
|
-
maxInputData = Operations().getValue("JobDescription/MaxInputData", 500)
|
|
171
|
-
result = self.__checkMaxInputData(maxInputData)
|
|
172
|
-
if not result["OK"]:
|
|
173
|
-
return result
|
|
174
|
-
|
|
175
|
-
operation = Operations(group=self.__manifest["OwnerGroup"])
|
|
176
|
-
allowedJobTypes = operation.getValue("JobDescription/AllowedJobTypes", ["User", "Test", "Hospital"])
|
|
177
|
-
transformationTypes = operation.getValue("Transformations/DataProcessing", [])
|
|
178
|
-
result = self.__checkMultiChoice("JobType", allowedJobTypes + transformationTypes)
|
|
179
|
-
if not result["OK"]:
|
|
180
|
-
return result
|
|
181
|
-
return S_OK()
|
|
182
|
-
|
|
183
|
-
def createSection(self, secName, contents=False):
|
|
184
|
-
if secName not in self.__manifest:
|
|
185
|
-
if contents and not isinstance(contents, CFG):
|
|
186
|
-
return S_ERROR(f"Contents for section {secName} is not a cfg object")
|
|
187
|
-
self.__dirty = True
|
|
188
|
-
return S_OK(self.__manifest.createNewSection(secName, contents=contents))
|
|
189
|
-
return S_ERROR(f"Section {secName} already exists")
|
|
190
|
-
|
|
191
|
-
def getSection(self, secName):
|
|
192
|
-
self.__dirty = True
|
|
193
|
-
if secName not in self.__manifest:
|
|
194
|
-
return S_ERROR(f"{secName} does not exist")
|
|
195
|
-
sec = self.__manifest[secName]
|
|
196
|
-
if not sec:
|
|
197
|
-
return S_ERROR(f"{secName} section empty")
|
|
198
|
-
return S_OK(sec)
|
|
199
|
-
|
|
200
|
-
def setSectionContents(self, secName, contents):
|
|
201
|
-
if contents and not isinstance(contents, CFG):
|
|
202
|
-
return S_ERROR(f"Contents for section {secName} is not a cfg object")
|
|
203
|
-
self.__dirty = True
|
|
204
|
-
if secName in self.__manifest:
|
|
205
|
-
self.__manifest[secName].reset()
|
|
206
|
-
self.__manifest[secName].mergeWith(contents)
|
|
207
|
-
else:
|
|
208
|
-
self.__manifest.createNewSection(secName, contents=contents)
|
|
209
|
-
|
|
210
|
-
def setOption(self, varName, varValue):
|
|
211
|
-
"""
|
|
212
|
-
Set a var in job manifest
|
|
213
|
-
"""
|
|
214
|
-
self.__dirty = True
|
|
215
|
-
levels = List.fromChar(varName, "/")
|
|
216
|
-
cfg = self.__manifest
|
|
217
|
-
for l in levels[:-1]:
|
|
218
|
-
if l not in cfg:
|
|
219
|
-
cfg.createNewSection(l)
|
|
220
|
-
cfg = cfg[l]
|
|
221
|
-
cfg.setOption(levels[-1], varValue)
|
|
222
|
-
|
|
223
|
-
def remove(self, opName):
|
|
224
|
-
levels = List.fromChar(opName, "/")
|
|
225
|
-
cfg = self.__manifest
|
|
226
|
-
for l in levels[:-1]:
|
|
227
|
-
if l not in cfg:
|
|
228
|
-
return S_ERROR(f"{opName} does not exist")
|
|
229
|
-
cfg = cfg[l]
|
|
230
|
-
if cfg.deleteKey(levels[-1]):
|
|
231
|
-
self.__dirty = True
|
|
232
|
-
return S_OK()
|
|
233
|
-
return S_ERROR(f"{opName} does not exist")
|
|
234
|
-
|
|
235
|
-
def getOption(self, varName, defaultValue=None):
|
|
236
|
-
"""
|
|
237
|
-
Get a variable from the job manifest
|
|
238
|
-
"""
|
|
239
|
-
cfg = self.__manifest
|
|
240
|
-
return cfg.getOption(varName, defaultValue)
|
|
241
|
-
|
|
242
|
-
def getOptionList(self, section=""):
|
|
243
|
-
"""
|
|
244
|
-
Get a list of variables in a section of the job manifest
|
|
245
|
-
"""
|
|
246
|
-
cfg = self.__manifest.getRecursive(section)
|
|
247
|
-
if not cfg or "value" not in cfg:
|
|
248
|
-
return []
|
|
249
|
-
cfg = cfg["value"]
|
|
250
|
-
return cfg.listOptions()
|
|
251
|
-
|
|
252
|
-
def isOption(self, opName):
|
|
253
|
-
"""
|
|
254
|
-
Check if it is a valid option
|
|
255
|
-
"""
|
|
256
|
-
return self.__manifest.isOption(opName)
|
|
257
|
-
|
|
258
|
-
def getSectionList(self, section=""):
|
|
259
|
-
"""
|
|
260
|
-
Get a list of sections in the job manifest
|
|
261
|
-
"""
|
|
262
|
-
cfg = self.__manifest.getRecursive(section)
|
|
263
|
-
if not cfg or "value" not in cfg:
|
|
264
|
-
return []
|
|
265
|
-
cfg = cfg["value"]
|
|
266
|
-
return cfg.listSections()
|
|
37
|
+
return super().check(config=makeJobManifestConfig(self.__manifest["OwnerGroup"]))
|