DIRAC 9.0.0a42__py3-none-any.whl → 9.0.7__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (236) hide show
  1. DIRAC/AccountingSystem/Client/AccountingCLI.py +0 -140
  2. DIRAC/AccountingSystem/Client/DataStoreClient.py +0 -13
  3. DIRAC/AccountingSystem/Client/Types/BaseAccountingType.py +0 -7
  4. DIRAC/AccountingSystem/ConfigTemplate.cfg +0 -5
  5. DIRAC/AccountingSystem/Service/DataStoreHandler.py +0 -72
  6. DIRAC/ConfigurationSystem/Client/Helpers/CSGlobals.py +0 -9
  7. DIRAC/ConfigurationSystem/Client/Helpers/Registry.py +38 -26
  8. DIRAC/ConfigurationSystem/Client/Helpers/Resources.py +11 -43
  9. DIRAC/ConfigurationSystem/Client/Helpers/test/Test_Helpers.py +0 -16
  10. DIRAC/ConfigurationSystem/Client/LocalConfiguration.py +14 -8
  11. DIRAC/ConfigurationSystem/Client/PathFinder.py +47 -8
  12. DIRAC/ConfigurationSystem/Client/SyncPlugins/CERNLDAPSyncPlugin.py +4 -1
  13. DIRAC/ConfigurationSystem/Client/VOMS2CSSynchronizer.py +32 -19
  14. DIRAC/ConfigurationSystem/Client/test/Test_PathFinder.py +41 -1
  15. DIRAC/ConfigurationSystem/private/RefresherBase.py +4 -2
  16. DIRAC/Core/Base/API.py +4 -7
  17. DIRAC/Core/Base/SQLAlchemyDB.py +1 -0
  18. DIRAC/Core/DISET/ServiceReactor.py +11 -3
  19. DIRAC/Core/DISET/private/BaseClient.py +1 -2
  20. DIRAC/Core/DISET/private/Transports/M2SSLTransport.py +9 -7
  21. DIRAC/Core/DISET/private/Transports/SSL/M2Utils.py +3 -1
  22. DIRAC/Core/LCG/GOCDBClient.py +5 -7
  23. DIRAC/Core/Security/DiracX.py +31 -17
  24. DIRAC/Core/Security/IAMService.py +5 -10
  25. DIRAC/Core/Security/Locations.py +27 -18
  26. DIRAC/Core/Security/ProxyInfo.py +9 -5
  27. DIRAC/Core/Security/VOMSService.py +2 -4
  28. DIRAC/Core/Security/m2crypto/X509Certificate.py +4 -6
  29. DIRAC/Core/Security/m2crypto/asn1_utils.py +17 -5
  30. DIRAC/Core/Security/test/test_diracx_token_from_pem.py +161 -0
  31. DIRAC/Core/Tornado/Client/ClientSelector.py +4 -1
  32. DIRAC/Core/Tornado/Server/TornadoService.py +1 -1
  33. DIRAC/Core/Utilities/CGroups2.py +328 -0
  34. DIRAC/Core/Utilities/ClassAd/ClassAdLight.py +4 -290
  35. DIRAC/Core/Utilities/DErrno.py +5 -309
  36. DIRAC/Core/Utilities/Extensions.py +10 -1
  37. DIRAC/Core/Utilities/File.py +1 -1
  38. DIRAC/Core/Utilities/Graphs/GraphData.py +1 -1
  39. DIRAC/Core/Utilities/Graphs/GraphUtilities.py +6 -1
  40. DIRAC/Core/Utilities/JDL.py +1 -195
  41. DIRAC/Core/Utilities/List.py +1 -124
  42. DIRAC/Core/Utilities/MySQL.py +103 -99
  43. DIRAC/Core/Utilities/Os.py +32 -1
  44. DIRAC/Core/Utilities/Platform.py +2 -107
  45. DIRAC/Core/Utilities/Proxy.py +0 -4
  46. DIRAC/Core/Utilities/ReturnValues.py +7 -252
  47. DIRAC/Core/Utilities/StateMachine.py +12 -178
  48. DIRAC/Core/Utilities/Subprocess.py +35 -14
  49. DIRAC/Core/Utilities/TimeUtilities.py +10 -253
  50. DIRAC/Core/Utilities/test/Test_JDL.py +0 -3
  51. DIRAC/Core/Utilities/test/Test_Profiler.py +20 -20
  52. DIRAC/Core/scripts/dirac_agent.py +1 -1
  53. DIRAC/Core/scripts/dirac_apptainer_exec.py +72 -46
  54. DIRAC/Core/scripts/dirac_configure.py +1 -3
  55. DIRAC/Core/scripts/dirac_install_db.py +24 -6
  56. DIRAC/Core/scripts/dirac_platform.py +1 -92
  57. DIRAC/DataManagementSystem/Agent/FTS3Agent.py +8 -7
  58. DIRAC/DataManagementSystem/Agent/RequestOperations/RemoveFile.py +7 -6
  59. DIRAC/DataManagementSystem/Client/FTS3Job.py +71 -34
  60. DIRAC/DataManagementSystem/DB/FTS3DB.py +7 -3
  61. DIRAC/DataManagementSystem/DB/FileCatalogComponents/DatasetManager/DatasetManager.py +1 -1
  62. DIRAC/DataManagementSystem/DB/FileCatalogDB.sql +9 -9
  63. DIRAC/DataManagementSystem/DB/FileCatalogWithFkAndPsDB.sql +9 -9
  64. DIRAC/DataManagementSystem/Utilities/DMSHelpers.py +6 -2
  65. DIRAC/DataManagementSystem/scripts/dirac_admin_allow_se.py +13 -8
  66. DIRAC/DataManagementSystem/scripts/dirac_admin_ban_se.py +13 -8
  67. DIRAC/DataManagementSystem/scripts/dirac_dms_create_moving_request.py +2 -0
  68. DIRAC/DataManagementSystem/scripts/dirac_dms_protocol_matrix.py +0 -1
  69. DIRAC/FrameworkSystem/Client/BundleDeliveryClient.py +2 -7
  70. DIRAC/FrameworkSystem/Client/ComponentInstaller.py +9 -4
  71. DIRAC/FrameworkSystem/Client/ProxyManagerClient.py +5 -2
  72. DIRAC/FrameworkSystem/Client/SystemAdministratorClientCLI.py +11 -6
  73. DIRAC/FrameworkSystem/ConfigTemplate.cfg +2 -0
  74. DIRAC/FrameworkSystem/DB/AuthDB.py +3 -3
  75. DIRAC/FrameworkSystem/DB/InstalledComponentsDB.py +4 -4
  76. DIRAC/FrameworkSystem/DB/ProxyDB.py +11 -3
  77. DIRAC/FrameworkSystem/DB/TokenDB.py +1 -1
  78. DIRAC/FrameworkSystem/Service/ProxyManagerHandler.py +8 -6
  79. DIRAC/FrameworkSystem/Utilities/MonitoringUtilities.py +2 -19
  80. DIRAC/FrameworkSystem/Utilities/TokenManagementUtilities.py +3 -2
  81. DIRAC/FrameworkSystem/Utilities/diracx.py +36 -14
  82. DIRAC/FrameworkSystem/private/authorization/AuthServer.py +2 -2
  83. DIRAC/FrameworkSystem/scripts/dirac_admin_update_pilot.py +18 -11
  84. DIRAC/FrameworkSystem/scripts/dirac_login.py +2 -2
  85. DIRAC/FrameworkSystem/scripts/dirac_proxy_init.py +7 -8
  86. DIRAC/Interfaces/API/Dirac.py +27 -15
  87. DIRAC/Interfaces/API/DiracAdmin.py +45 -17
  88. DIRAC/Interfaces/API/Job.py +9 -13
  89. DIRAC/Interfaces/scripts/dirac_admin_allow_site.py +12 -18
  90. DIRAC/Interfaces/scripts/dirac_admin_ban_site.py +12 -10
  91. DIRAC/Interfaces/scripts/dirac_admin_get_site_mask.py +4 -13
  92. DIRAC/Interfaces/scripts/dirac_admin_reset_job.py +3 -6
  93. DIRAC/Interfaces/scripts/dirac_wms_job_parameters.py +0 -1
  94. DIRAC/MonitoringSystem/Client/Types/WMSHistory.py +4 -0
  95. DIRAC/MonitoringSystem/Client/WebAppClient.py +26 -0
  96. DIRAC/MonitoringSystem/ConfigTemplate.cfg +9 -0
  97. DIRAC/MonitoringSystem/DB/MonitoringDB.py +6 -25
  98. DIRAC/MonitoringSystem/Service/MonitoringHandler.py +0 -33
  99. DIRAC/MonitoringSystem/Service/WebAppHandler.py +599 -0
  100. DIRAC/MonitoringSystem/private/MainReporter.py +0 -3
  101. DIRAC/ProductionSystem/DB/ProductionDB.sql +4 -4
  102. DIRAC/ProductionSystem/scripts/dirac_prod_get.py +2 -2
  103. DIRAC/ProductionSystem/scripts/dirac_prod_get_all.py +2 -2
  104. DIRAC/ProductionSystem/scripts/dirac_prod_get_trans.py +2 -3
  105. DIRAC/RequestManagementSystem/Agent/RequestExecutingAgent.py +8 -6
  106. DIRAC/RequestManagementSystem/Agent/RequestOperations/ForwardDISET.py +2 -14
  107. DIRAC/RequestManagementSystem/Client/ReqClient.py +66 -13
  108. DIRAC/RequestManagementSystem/ConfigTemplate.cfg +6 -6
  109. DIRAC/RequestManagementSystem/DB/RequestDB.py +10 -5
  110. DIRAC/RequestManagementSystem/DB/test/RMSTestScenari.py +2 -0
  111. DIRAC/RequestManagementSystem/private/RequestValidator.py +40 -46
  112. DIRAC/ResourceStatusSystem/Client/SiteStatus.py +4 -2
  113. DIRAC/ResourceStatusSystem/Command/FreeDiskSpaceCommand.py +3 -1
  114. DIRAC/ResourceStatusSystem/DB/ResourceManagementDB.py +8 -8
  115. DIRAC/ResourceStatusSystem/DB/ResourceStatusDB.py +2 -2
  116. DIRAC/ResourceStatusSystem/Utilities/CSHelpers.py +2 -31
  117. DIRAC/ResourceStatusSystem/scripts/dirac_rss_set_status.py +30 -12
  118. DIRAC/Resources/Catalog/RucioFileCatalogClient.py +195 -1
  119. DIRAC/Resources/Catalog/test/Test_RucioFileCatalogClient.py +181 -0
  120. DIRAC/Resources/Computing/AREXComputingElement.py +25 -8
  121. DIRAC/Resources/Computing/BatchSystems/Condor.py +126 -108
  122. DIRAC/Resources/Computing/BatchSystems/SLURM.py +5 -1
  123. DIRAC/Resources/Computing/BatchSystems/test/Test_SLURM.py +46 -0
  124. DIRAC/Resources/Computing/ComputingElement.py +1 -1
  125. DIRAC/Resources/Computing/HTCondorCEComputingElement.py +44 -44
  126. DIRAC/Resources/Computing/InProcessComputingElement.py +4 -2
  127. DIRAC/Resources/Computing/LocalComputingElement.py +1 -18
  128. DIRAC/Resources/Computing/SSHBatchComputingElement.py +1 -17
  129. DIRAC/Resources/Computing/SSHComputingElement.py +1 -18
  130. DIRAC/Resources/Computing/SingularityComputingElement.py +19 -5
  131. DIRAC/Resources/Computing/test/Test_HTCondorCEComputingElement.py +67 -49
  132. DIRAC/Resources/Computing/test/Test_PoolComputingElement.py +2 -1
  133. DIRAC/Resources/IdProvider/CheckInIdProvider.py +13 -0
  134. DIRAC/Resources/IdProvider/IdProviderFactory.py +11 -3
  135. DIRAC/Resources/MessageQueue/StompMQConnector.py +1 -1
  136. DIRAC/Resources/Storage/GFAL2_StorageBase.py +24 -15
  137. DIRAC/Resources/Storage/OccupancyPlugins/WLCGAccountingHTTPJson.py +1 -3
  138. DIRAC/Resources/Storage/StorageBase.py +4 -2
  139. DIRAC/Resources/Storage/StorageElement.py +6 -7
  140. DIRAC/StorageManagementSystem/DB/StorageManagementDB.sql +2 -2
  141. DIRAC/TransformationSystem/Agent/TaskManagerAgentBase.py +10 -16
  142. DIRAC/TransformationSystem/Agent/TransformationAgent.py +22 -1
  143. DIRAC/TransformationSystem/Agent/TransformationCleaningAgent.py +16 -16
  144. DIRAC/TransformationSystem/Client/TaskManager.py +2 -4
  145. DIRAC/TransformationSystem/Client/Transformation.py +6 -7
  146. DIRAC/TransformationSystem/Client/TransformationClient.py +21 -11
  147. DIRAC/TransformationSystem/Client/Utilities.py +9 -0
  148. DIRAC/TransformationSystem/DB/TransformationDB.py +11 -14
  149. DIRAC/TransformationSystem/DB/TransformationDB.sql +9 -9
  150. DIRAC/TransformationSystem/Service/TransformationManagerHandler.py +0 -333
  151. DIRAC/TransformationSystem/Utilities/ReplicationCLIParameters.py +3 -3
  152. DIRAC/TransformationSystem/Utilities/TransformationInfo.py +7 -5
  153. DIRAC/TransformationSystem/scripts/dirac_production_runjoblocal.py +2 -4
  154. DIRAC/TransformationSystem/test/Test_TransformationInfo.py +22 -15
  155. DIRAC/TransformationSystem/test/Test_replicationTransformation.py +5 -6
  156. DIRAC/Workflow/Modules/test/Test_Modules.py +5 -0
  157. DIRAC/WorkloadManagementSystem/Agent/JobAgent.py +38 -26
  158. DIRAC/WorkloadManagementSystem/Agent/JobCleaningAgent.py +12 -8
  159. DIRAC/WorkloadManagementSystem/Agent/PilotSyncAgent.py +4 -3
  160. DIRAC/WorkloadManagementSystem/Agent/PushJobAgent.py +13 -13
  161. DIRAC/WorkloadManagementSystem/Agent/SiteDirector.py +18 -14
  162. DIRAC/WorkloadManagementSystem/Agent/StalledJobAgent.py +18 -51
  163. DIRAC/WorkloadManagementSystem/Agent/StatesAccountingAgent.py +41 -1
  164. DIRAC/WorkloadManagementSystem/Agent/test/Test_Agent_JobAgent.py +45 -4
  165. DIRAC/WorkloadManagementSystem/Agent/test/Test_Agent_JobCleaningAgent.py +7 -9
  166. DIRAC/WorkloadManagementSystem/Agent/test/Test_Agent_PushJobAgent.py +1 -0
  167. DIRAC/WorkloadManagementSystem/Agent/test/Test_Agent_SiteDirector.py +9 -2
  168. DIRAC/WorkloadManagementSystem/Agent/test/Test_Agent_StalledJobAgent.py +4 -5
  169. DIRAC/WorkloadManagementSystem/Client/DownloadInputData.py +9 -9
  170. DIRAC/WorkloadManagementSystem/Client/InputDataResolution.py +6 -6
  171. DIRAC/WorkloadManagementSystem/Client/JobMonitoringClient.py +10 -11
  172. DIRAC/WorkloadManagementSystem/Client/JobReport.py +1 -1
  173. DIRAC/WorkloadManagementSystem/Client/JobState/CachedJobState.py +3 -0
  174. DIRAC/WorkloadManagementSystem/Client/JobState/JobManifest.py +32 -261
  175. DIRAC/WorkloadManagementSystem/Client/JobState/JobState.py +6 -0
  176. DIRAC/WorkloadManagementSystem/Client/JobStateUpdateClient.py +3 -0
  177. DIRAC/WorkloadManagementSystem/Client/JobStatus.py +8 -152
  178. DIRAC/WorkloadManagementSystem/Client/PoolXMLSlice.py +12 -19
  179. DIRAC/WorkloadManagementSystem/Client/SandboxStoreClient.py +25 -38
  180. DIRAC/WorkloadManagementSystem/Client/WMSClient.py +2 -3
  181. DIRAC/WorkloadManagementSystem/Client/test/Test_Client_DownloadInputData.py +29 -0
  182. DIRAC/WorkloadManagementSystem/ConfigTemplate.cfg +4 -8
  183. DIRAC/WorkloadManagementSystem/DB/JobDB.py +89 -132
  184. DIRAC/WorkloadManagementSystem/DB/JobDB.sql +8 -8
  185. DIRAC/WorkloadManagementSystem/DB/JobDBUtils.py +18 -147
  186. DIRAC/WorkloadManagementSystem/DB/JobLoggingDB.py +19 -6
  187. DIRAC/WorkloadManagementSystem/DB/JobParametersDB.py +9 -9
  188. DIRAC/WorkloadManagementSystem/DB/PilotAgentsDB.py +16 -5
  189. DIRAC/WorkloadManagementSystem/DB/PilotAgentsDB.sql +3 -3
  190. DIRAC/WorkloadManagementSystem/DB/SandboxMetadataDB.py +44 -82
  191. DIRAC/WorkloadManagementSystem/DB/StatusUtils.py +125 -0
  192. DIRAC/WorkloadManagementSystem/DB/tests/Test_JobDB.py +1 -1
  193. DIRAC/WorkloadManagementSystem/DB/tests/Test_StatusUtils.py +28 -0
  194. DIRAC/WorkloadManagementSystem/Executor/JobSanity.py +5 -4
  195. DIRAC/WorkloadManagementSystem/Executor/JobScheduling.py +4 -0
  196. DIRAC/WorkloadManagementSystem/FutureClient/JobStateUpdateClient.py +75 -33
  197. DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py +22 -11
  198. DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapperTemplate.py +9 -10
  199. DIRAC/WorkloadManagementSystem/JobWrapper/test/Test_JobWrapper.py +60 -10
  200. DIRAC/WorkloadManagementSystem/JobWrapper/test/Test_JobWrapperTemplate.py +4 -0
  201. DIRAC/WorkloadManagementSystem/Service/JobManagerHandler.py +33 -154
  202. DIRAC/WorkloadManagementSystem/Service/JobMonitoringHandler.py +5 -323
  203. DIRAC/WorkloadManagementSystem/Service/JobStateUpdateHandler.py +0 -16
  204. DIRAC/WorkloadManagementSystem/Service/PilotManagerHandler.py +6 -103
  205. DIRAC/WorkloadManagementSystem/Service/SandboxStoreHandler.py +7 -53
  206. DIRAC/WorkloadManagementSystem/Service/WMSAdministratorHandler.py +16 -79
  207. DIRAC/WorkloadManagementSystem/Service/WMSUtilities.py +4 -18
  208. DIRAC/WorkloadManagementSystem/Utilities/JobModel.py +28 -209
  209. DIRAC/WorkloadManagementSystem/Utilities/JobParameters.py +65 -3
  210. DIRAC/WorkloadManagementSystem/Utilities/JobStatusUtility.py +2 -64
  211. DIRAC/WorkloadManagementSystem/Utilities/ParametricJob.py +7 -171
  212. DIRAC/WorkloadManagementSystem/Utilities/PilotCStoJSONSynchronizer.py +73 -7
  213. DIRAC/WorkloadManagementSystem/Utilities/PilotWrapper.py +41 -11
  214. DIRAC/WorkloadManagementSystem/Utilities/RemoteRunner.py +16 -0
  215. DIRAC/WorkloadManagementSystem/Utilities/Utils.py +36 -1
  216. DIRAC/WorkloadManagementSystem/Utilities/jobAdministration.py +15 -0
  217. DIRAC/WorkloadManagementSystem/Utilities/test/Test_JobModel.py +1 -15
  218. DIRAC/WorkloadManagementSystem/Utilities/test/Test_ParametricJob.py +45 -128
  219. DIRAC/WorkloadManagementSystem/Utilities/test/Test_PilotWrapper.py +16 -0
  220. DIRAC/WorkloadManagementSystem/scripts/dirac_jobexec.py +7 -2
  221. DIRAC/WorkloadManagementSystem/scripts/dirac_wms_pilot_job_info.py +1 -1
  222. DIRAC/__init__.py +62 -60
  223. DIRAC/tests/Utilities/testJobDefinitions.py +22 -28
  224. {DIRAC-9.0.0a42.dist-info → dirac-9.0.7.dist-info}/METADATA +8 -5
  225. {DIRAC-9.0.0a42.dist-info → dirac-9.0.7.dist-info}/RECORD +229 -228
  226. {DIRAC-9.0.0a42.dist-info → dirac-9.0.7.dist-info}/WHEEL +1 -1
  227. {DIRAC-9.0.0a42.dist-info → dirac-9.0.7.dist-info}/entry_points.txt +0 -3
  228. DIRAC/Core/Utilities/test/Test_List.py +0 -150
  229. DIRAC/Core/Utilities/test/Test_Time.py +0 -88
  230. DIRAC/Resources/Computing/PilotBundle.py +0 -70
  231. DIRAC/TransformationSystem/scripts/dirac_transformation_archive.py +0 -30
  232. DIRAC/TransformationSystem/scripts/dirac_transformation_clean.py +0 -30
  233. DIRAC/TransformationSystem/scripts/dirac_transformation_remove_output.py +0 -30
  234. DIRAC/WorkloadManagementSystem/Utilities/test/Test_JobManager.py +0 -58
  235. {DIRAC-9.0.0a42.dist-info → dirac-9.0.7.dist-info/licenses}/LICENSE +0 -0
  236. {DIRAC-9.0.0a42.dist-info → dirac-9.0.7.dist-info}/top_level.txt +0 -0
@@ -260,6 +260,7 @@ def test__checkMatcherInfo(mocker, matcherInfo, matcherParams, expectedResult):
260
260
  #############################################################################
261
261
 
262
262
 
263
+ @pytest.mark.slow
263
264
  @pytest.mark.parametrize(
264
265
  "mockGCReply, mockPMReply, expected",
265
266
  [
@@ -308,6 +309,7 @@ def test__setupProxy(mocker, mockGCReply, mockPMReply, expected):
308
309
  assert result["Message"] == expected["Message"]
309
310
 
310
311
 
312
+ @pytest.mark.slow
311
313
  @pytest.mark.parametrize(
312
314
  "mockGCReply, mockPMReply, expected",
313
315
  [
@@ -611,7 +613,7 @@ def test_submitJob(mocker, mockJWInput, expected):
611
613
  ("Pool/Singularity", jobScript % "1", (["Failed to find singularity"], []), ([], [])),
612
614
  ],
613
615
  )
614
- def test_submitAndCheckJob(mocker, manageJobFiles, localCE, job, expectedResult1, expectedResult2):
616
+ def test_submitAndCheckJob(mocker, manageJobFiles, localCE, job, expectedResult1, expectedResult2, tmp_path):
615
617
  """Test the submission and the management of the job status."""
616
618
  jobID = "123"
617
619
  jobExecutablePath, jobWrapperPath, jobWrapperConfigPath = manageJobFiles
@@ -635,7 +637,12 @@ def test_submitAndCheckJob(mocker, manageJobFiles, localCE, job, expectedResult1
635
637
  ),
636
638
  )
637
639
  mocker.patch("DIRAC.WorkloadManagementSystem.Agent.JobAgent.JobAgent._sendFailoverRequest", return_value=S_OK())
638
- mocker.patch("DIRAC.Core.Security.X509Chain.X509Chain.dumpAllToString", return_value=S_OK())
640
+ empty_file_path = tmp_path / "empty_file"
641
+ empty_file_path.touch()
642
+ mocker.patch(
643
+ "DIRAC.WorkloadManagementSystem.Agent.JobAgent.writeChainToTemporaryFile",
644
+ return_value=S_OK(str(empty_file_path)),
645
+ )
639
646
  mocker.patch(
640
647
  "DIRAC.Resources.Computing.SingularityComputingElement.SingularityComputingElement.submitJob",
641
648
  return_value=S_ERROR("Failed to find singularity"),
@@ -712,7 +719,7 @@ def test_submitAndCheckJob(mocker, manageJobFiles, localCE, job, expectedResult1
712
719
  assert len(jobAgent.computingElement.taskResults) == 0
713
720
 
714
721
 
715
- def test_submitAndCheck2Jobs(mocker):
722
+ def test_submitAndCheck2Jobs(mocker, tmp_path):
716
723
  """Test the submission and the management of the job status.
717
724
 
718
725
  This time, a first job is successfully submitted, but the second submission fails.
@@ -728,7 +735,17 @@ def test_submitAndCheck2Jobs(mocker):
728
735
  ),
729
736
  )
730
737
  mocker.patch("DIRAC.WorkloadManagementSystem.Agent.JobAgent.JobAgent._sendFailoverRequest", return_value=S_OK())
731
- mocker.patch("DIRAC.Core.Security.X509Chain.X509Chain.dumpAllToString", return_value=S_OK())
738
+
739
+ def make_empty_file(*args, **kwargs):
740
+ """Create an empty file and return its path."""
741
+ empty_file_path = tmp_path / "empty_file"
742
+ empty_file_path.touch()
743
+ return S_OK(str(empty_file_path))
744
+
745
+ mocker.patch(
746
+ "DIRAC.WorkloadManagementSystem.Agent.JobAgent.writeChainToTemporaryFile",
747
+ make_empty_file,
748
+ )
732
749
  mocker.patch(
733
750
  "DIRAC.Resources.Computing.InProcessComputingElement.InProcessComputingElement.submitJob",
734
751
  side_effect=[S_OK(), S_ERROR("ComputingElement error")],
@@ -788,3 +805,27 @@ def test_submitAndCheck2Jobs(mocker):
788
805
 
789
806
  # From here, taskResults should be empty
790
807
  assert len(jobAgent.computingElement.taskResults) == 0
808
+
809
+
810
+ def test_failureBeforeSubmission(mocker):
811
+ """Test that a failure before job submission is handled correctly.
812
+
813
+ We want to make sure that there is no endless loop in the finalize method.
814
+ """
815
+ # Mock the JobAgent
816
+ mocker.patch("DIRAC.WorkloadManagementSystem.Agent.JobAgent.AgentModule.__init__")
817
+
818
+ jobAgent = JobAgent("JobAgent", "Test")
819
+ jobAgent.log = gLogger.getSubLogger("JobAgent")
820
+
821
+ # Here we simulate a failure before the job submission: no TaskID
822
+ jobID = "123"
823
+ jobAgent.jobs[jobID] = {}
824
+ jobAgent.jobs[jobID]["JobReport"] = JobReport(jobID)
825
+
826
+ # Make sure that the job is removed from jobAgent.jobs
827
+ result = jobAgent._checkSubmittedJobs()
828
+ assert result["OK"]
829
+ assert result["Value"] == ([], [])
830
+
831
+ assert jobAgent.jobs == {}
@@ -1,10 +1,11 @@
1
1
  """ Test class for Job Cleaning Agent
2
2
  """
3
- import pytest
4
3
  from unittest.mock import MagicMock
5
4
 
5
+ import pytest
6
+
6
7
  # DIRAC Components
7
- from DIRAC import gLogger, S_OK
8
+ from DIRAC import S_OK, gLogger
8
9
  from DIRAC.WorkloadManagementSystem.Agent.JobCleaningAgent import JobCleaningAgent
9
10
 
10
11
  gLogger.setLevel("DEBUG")
@@ -32,7 +33,6 @@ def jca(mocker):
32
33
  mocker.patch("DIRAC.WorkloadManagementSystem.Agent.JobCleaningAgent.JobDB.selectJobs", side_effect=mockReply)
33
34
  mocker.patch("DIRAC.WorkloadManagementSystem.Agent.JobCleaningAgent.JobDB.__init__", side_effect=mockNone)
34
35
  mocker.patch("DIRAC.WorkloadManagementSystem.Agent.JobCleaningAgent.ReqClient", return_value=mockNone)
35
- mocker.patch("DIRAC.WorkloadManagementSystem.Agent.JobCleaningAgent.JobMonitoringClient", return_value=mockJMC)
36
36
 
37
37
  jca = JobCleaningAgent()
38
38
  jca.log = gLogger
@@ -98,7 +98,7 @@ def test_deleteJobsByStatus(jca, conditions, mockReplyInput, expected):
98
98
  "inputs, params, expected",
99
99
  [
100
100
  ([], {"OK": True, "Value": {}}, {"OK": True, "Value": {"Failed": {}, "Successful": {}}}),
101
- (["a", "b"], {"OK": True, "Value": {}}, {"OK": True, "Value": {"Failed": {}, "Successful": {}}}),
101
+ (["123", "456"], {"OK": True, "Value": {}}, {"OK": True, "Value": {"Failed": {}, "Successful": {}}}),
102
102
  (
103
103
  [],
104
104
  {"OK": True, "Value": {1: {"OutputSandboxLFN": "/some/lfn/1.txt"}}},
@@ -113,11 +113,11 @@ def test_deleteJobsByStatus(jca, conditions, mockReplyInput, expected):
113
113
  {"OK": True, "Value": {"Failed": {}, "Successful": {1: "/some/lfn/1.txt", 2: "/some/other/lfn/2.txt"}}},
114
114
  ),
115
115
  (
116
- ["a", "b"],
116
+ ["123", "456"],
117
117
  {"OK": True, "Value": {1: {"OutputSandboxLFN": "/some/lfn/1.txt"}}},
118
118
  {"OK": True, "Value": {"Failed": {}, "Successful": {1: "/some/lfn/1.txt"}}},
119
119
  ),
120
- (["a", "b"], {"OK": False}, {"OK": False}),
120
+ (["123", "456"], {"OK": False}, {"OK": False}),
121
121
  ],
122
122
  )
123
123
  def test_deleteJobOversizedSandbox(mocker, inputs, params, expected):
@@ -127,10 +127,10 @@ def test_deleteJobOversizedSandbox(mocker, inputs, params, expected):
127
127
  mocker.patch("DIRAC.WorkloadManagementSystem.Agent.JobCleaningAgent.AgentModule.am_getOption", return_value=mockAM)
128
128
  mocker.patch("DIRAC.WorkloadManagementSystem.Agent.JobCleaningAgent.JobDB", return_value=mockNone)
129
129
  mocker.patch("DIRAC.WorkloadManagementSystem.Agent.JobCleaningAgent.ReqClient", return_value=mockNone)
130
- mocker.patch("DIRAC.WorkloadManagementSystem.Agent.JobCleaningAgent.JobMonitoringClient", return_value=mockJMC)
131
130
  mocker.patch(
132
131
  "DIRAC.WorkloadManagementSystem.Agent.JobCleaningAgent.getDNForUsername", return_value=S_OK(["/bih/boh/DN"])
133
132
  )
133
+ mocker.patch("DIRAC.WorkloadManagementSystem.Agent.JobCleaningAgent.getJobParameters", return_value=params)
134
134
 
135
135
  jobCleaningAgent = JobCleaningAgent()
136
136
  jobCleaningAgent.log = gLogger
@@ -138,8 +138,6 @@ def test_deleteJobOversizedSandbox(mocker, inputs, params, expected):
138
138
  jobCleaningAgent._AgentModule__configDefaults = mockAM
139
139
  jobCleaningAgent.initialize()
140
140
 
141
- mockJMC.getJobParameters.return_value = params
142
-
143
141
  result = jobCleaningAgent.deleteJobOversizedSandbox(inputs)
144
142
 
145
143
  assert result == expected
@@ -178,6 +178,7 @@ def jobID():
178
178
  shutil.rmtree(jobID)
179
179
 
180
180
 
181
+ @pytest.mark.slow
181
182
  def test_submitJobWrapper(mocker, jobID):
182
183
  """Test JobAgent._submitJobWrapper()"""
183
184
  mocker.patch("DIRAC.WorkloadManagementSystem.Agent.JobAgent.AgentModule.__init__")
@@ -169,10 +169,16 @@ def sd(mocker, config):
169
169
  gConfig.getSections("Resources/Sites/LCG")["Value"] + gConfig.getSections("Resources/Sites/DIRAC")["Value"]
170
170
  )
171
171
  mocker.patch(
172
- "DIRAC.WorkloadManagementSystem.Agent.SiteDirector.SiteStatus.getUsableSites", return_values=usableSites
172
+ "DIRAC.WorkloadManagementSystem.Agent.SiteDirector.SiteStatus.getUsableSites", return_value=S_OK(usableSites)
173
173
  )
174
+
175
+ # Mock getElementStatus to return a properly formatted dictionary
176
+ def mock_getElementStatus(ceNamesList, *args, **kwargs):
177
+ return S_OK({ceName: {"all": "Active"} for ceName in ceNamesList})
178
+
174
179
  mocker.patch(
175
- "DIRAC.WorkloadManagementSystem.Agent.SiteDirector.ResourceStatus.getElementStatus", return_values=usableSites
180
+ "DIRAC.WorkloadManagementSystem.Agent.SiteDirector.ResourceStatus.getElementStatus",
181
+ side_effect=mock_getElementStatus,
176
182
  )
177
183
  mocker.patch(
178
184
  "DIRAC.WorkloadManagementSystem.Agent.SiteDirector.gProxyManager.downloadProxy", side_effect=mockPMProxyReply
@@ -261,6 +267,7 @@ def test_getPilotWrapper(mocker, sd, pilotWrapperDirectory):
261
267
  # Get pilot options
262
268
  pilotOptions = sd._getPilotOptions("ce1.site1.com_condor")
263
269
  assert {
270
+ "--architectureScript=123",
264
271
  "--preinstalledEnv=123",
265
272
  "--wnVO=dteam",
266
273
  "-n LCG.Site1.com",
@@ -25,11 +25,10 @@ def sja(mocker):
25
25
  )
26
26
  mocker.patch("DIRAC.WorkloadManagementSystem.Agent.StalledJobAgent.JobDB")
27
27
  mocker.patch("DIRAC.WorkloadManagementSystem.Agent.StalledJobAgent.JobLoggingDB")
28
- mocker.patch("DIRAC.WorkloadManagementSystem.Agent.StalledJobAgent.JobMonitoringClient", return_value=MagicMock())
29
- mocker.patch("DIRAC.WorkloadManagementSystem.Agent.StalledJobAgent.JobManagerClient", return_value=MagicMock())
30
- mocker.patch("DIRAC.WorkloadManagementSystem.Agent.StalledJobAgent.PilotManagerClient", return_value=MagicMock())
31
- mocker.patch("DIRAC.WorkloadManagementSystem.Agent.StalledJobAgent.WMSClient", return_value=MagicMock())
32
- mocker.patch("DIRAC.WorkloadManagementSystem.Agent.StalledJobAgent.getDNForUsername", return_value=MagicMock())
28
+ mocker.patch("DIRAC.WorkloadManagementSystem.Agent.StalledJobAgent.rescheduleJobs", return_value=MagicMock())
29
+ mocker.patch("DIRAC.WorkloadManagementSystem.Agent.StalledJobAgent.PilotAgentsDB", return_value=MagicMock())
30
+ mocker.patch("DIRAC.WorkloadManagementSystem.Agent.StalledJobAgent.getJobParameters", return_value=MagicMock())
31
+ mocker.patch("DIRAC.WorkloadManagementSystem.Agent.StalledJobAgent.kill_delete_jobs", return_value=MagicMock())
33
32
 
34
33
  stalledJobAgent = StalledJobAgent()
35
34
  stalledJobAgent._AgentModule__configDefaults = mockAM
@@ -1,6 +1,7 @@
1
- """ The Download Input Data module wraps around the Replica Management
2
- components to provide access to datasets by downloading locally
1
+ """The Download Input Data module wraps around the Replica Management
2
+ components to provide access to datasets by downloading locally
3
3
  """
4
+
4
5
  import os
5
6
  import random
6
7
  import tempfile
@@ -12,8 +13,6 @@ from DIRAC.DataManagementSystem.Utilities.DMSHelpers import DMSHelpers
12
13
  from DIRAC.Resources.Storage.StorageElement import StorageElement
13
14
  from DIRAC.WorkloadManagementSystem.Client.JobStateUpdateClient import JobStateUpdateClient
14
15
 
15
- COMPONENT_NAME = "DownloadInputData"
16
-
17
16
 
18
17
  def _isCached(lfn, seName):
19
18
  result = returnSingleResult(StorageElement(seName).getFileMetadata(lfn))
@@ -137,7 +136,7 @@ class DownloadInputData:
137
136
  if not result["Value"]:
138
137
  self.log.warn("Not enough disk space available for download", f"{result['Value']} / {totalSize} bytes")
139
138
  self.__setJobParam(
140
- COMPONENT_NAME,
139
+ self.__class__.__name__,
141
140
  f"Not enough disk space available for download: {result['Value']} / {totalSize} bytes",
142
141
  )
143
142
  return S_OK({"Failed": self.inputData, "Successful": {}})
@@ -216,7 +215,7 @@ class DownloadInputData:
216
215
  report += "\n".join(failedReplicas)
217
216
 
218
217
  if report:
219
- self.__setJobParam(COMPONENT_NAME, report)
218
+ self.__setJobParam(self.__class__.__name__, report)
220
219
 
221
220
  return S_OK({"Successful": resolvedData, "Failed": failedReplicas})
222
221
 
@@ -228,7 +227,7 @@ class DownloadInputData:
228
227
  diskSpace = getDiskSpace(self.__getDownloadDir(False)) # MB
229
228
  availableBytes = diskSpace * 1024 * 1024 # bytes
230
229
  bufferGBs = gConfig.getValue(
231
- os.path.join("/Systems/WorkloadManagement/JobWrapper", "JobWrapper", "MinOutputDataBufferGB"), 5.0
230
+ os.path.join("/Systems/WorkloadManagement/JobWrapper", "MinOutputDataBufferGB"), 5.0
232
231
  )
233
232
  data = bufferGBs * 1024 * 1024 * 1024 # bufferGBs in bytes
234
233
  if (data + totalSize) < availableBytes:
@@ -245,12 +244,13 @@ class DownloadInputData:
245
244
  return S_ERROR(msg)
246
245
 
247
246
  def __getDownloadDir(self, incrementCounter=True):
247
+ jobIDPath = str(self.configuration.get("JobIDPath", os.getcwd()))
248
248
  if self.inputDataDirectory == "PerFile":
249
249
  if incrementCounter:
250
250
  self.counter += 1
251
- return tempfile.mkdtemp(prefix=f"InputData_{self.counter}", dir=os.getcwd())
251
+ return tempfile.mkdtemp(prefix=f"InputData_{self.counter}", dir=jobIDPath)
252
252
  elif self.inputDataDirectory == "CWD":
253
- return os.getcwd()
253
+ return jobIDPath
254
254
  else:
255
255
  return self.inputDataDirectory
256
256
 
@@ -6,13 +6,14 @@
6
6
  result and in principle has all the necessary information to resolve input data
7
7
  for applications.
8
8
  """
9
+ from pathlib import Path
10
+
9
11
  import DIRAC
10
- from DIRAC import S_OK, S_ERROR, gLogger, gConfig
12
+ from DIRAC import S_ERROR, S_OK, gConfig, gLogger
13
+ from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
11
14
  from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader
12
15
  from DIRAC.WorkloadManagementSystem.Client.PoolXMLSlice import PoolXMLSlice
13
- from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
14
16
 
15
- COMPONENT_NAME = "InputDataResolution"
16
17
  CREATE_CATALOG = False
17
18
 
18
19
 
@@ -22,8 +23,7 @@ class InputDataResolution:
22
23
  def __init__(self, argumentsDict):
23
24
  """Standard constructor"""
24
25
  self.arguments = argumentsDict
25
- self.name = COMPONENT_NAME
26
- self.log = gLogger.getSubLogger(self.name)
26
+ self.log = gLogger.getSubLogger(self.__class__.__name__)
27
27
  op = Operations()
28
28
  self.arguments.setdefault("Configuration", {})["AllReplicas"] = op.getValue(
29
29
  "InputDataPolicy/AllReplicas", False
@@ -78,7 +78,7 @@ class InputDataResolution:
78
78
  self.log.verbose(f"Catalog name will be: {catalogName}")
79
79
 
80
80
  resolvedData = tmpDict
81
- appCatalog = PoolXMLSlice(catalogName)
81
+ appCatalog = PoolXMLSlice(catalogName, Path(self.arguments["Configuration"].get("JobIDPath", "")))
82
82
  return appCatalog.execute(resolvedData)
83
83
 
84
84
  def __resolveInputData(self):
@@ -14,12 +14,14 @@ except ImportError:
14
14
 
15
15
  @createClient("WorkloadManagement/JobMonitoring")
16
16
  class JobMonitoringClient(Client):
17
+ # Set to None to raise an error if this service is set as "legacy adapted"
18
+ # See ClientSelector
19
+ diracxClient = None
20
+
17
21
  def __init__(self, **kwargs):
18
22
  super().__init__(**kwargs)
19
23
  self.setServer("WorkloadManagement/JobMonitoring")
20
24
 
21
- diracxClient = futureJobMonitoringClient
22
-
23
25
  @ignoreEncodeWarning
24
26
  def getJobsStatus(self, jobIDs):
25
27
  res = self._getRPC().getJobsStatus(jobIDs)
@@ -38,15 +40,6 @@ class JobMonitoringClient(Client):
38
40
  res["Value"] = strToIntDict(res["Value"])
39
41
  return res
40
42
 
41
- @ignoreEncodeWarning
42
- def getJobsParameters(self, jobIDs, parameters):
43
- res = self._getRPC().getJobsParameters(jobIDs, parameters)
44
-
45
- # Cast the str keys to int
46
- if res["OK"]:
47
- res["Value"] = strToIntDict(res["Value"])
48
- return res
49
-
50
43
  @ignoreEncodeWarning
51
44
  def getJobsMinorStatus(self, jobIDs):
52
45
  res = self._getRPC().getJobsMinorStatus(jobIDs)
@@ -79,3 +72,9 @@ class JobMonitoringClient(Client):
79
72
  if res["OK"]:
80
73
  res["Value"] = strToIntDict(res["Value"])
81
74
  return res
75
+
76
+ def getInputData(self, jobIDs):
77
+ res = self._getRPC().getInputData(jobIDs)
78
+ if res["OK"] and isinstance(res["Value"], dict):
79
+ res["Value"] = strToIntDict(res["Value"])
80
+ return res
@@ -4,7 +4,7 @@
4
4
  import datetime
5
5
  from collections import defaultdict
6
6
 
7
- from DIRAC import S_OK, S_ERROR, gLogger
7
+ from DIRAC import S_ERROR, S_OK, gLogger
8
8
  from DIRAC.Core.Utilities import DEncode
9
9
  from DIRAC.RequestManagementSystem.Client.Operation import Operation
10
10
  from DIRAC.WorkloadManagementSystem.Client.JobStateUpdateClient import JobStateUpdateClient
@@ -364,6 +364,9 @@ class CachedJobState:
364
364
  def getInputData(self):
365
365
  return self.__cacheResult("inputData", self.__jobState.getInputData)
366
366
 
367
+ def setInputData(self, inputData):
368
+ return self.__jobState.setInputData(inputData)
369
+
367
370
  def insertIntoTQ(self):
368
371
  if self.valid:
369
372
  self.__insertIntoTQ = True
@@ -1,266 +1,37 @@
1
- from diraccfg import CFG
1
+ from __future__ import annotations
2
2
 
3
- from DIRAC import S_OK, S_ERROR
4
- from DIRAC.Core.Utilities import List
5
- from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
6
- from DIRAC.Core.Utilities.JDL import loadJDLAsCFG, dumpCFGAsJDL
7
-
8
-
9
- class JobManifest:
10
- def __init__(self, manifest=""):
11
- self.__manifest = CFG()
12
- self.__dirty = False
13
- self.__ops = False
14
- if manifest:
15
- result = self.load(manifest)
16
- if not result["OK"]:
17
- raise Exception(result["Message"])
18
-
19
- def isDirty(self):
20
- return self.__dirty
21
-
22
- def setDirty(self):
23
- self.__dirty = True
24
-
25
- def clearDirty(self):
26
- self.__dirty = False
27
-
28
- def load(self, dataString):
29
- """
30
- Auto discover format type based on [ .. ] of JDL
31
- """
32
- dataString = dataString.strip()
33
- if dataString[0] == "[" and dataString[-1] == "]":
34
- return self.loadJDL(dataString)
35
- else:
36
- return self.loadCFG(dataString)
37
-
38
- def loadJDL(self, jdlString):
39
- """
40
- Load job manifest from JDL format
41
- """
42
- result = loadJDLAsCFG(jdlString.strip())
43
- if not result["OK"]:
44
- self.__manifest = CFG()
45
- return result
46
- self.__manifest = result["Value"][0]
47
- return S_OK()
48
-
49
- def loadCFG(self, cfgString):
50
- """
51
- Load job manifest from CFG format
52
- """
53
- try:
54
- self.__manifest.loadFromBuffer(cfgString)
55
- except Exception as e:
56
- return S_ERROR(f"Can't load manifest from cfg: {str(e)}")
57
- return S_OK()
58
-
59
- def dumpAsCFG(self):
60
- return str(self.__manifest)
61
-
62
- def getAsCFG(self):
63
- return self.__manifest.clone()
64
-
65
- def dumpAsJDL(self):
66
- return dumpCFGAsJDL(self.__manifest)
67
-
68
- def __getCSValue(self, varName, defaultVal=None):
69
- if not self.__ops:
70
- self.__ops = Operations(group=self.__manifest["OwnerGroup"])
71
- if varName[0] != "/":
72
- varName = f"JobDescription/{varName}"
73
- return self.__ops.getValue(varName, defaultVal)
74
-
75
- def __checkNumericalVar(self, varName, defaultVal, minVal, maxVal):
76
- """
77
- Check a numerical var
78
- """
79
- initialVal = False
80
- if varName not in self.__manifest:
81
- varValue = self.__getCSValue(f"Default{varName}", defaultVal)
82
- else:
83
- varValue = self.__manifest[varName]
84
- initialVal = varValue
85
- try:
86
- varValue = int(varValue)
87
- except ValueError:
88
- return S_ERROR(f"{varName} must be a number")
89
- minVal = self.__getCSValue(f"Min{varName}", minVal)
90
- maxVal = self.__getCSValue(f"Max{varName}", maxVal)
91
- varValue = max(minVal, min(varValue, maxVal))
92
- if initialVal != varValue:
93
- self.__manifest.setOption(varName, varValue)
94
- return S_OK(varValue)
95
-
96
- def __checkChoiceVar(self, varName, defaultVal, choices):
97
- """
98
- Check a choice var
99
- """
100
- initialVal = False
101
- if varName not in self.__manifest:
102
- varValue = self.__getCSValue(f"Default{varName}", defaultVal)
103
- else:
104
- varValue = self.__manifest[varName]
105
- initialVal = varValue
106
- if varValue not in self.__getCSValue(f"Choices{varName}", choices):
107
- return S_ERROR(f"{varValue} is not a valid value for {varName}")
108
- if initialVal != varValue:
109
- self.__manifest.setOption(varName, varValue)
110
- return S_OK(varValue)
111
-
112
- def __checkMultiChoice(self, varName, choices):
113
- """
114
- Check a multi choice var
115
- """
116
- initialVal = False
117
- if varName not in self.__manifest:
118
- return S_OK()
119
- else:
120
- varValue = self.__manifest[varName]
121
- initialVal = varValue
122
- choices = self.__getCSValue(f"Choices{varName}", choices)
123
- for v in List.fromChar(varValue):
124
- if v not in choices:
125
- return S_ERROR(f"{v} is not a valid value for {varName}")
126
- if initialVal != varValue:
127
- self.__manifest.setOption(varName, varValue)
128
- return S_OK(varValue)
3
+ from DIRACCommon.WorkloadManagementSystem.Client.JobState.JobManifest import * # noqa: F401, F403
129
4
 
130
- def __checkMaxInputData(self, maxNumber):
131
- """
132
- Check Maximum Number of Input Data files allowed
133
- """
134
- varName = "InputData"
135
- if varName not in self.__manifest:
136
- return S_OK()
137
- varValue = self.__manifest[varName]
138
- if len(List.fromChar(varValue)) > maxNumber:
139
- return S_ERROR(
140
- "Number of Input Data Files (%s) greater than current limit: %s"
141
- % (len(List.fromChar(varValue)), maxNumber)
142
- )
143
- return S_OK()
144
-
145
- def __contains__(self, key):
146
- """Check if the manifest has the required key"""
147
- return key in self.__manifest
5
+ from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
148
6
 
149
- def setOptionsFromDict(self, varDict):
150
- for k in sorted(varDict):
151
- self.setOption(k, varDict[k])
152
7
 
8
+ def makeJobManifestConfig(ownerGroup: str) -> JobManifestConfig:
9
+ ops = Operations(group=ownerGroup)
10
+
11
+ allowedJobTypesForGroup = ops.getValue(
12
+ "JobDescription/ChoicesJobType",
13
+ ops.getValue("JobDescription/AllowedJobTypes", ["User", "Test", "Hospital"])
14
+ + ops.getValue("Transformations/DataProcessing", []),
15
+ )
16
+
17
+ return {
18
+ "defaultForGroup": {
19
+ "CPUTime": ops.getValue("JobDescription/DefaultCPUTime", 86400),
20
+ "Priority": ops.getValue("JobDescription/DefaultPriority", 1),
21
+ },
22
+ "minForGroup": {
23
+ "CPUTime": ops.getValue("JobDescription/MinCPUTime", 100),
24
+ "Priority": ops.getValue("JobDescription/MinPriority", 0),
25
+ },
26
+ "maxForGroup": {
27
+ "CPUTime": ops.getValue("JobDescription/MaxCPUTime", 500000),
28
+ "Priority": ops.getValue("JobDescription/MaxPriority", 10),
29
+ },
30
+ "allowedJobTypesForGroup": allowedJobTypesForGroup,
31
+ "maxInputData": Operations().getValue("JobDescription/MaxInputData", 500),
32
+ }
33
+
34
+
35
+ class JobManifest(JobManifest): # noqa: F405 pylint: disable=function-redefined
153
36
  def check(self):
154
- """
155
- Check that the manifest is OK
156
- """
157
- for k in ["Owner", "OwnerGroup"]:
158
- if k not in self.__manifest:
159
- return S_ERROR(f"Missing var {k} in manifest")
160
-
161
- # Check CPUTime
162
- result = self.__checkNumericalVar("CPUTime", 86400, 100, 500000)
163
- if not result["OK"]:
164
- return result
165
-
166
- result = self.__checkNumericalVar("Priority", 1, 0, 10)
167
- if not result["OK"]:
168
- return result
169
-
170
- maxInputData = Operations().getValue("JobDescription/MaxInputData", 500)
171
- result = self.__checkMaxInputData(maxInputData)
172
- if not result["OK"]:
173
- return result
174
-
175
- operation = Operations(group=self.__manifest["OwnerGroup"])
176
- allowedJobTypes = operation.getValue("JobDescription/AllowedJobTypes", ["User", "Test", "Hospital"])
177
- transformationTypes = operation.getValue("Transformations/DataProcessing", [])
178
- result = self.__checkMultiChoice("JobType", allowedJobTypes + transformationTypes)
179
- if not result["OK"]:
180
- return result
181
- return S_OK()
182
-
183
- def createSection(self, secName, contents=False):
184
- if secName not in self.__manifest:
185
- if contents and not isinstance(contents, CFG):
186
- return S_ERROR(f"Contents for section {secName} is not a cfg object")
187
- self.__dirty = True
188
- return S_OK(self.__manifest.createNewSection(secName, contents=contents))
189
- return S_ERROR(f"Section {secName} already exists")
190
-
191
- def getSection(self, secName):
192
- self.__dirty = True
193
- if secName not in self.__manifest:
194
- return S_ERROR(f"{secName} does not exist")
195
- sec = self.__manifest[secName]
196
- if not sec:
197
- return S_ERROR(f"{secName} section empty")
198
- return S_OK(sec)
199
-
200
- def setSectionContents(self, secName, contents):
201
- if contents and not isinstance(contents, CFG):
202
- return S_ERROR(f"Contents for section {secName} is not a cfg object")
203
- self.__dirty = True
204
- if secName in self.__manifest:
205
- self.__manifest[secName].reset()
206
- self.__manifest[secName].mergeWith(contents)
207
- else:
208
- self.__manifest.createNewSection(secName, contents=contents)
209
-
210
- def setOption(self, varName, varValue):
211
- """
212
- Set a var in job manifest
213
- """
214
- self.__dirty = True
215
- levels = List.fromChar(varName, "/")
216
- cfg = self.__manifest
217
- for l in levels[:-1]:
218
- if l not in cfg:
219
- cfg.createNewSection(l)
220
- cfg = cfg[l]
221
- cfg.setOption(levels[-1], varValue)
222
-
223
- def remove(self, opName):
224
- levels = List.fromChar(opName, "/")
225
- cfg = self.__manifest
226
- for l in levels[:-1]:
227
- if l not in cfg:
228
- return S_ERROR(f"{opName} does not exist")
229
- cfg = cfg[l]
230
- if cfg.deleteKey(levels[-1]):
231
- self.__dirty = True
232
- return S_OK()
233
- return S_ERROR(f"{opName} does not exist")
234
-
235
- def getOption(self, varName, defaultValue=None):
236
- """
237
- Get a variable from the job manifest
238
- """
239
- cfg = self.__manifest
240
- return cfg.getOption(varName, defaultValue)
241
-
242
- def getOptionList(self, section=""):
243
- """
244
- Get a list of variables in a section of the job manifest
245
- """
246
- cfg = self.__manifest.getRecursive(section)
247
- if not cfg or "value" not in cfg:
248
- return []
249
- cfg = cfg["value"]
250
- return cfg.listOptions()
251
-
252
- def isOption(self, opName):
253
- """
254
- Check if it is a valid option
255
- """
256
- return self.__manifest.isOption(opName)
257
-
258
- def getSectionList(self, section=""):
259
- """
260
- Get a list of sections in the job manifest
261
- """
262
- cfg = self.__manifest.getRecursive(section)
263
- if not cfg or "value" not in cfg:
264
- return []
265
- cfg = cfg["value"]
266
- return cfg.listSections()
37
+ return super().check(config=makeJobManifestConfig(self.__manifest["OwnerGroup"]))