DIRAC 9.0.9__py3-none-any.whl → 9.0.11__py3-none-any.whl

DIRAC/Core/Utilities/ElasticSearchDB.py

@@ -82,7 +82,6 @@ def generateDocs(data, withTimeStamp=True):
 
 
 class ElasticSearchDB:
-
     """
     .. class:: ElasticSearchDB
 
@@ -506,7 +505,7 @@ class ElasticSearchDB:
             indexName = self.generateFullIndexName(indexPrefix, period)
         else:
             indexName = indexPrefix
-        sLog.debug(f"Bulk indexing into {indexName} of {data}")
+        sLog.debug(f"Bulk indexing into {indexName} of {len(data)}")
 
         res = self.existingIndex(indexName)
         if not res["OK"]:

DIRAC/DataManagementSystem/Client/DataManager.py

@@ -10,11 +10,11 @@ This module consists of DataManager and related classes.
 """
 
 # # imports
-from datetime import datetime, timedelta
+import errno
 import fnmatch
 import os
 import time
-import errno
+from datetime import datetime, timedelta
 
 # # from DIRAC
 import DIRAC
@@ -25,13 +25,13 @@ from DIRAC.Core.Utilities.File import makeGuid, getSize
 from DIRAC.Core.Utilities.List import randomize, breakListIntoChunks
 from DIRAC.Core.Utilities.ReturnValues import returnSingleResult
 from DIRAC.Core.Security.ProxyInfo import getProxyInfo
+from DIRAC.Core.Security.ProxyInfo import getVOfromProxyGroup
 from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
 from DIRAC.DataManagementSystem.Client import MAX_FILENAME_LENGTH
-from DIRAC.MonitoringSystem.Client.DataOperationSender import DataOperationSender
 from DIRAC.DataManagementSystem.Utilities.DMSHelpers import DMSHelpers
+from DIRAC.MonitoringSystem.Client.DataOperationSender import DataOperationSender
 from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
 from DIRAC.Resources.Storage.StorageElement import StorageElement
-from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus
 
 
 # # RSCID
@@ -89,7 +89,7 @@ class DataManager:
         :param vo: the VO for which the DataManager is created, get VO from the current proxy if not specified
         """
         self.log = gLogger.getSubLogger(self.__class__.__name__)
-        self.voName = vo
+        self.voName = vo if vo else getVOfromProxyGroup().get("Value", None)
 
         if catalogs is None:
             catalogs = []
@@ -97,10 +97,9 @@ class DataManager:
 
         self.fileCatalog = FileCatalog(catalogs=catalogsToUse, vo=self.voName)
         self.accountingClient = None
-        self.resourceStatus = ResourceStatus()
         self.ignoreMissingInFC = Operations(vo=self.voName).getValue("DataManagement/IgnoreMissingInFC", False)
         self.useCatalogPFN = Operations(vo=self.voName).getValue("DataManagement/UseCatalogPFN", True)
-        self.dmsHelper = DMSHelpers(vo=vo)
+        self.dmsHelper = DMSHelpers(vo=self.voName)
         self.registrationProtocol = self.dmsHelper.getRegistrationProtocols()
         self.thirdPartyProtocols = self.dmsHelper.getThirdPartyProtocols()
         self.dataOpSender = DataOperationSender()

DIRAC/FrameworkSystem/DB/InstalledComponentsDB.py

@@ -90,7 +90,7 @@ class Host(componentsBase):
     __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8mb4"}
 
     hostID = Column("HostID", Integer, primary_key=True)
-    hostName = Column("HostName", String(32), nullable=False)
+    hostName = Column("HostName", String(64), nullable=False)
     cpu = Column("CPU", String(64), nullable=False)
     installationList = relationship("InstalledComponent", backref="installationHost")
 

DIRAC/FrameworkSystem/Utilities/MonitoringUtilities.py

@@ -30,6 +30,7 @@ def monitorInstallation(componentType, system, component, module=None, cpu=None,
 
     if not hostname:
         hostname = socket.getfqdn()
+        hostname = hostname[0:64]
     instance = component[0:32]
 
     result = monitoringClient.installationExists(

DIRAC/Interfaces/Utilities/DConfigCache.py

@@ -9,6 +9,7 @@ from DIRAC import gLogger
 from DIRAC.Core.Base.Script import Script
 from DIRAC.Core.Utilities.File import secureOpenForWrite
 from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData
+from DIRAC.ConfigurationSystem.Client.Helpers.Registry import reset_all_caches
 
 
 class ConfigCache:
@@ -69,5 +70,6 @@ class ConfigCache:
             try:
                 with open(self.configCacheName, "rb") as fh:
                     gConfigurationData.mergedCFG = pickle.load(fh)
+                    reset_all_caches()
             except:
                 gLogger.error("Cache corrupt or unreadable")

DIRAC/Resources/Computing/BatchSystems/Condor.py

@@ -203,9 +203,6 @@ class Condor(object):
             resultDict["Jobs"] = []
             for i in range(submittedJobs):
                 resultDict["Jobs"].append(".".join([cluster, str(i)]))
-            # Executable is transferred afterward
-            # Inform the caller that Condor cannot delete it before the end of the execution
-            resultDict["ExecutableToKeep"] = executable
         else:
             resultDict["Status"] = status
             resultDict["Message"] = error

DIRAC/Resources/Computing/BatchSystems/executeBatch.py

@@ -35,8 +35,10 @@ if __name__ == "__main__":
       from urllib.parse import unquote as urlunquote
 
 
-  arguments = sys.argv[1]
-  inputDict = json.loads(urlunquote(arguments))
+  # Read options from JSON file
+  optionsFilePath = sys.argv[1]
+  with open(optionsFilePath, 'r') as f:
+    inputDict = json.load(f)
 
   method = inputDict.pop('Method')
   batchSystem = inputDict.pop('BatchSystem')
@@ -45,9 +47,15 @@ if __name__ == "__main__":
   try:
     result = getattr(batch, method)(**inputDict)
   except Exception:
-    result = traceback.format_exc()
-
-  resultJson = urlquote(json.dumps(result))
-  print("============= Start output ===============")
-  print(resultJson)
+    # Wrap the traceback in a proper error structure
+    result = {
+      'Status': -1,
+      'Message': 'Exception during batch method execution',
+      'Traceback': traceback.format_exc()
+    }
+
+  # Write result to JSON file
+  resultFilePath = optionsFilePath.replace('.json', '_result.json')
+  with open(resultFilePath, 'w') as f:
+    json.dump(result, f)
 """

DIRAC/Resources/Computing/LocalComputingElement.py

@@ -182,8 +182,6 @@ class LocalComputingElement(ComputingElement):
             batchSystemName = self.batchSystem.__class__.__name__.lower()
             jobIDs = ["ssh" + batchSystemName + "://" + self.ceName + "/" + _id for _id in resultSubmit["Jobs"]]
             result = S_OK(jobIDs)
-            if "ExecutableToKeep" in resultSubmit:
-                result["ExecutableToKeep"] = resultSubmit["ExecutableToKeep"]
         else:
             result = S_ERROR(resultSubmit["Message"])
 

DIRAC/Resources/Computing/SSHComputingElement.py

@@ -67,9 +67,10 @@ import json
 import os
 import shutil
 import stat
+import tempfile
 import uuid
 from shlex import quote as shlex_quote
-from urllib.parse import quote, unquote, urlparse
+from urllib.parse import urlparse
 
 import pexpect
 
@@ -484,47 +485,69 @@ class SSHComputingElement(ComputingElement):
         options["User"] = self.user
         options["Queue"] = self.queue
 
-        options = json.dumps(options)
-        options = quote(options)
+        localOptionsFile = None
+        remoteOptionsFile = None
+        localResultFile = None
+        remoteResultFile = None
+        try:
+            # Write options to a local temporary file
+            with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f:
+                json.dump(options, f)
+                localOptionsFile = f.name
+
+            # Upload the options file to the remote host
+            remoteOptionsFile = f"{self.sharedArea}/batch_options_{uuid.uuid4().hex}.json"
+            result = ssh.scpCall(30, localOptionsFile, remoteOptionsFile)
+            if not result["OK"]:
+                return result
 
-        cmd = (
-            "bash --login -c 'python3 %s/execute_batch %s || python %s/execute_batch %s || python2 %s/execute_batch %s'"
-            % (self.sharedArea, options, self.sharedArea, options, self.sharedArea, options)
-        )
+            # Execute the batch command with the options file path
+            cmd = (
+                f"bash --login -c 'python3 {self.sharedArea}/execute_batch {remoteOptionsFile} || "
+                f"python {self.sharedArea}/execute_batch {remoteOptionsFile} || "
+                f"python2 {self.sharedArea}/execute_batch {remoteOptionsFile}'"
+            )
 
-        self.log.verbose(f"CE submission command: {cmd}")
+            self.log.verbose(f"CE submission command: {cmd}")
 
-        result = ssh.sshCall(120, cmd)
-        if not result["OK"]:
-            self.log.error(f"{self.ceType} CE job submission failed", result["Message"])
-            return result
+            result = ssh.sshCall(120, cmd)
+            if not result["OK"]:
+                self.log.error(f"{self.ceType} CE job submission failed", result["Message"])
+                return result
 
-        sshStatus = result["Value"][0]
-        sshStdout = result["Value"][1]
-        sshStderr = result["Value"][2]
-
-        # Examine results of the job submission
-        if sshStatus == 0:
-            output = sshStdout.strip().replace("\r", "").strip()
-            if not output:
-                return S_ERROR("No output from remote command")
-
-            try:
-                index = output.index("============= Start output ===============")
-                output = output[index + 42 :]
-            except ValueError:
-                return S_ERROR(f"Invalid output from remote command: {output}")
-
-            try:
-                output = unquote(output)
-                result = json.loads(output)
-                if isinstance(result, str) and result.startswith("Exception:"):
-                    return S_ERROR(result)
-                return S_OK(result)
-            except Exception:
-                return S_ERROR("Invalid return structure from job submission")
-        else:
-            return S_ERROR("\n".join([sshStdout, sshStderr]))
+            sshStatus = result["Value"][0]
+            if sshStatus != 0:
+                sshStdout = result["Value"][1]
+                sshStderr = result["Value"][2]
+                return S_ERROR(f"CE job submission command failed with status {sshStatus}: {sshStdout} {sshStderr}")
+
+            # The result should be written to a JSON file by execute_batch
+            # Compute the expected result file path
+            remoteResultFile = remoteOptionsFile.replace(".json", "_result.json")
+
+            # Try to download the result file
+            with tempfile.NamedTemporaryFile(mode="r", suffix=".json", delete=False) as f:
+                localResultFile = f.name
+
+            result = ssh.scpCall(30, localResultFile, remoteResultFile, upload=False)
+            if not result["OK"]:
+                return result
+
+            # Read the result from the downloaded file
+            with open(localResultFile) as f:
+                result = json.load(f)
+            return S_OK(result)
+        finally:
+            # Clean up local temporary file
+            if localOptionsFile and os.path.exists(localOptionsFile):
+                os.remove(localOptionsFile)
+            if localResultFile and os.path.exists(localResultFile):
+                os.remove(localResultFile)
+            # Clean up remote temporary files
+            if remoteOptionsFile:
+                ssh.sshCall(30, f"rm -f {remoteOptionsFile}")
+            if remoteResultFile:
+                ssh.sshCall(30, f"rm -f {remoteResultFile}")
 
     def submitJob(self, executableFile, proxy, numberOfJobs=1):
         #    self.log.verbose( "Executable file path: %s" % executableFile )

DIRAC/TransformationSystem/Agent/InputDataAgent.py

@@ -74,7 +74,10 @@ class InputDataAgent(AgentModule):
         """Main execution method"""
 
         # Get all the transformations
-        result = self.transClient.getTransformations({"Status": "Active", "Type": self.transformationTypes})
+        result = self.transClient.getTransformations(
+            {"Status": "Active", "Type": self.transformationTypes},
+            columns=["TransformationID", "AuthorDN", "AuthorGroup"],
+        )
         if not result["OK"]:
             self.log.error("InputDataAgent.execute: Failed to get transformations.", result["Message"])
             return S_OK()

DIRAC/TransformationSystem/Agent/MCExtensionAgent.py

@@ -1,4 +1,4 @@
-""" Agent to extend the number of tasks given the Transformation definition
+"""Agent to extend the number of tasks given the Transformation definition
 
 The following options can be set for the MCExtensionAgent.
 
@@ -8,6 +8,7 @@ The following options can be set for the MCExtensionAgent.
   :dedent: 2
   :caption: MCExtensionAgent options
 """
+
 from DIRAC import S_OK, gLogger
 from DIRAC.Core.Base.AgentModule import AgentModule
 from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
@@ -54,7 +55,9 @@ class MCExtensionAgent(AgentModule):
             return S_OK("Disabled via CS flag")
 
         # Obtain the transformations in Cleaning status and remove any mention of the jobs/files
-        res = self.transClient.getTransformations({"Status": "Active", "Type": self.transformationTypes})
+        res = self.transClient.getTransformations(
+            {"Status": "Active", "Type": self.transformationTypes}, columns=["TransformationID", "MaxNumberOfTasks"]
+        )
         if res["OK"]:
             for transDict in res["Value"]:
                 transID = transDict["TransformationID"]

DIRAC/TransformationSystem/Agent/TaskManagerAgentBase.py

@@ -231,7 +231,9 @@ class TaskManagerAgentBase(AgentModule, TransformationAgentsUtilities):
             selectCond["Type"] = transType
         if agentType:
             selectCond["AgentType"] = agentType
-        res = self.transClient.getTransformations(condDict=selectCond)
+        res = self.transClient.getTransformations(
+            condDict=selectCond, columns=["TransformationID", "Body", "Author", "AuthorGroup"]
+        )
         if not res["OK"]:
             self.log.error("Failed to get transformations:", res["Message"])
         elif not res["Value"]:

DIRAC/TransformationSystem/Agent/TransformationCleaningAgent.py

@@ -20,6 +20,7 @@ from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
 from DIRAC.Core.Base.AgentModule import AgentModule
 from DIRAC.Core.Utilities.DErrno import cmpError
 from DIRAC.Core.Utilities.List import breakListIntoChunks
+from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader
 from DIRAC.Core.Utilities.Proxy import executeWithUserProxy
 from DIRAC.Core.Utilities.ReturnValues import returnSingleResult
 from DIRAC.RequestManagementSystem.Client.File import File
@@ -32,7 +33,6 @@ from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient
 from DIRAC.Resources.Storage.StorageElement import StorageElement
 from DIRAC.TransformationSystem.Client import TransformationStatus
 from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
-from DIRAC.WorkloadManagementSystem.DB.JobDB import JobDB
 from DIRAC.WorkloadManagementSystem.Service.JobPolicy import (
     RIGHT_DELETE,
     RIGHT_KILL,
@@ -65,8 +65,11 @@ class TransformationCleaningAgent(AgentModule):
         self.reqClient = None
         # # file catalog client
         self.metadataClient = None
-        # # JobDB
+        # # databases
         self.jobDB = None
+        self.pilotAgentsDB = None
+        self.taskQueueDB = None
+        self.storageManagementDB = None
 
         # # transformations types
         self.transformationTypes = None
@@ -125,8 +128,26 @@ class TransformationCleaningAgent(AgentModule):
         self.reqClient = ReqClient()
         # # file catalog client
         self.metadataClient = FileCatalogClient()
-        # # job DB
-        self.jobDB = JobDB()
+        # # databases
+        result = ObjectLoader().loadObject("WorkloadManagementSystem.DB.JobDB", "JobDB")
+        if not result["OK"]:
+            return result
+        self.jobDB = result["Value"]()
+
+        result = ObjectLoader().loadObject("WorkloadManagementSystem.DB.PilotAgentsDB", "PilotAgentsDB")
+        if not result["OK"]:
+            return result
+        self.pilotAgentsDB = result["Value"]()
+
+        result = ObjectLoader().loadObject("WorkloadManagementSystem.DB.TaskQueueDB", "TaskQueueDB")
+        if not result["OK"]:
+            return result
+        self.taskQueueDB = result["Value"]()
+
+        result = ObjectLoader().loadObject("StorageManagementSystem.DB.StorageManagementDB", "StorageManagementDB")
+        if not result["OK"]:
+            return result
+        self.storageManagementDB = result["Value"]()
 
         return S_OK()
 
@@ -144,7 +165,8 @@ class TransformationCleaningAgent(AgentModule):
 
         # Obtain the transformations in Cleaning status and remove any mention of the jobs/files
         res = self.transClient.getTransformations(
-            {"Status": TransformationStatus.CLEANING, "Type": self.transformationTypes}
+            {"Status": TransformationStatus.CLEANING, "Type": self.transformationTypes},
+            columns=["TransformationID", "Author", "AuthorGroup", "Type"],
         )
         if res["OK"]:
             for transDict in res["Value"]:
@@ -161,7 +183,10 @@ class TransformationCleaningAgent(AgentModule):
             self.log.error("Failed to get transformations", res["Message"])
 
         # Obtain the transformations in RemovingFiles status and removes the output files
-        res = self.transClient.getTransformations({"Status": "RemovingFiles", "Type": self.transformationTypes})
+        res = self.transClient.getTransformations(
+            {"Status": "RemovingFiles", "Type": self.transformationTypes},
+            columns=["TransformationID", "Author", "AuthorGroup"],
+        )
         if res["OK"]:
             for transDict in res["Value"]:
                 if self.shifterProxy:
@@ -183,6 +208,7 @@ class TransformationCleaningAgent(AgentModule):
             {"Status": TransformationStatus.COMPLETED, "Type": self.transformationTypes},
             older=olderThanTime,
             timeStamp="LastUpdate",
+            columns=["TransformationID", "Author", "AuthorGroup"],
         )
         if res["OK"]:
             for transDict in res["Value"]:
@@ -230,7 +256,10 @@ class TransformationCleaningAgent(AgentModule):
             return res
         transformationIDs = res["Value"]
         if transformationIDs:
-            res = self.transClient.getTransformations({"TransformationID": transformationIDs})
+            res = self.transClient.getTransformations(
+                {"TransformationID": transformationIDs},
+                columns=["TransformationID", "Status", "Author", "AuthorGroup", "Type"],
+            )
             if not res["OK"]:
                 self.log.error("Failed to get transformations", res["Message"])
                 return res
@@ -607,11 +636,17 @@ class TransformationCleaningAgent(AgentModule):
         :param self: self reference
         :param list trasnJobIDs: job IDs
         """
+        db_kwargs = dict(
+            jobdb=self.jobDB,
+            taskqueuedb=self.taskQueueDB,
+            pilotagentsdb=self.pilotAgentsDB,
+            storagemanagementdb=self.storageManagementDB,
+        )
         # Prevent 0 job IDs
         jobIDs = [int(j) for j in transJobIDs if int(j)]
         allRemove = True
         for jobList in breakListIntoChunks(jobIDs, 1000):
-            res = kill_delete_jobs(RIGHT_KILL, jobList, force=True)
+            res = kill_delete_jobs(RIGHT_KILL, jobList, force=True, **db_kwargs)
             if res["OK"]:
                 self.log.info(f"Successfully killed {len(jobList)} jobs from WMS")
             elif ("InvalidJobIDs" in res) and ("NonauthorizedJobIDs" not in res) and ("FailedJobIDs" not in res):
@@ -623,7 +658,7 @@ class TransformationCleaningAgent(AgentModule):
                 self.log.error("Failed to kill jobs", f"(n={len(res['FailedJobIDs'])})")
                 allRemove = False
 
-            res = kill_delete_jobs(RIGHT_DELETE, jobList, force=True)
+            res = kill_delete_jobs(RIGHT_DELETE, jobList, force=True, **db_kwargs)
             if res["OK"]:
                 self.log.info("Successfully deleted jobs from WMS", f"(n={len(jobList)})")
             elif ("InvalidJobIDs" in res) and ("NonauthorizedJobIDs" not in res) and ("FailedJobIDs" not in res):

DIRAC/TransformationSystem/Agent/ValidateOutputDataAgent.py

@@ -76,7 +76,9 @@ class ValidateOutputDataAgent(AgentModule):
         self.updateWaitingIntegrity()
         gLogger.info("-" * 40)
 
-        res = self.transClient.getTransformations({"Status": "ValidatingOutput", "Type": self.transformationTypes})
+        res = self.transClient.getTransformations(
+            {"Status": "ValidatingOutput", "Type": self.transformationTypes}, columns=["TransformationID"]
+        )
         if not res["OK"]:
             gLogger.error("Failed to get ValidatingOutput transformations", res["Message"])
             return res
@@ -98,7 +100,7 @@ class ValidateOutputDataAgent(AgentModule):
     def updateWaitingIntegrity(self):
         """Get 'WaitingIntegrity' transformations, update to 'ValidatedOutput'"""
         gLogger.info("Looking for transformations in the WaitingIntegrity status to update")
-        res = self.transClient.getTransformations({"Status": "WaitingIntegrity"})
+        res = self.transClient.getTransformations({"Status": "WaitingIntegrity"}, columns=["TransformationID"])
         if not res["OK"]:
             gLogger.error("Failed to get WaitingIntegrity transformations", res["Message"])
             return res

DIRAC/TransformationSystem/Client/TransformationClient.py

@@ -114,13 +114,21 @@ class TransformationClient(Client):
         newer=None,
         timeStamp=None,
         orderAttribute=None,
-        limit=100,
+        limit=None,
         extraParams=False,
         columns=None,
     ):
         """gets all the transformations in the system, incrementally. "limit" here is just used to determine the offset."""
         rpcClient = self._getRPC()
 
+        # If the body is requested (or is served by default)
+        # we take smaller chunk, not to take too much memory
+        # on the server
+        if columns and "Body" not in columns:
+            limit = 100_000
+        else:
+            limit = 1_000
+
         transformations = []
         if condDict is None:
             condDict = {}