DIRAC 9.0.0a42__py3-none-any.whl → 9.0.7__py3-none-any.whl

DIRAC/TransformationSystem/Agent/TransformationCleaningAgent.py

@@ -16,14 +16,12 @@ from datetime import datetime, timedelta
 
 # # from DIRAC
 from DIRAC import S_ERROR, S_OK
-from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData
 from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
 from DIRAC.Core.Base.AgentModule import AgentModule
 from DIRAC.Core.Utilities.DErrno import cmpError
 from DIRAC.Core.Utilities.List import breakListIntoChunks
 from DIRAC.Core.Utilities.Proxy import executeWithUserProxy
 from DIRAC.Core.Utilities.ReturnValues import returnSingleResult
-from DIRAC.DataManagementSystem.Client.DataManager import DataManager
 from DIRAC.RequestManagementSystem.Client.File import File
 from DIRAC.RequestManagementSystem.Client.Operation import Operation
 from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient
@@ -34,8 +32,12 @@ from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient
 from DIRAC.Resources.Storage.StorageElement import StorageElement
 from DIRAC.TransformationSystem.Client import TransformationStatus
 from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
-from DIRAC.WorkloadManagementSystem.Client.JobMonitoringClient import JobMonitoringClient
-from DIRAC.WorkloadManagementSystem.Client.WMSClient import WMSClient
+from DIRAC.WorkloadManagementSystem.DB.JobDB import JobDB
+from DIRAC.WorkloadManagementSystem.Service.JobPolicy import (
+    RIGHT_DELETE,
+    RIGHT_KILL,
+)
+from DIRAC.WorkloadManagementSystem.DB.StatusUtils import kill_delete_jobs
 
 # # agent's name
 AGENT_NAME = "Transformation/TransformationCleaningAgent"
@@ -59,12 +61,12 @@ class TransformationCleaningAgent(AgentModule):
 
         # # transformation client
         self.transClient = None
-        # # wms client
-        self.wmsClient = None
         # # request client
         self.reqClient = None
         # # file catalog client
         self.metadataClient = None
+        # # JobDB
+        self.jobDB = None
 
         # # transformations types
         self.transformationTypes = None
@@ -119,14 +121,12 @@ class TransformationCleaningAgent(AgentModule):
 
         # # transformation client
         self.transClient = TransformationClient()
-        # # wms client
-        self.wmsClient = WMSClient()
         # # request client
         self.reqClient = ReqClient()
         # # file catalog client
         self.metadataClient = FileCatalogClient()
-        # # job monitoring client
-        self.jobMonitoringClient = JobMonitoringClient()
+        # # job DB
+        self.jobDB = JobDB()
 
         return S_OK()
 
@@ -224,7 +224,7 @@ class TransformationCleaningAgent(AgentModule):
         So, we should just clean from time to time.
         What I added here is done only when the agent finalize, and it's quite light-ish operation anyway.
         """
-        res = self.jobMonitoringClient.getJobGroups(None, datetime.utcnow() - timedelta(days=365))
+        res = self.jobDB.getDistinctJobAttributes("JobGroup", None, datetime.utcnow() - timedelta(days=365))
         if not res["OK"]:
             self.log.error("Failed to get job groups", res["Message"])
             return res
@@ -268,7 +268,7 @@ class TransformationCleaningAgent(AgentModule):
 
             # Remove JobIDs that were unknown to the TransformationSystem
             jobGroupsToCheck = [str(transDict["TransformationID"]).zfill(8) for transDict in toClean + toArchive]
-            res = self.jobMonitoringClient.getJobs({"JobGroup": jobGroupsToCheck})
+            res = self.jobDB.selectJobs({"JobGroup": jobGroupsToCheck})
             if not res["OK"]:
                 return res
             jobIDsToRemove = [int(jobID) for jobID in res["Value"]]
@@ -610,8 +610,8 @@ class TransformationCleaningAgent(AgentModule):
         # Prevent 0 job IDs
         jobIDs = [int(j) for j in transJobIDs if int(j)]
         allRemove = True
-        for jobList in breakListIntoChunks(jobIDs, 500):
-            res = self.wmsClient.killJob(jobList, force=True)
+        for jobList in breakListIntoChunks(jobIDs, 1000):
+            res = kill_delete_jobs(RIGHT_KILL, jobList, force=True)
             if res["OK"]:
                 self.log.info(f"Successfully killed {len(jobList)} jobs from WMS")
             elif ("InvalidJobIDs" in res) and ("NonauthorizedJobIDs" not in res) and ("FailedJobIDs" not in res):
@@ -623,7 +623,7 @@ class TransformationCleaningAgent(AgentModule):
                 self.log.error("Failed to kill jobs", f"(n={len(res['FailedJobIDs'])})")
                 allRemove = False
 
-            res = self.wmsClient.deleteJob(jobList)
+            res = kill_delete_jobs(RIGHT_DELETE, jobList, force=True)
             if res["OK"]:
                 self.log.info("Successfully deleted jobs from WMS", f"(n={len(jobList)})")
             elif ("InvalidJobIDs" in res) and ("NonauthorizedJobIDs" not in res) and ("FailedJobIDs" not in res):
@@ -684,7 +684,7 @@ class TransformationCleaningAgent(AgentModule):
 
         for index, lfnList in enumerate(breakListIntoChunks(lfns, 300)):
             oRequest = Request()
-            requestName = "TCA_{transID}_{index}_{md5(repr(time.time()).encode()).hexdigest()[:5]}"
+            requestName = f"TCA_{transID}_{index}_{md5(repr(time.time()).encode()).hexdigest()[:5]}"
             oRequest.RequestName = requestName
             oOperation = Operation()
             oOperation.Type = "RemoveFile"

DIRAC/TransformationSystem/Client/TaskManager.py

@@ -6,11 +6,9 @@ for managing jobs and requests tasks
 
 import time
 
-from DIRAC import S_OK, S_ERROR, gLogger
-from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
+from DIRAC import S_ERROR, S_OK, gLogger
 from DIRAC.TransformationSystem.Agent.TransformationAgentsUtilities import TransformationAgentsUtilities
-
-COMPONENT_NAME = "TaskManager"
+from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
 
 
 class TaskBase(TransformationAgentsUtilities):

DIRAC/TransformationSystem/Client/Transformation.py

@@ -4,16 +4,15 @@ See the information about transformation parameters below.
 """
 import json
 
-from DIRAC import gLogger, gConfig, S_OK, S_ERROR
+from DIRAC import S_ERROR, S_OK, gConfig, gLogger
+from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
+from DIRAC.Core.Base.API import API
 from DIRAC.Core.Utilities.JEncode import encode
 from DIRAC.Core.Utilities.PromptUser import promptUser
-from DIRAC.Core.Base.API import API
+from DIRAC.MonitoringSystem.Client.WebAppClient import WebAppClient
+from DIRAC.RequestManagementSystem.Client.Operation import Operation
 from DIRAC.TransformationSystem.Client.BodyPlugin.BaseBody import BaseBody
 from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
-from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
-from DIRAC.RequestManagementSystem.Client.Operation import Operation
-
-COMPONENT_NAME = "Transformation"
 
 
 class Transformation(API):
@@ -500,7 +499,7 @@ class Transformation(API):
         ]
         dictList = []
 
-        result = self.transClient.getTransformationSummaryWeb(condDict, orderby, start, maxitems)
+        result = WebAppClient().getTransformationSummaryWeb(condDict, orderby, start, maxitems)
         if not result["OK"]:
             self._prettyPrint(result)
             return result

DIRAC/TransformationSystem/Client/TransformationClient.py

@@ -1,9 +1,10 @@
-""" Class that contains client access to the transformation DB handler. """
+"""Class that contains client access to the transformation DB handler."""
 
 from DIRAC import S_OK, S_ERROR, gLogger
 from DIRAC.Core.Base.Client import Client, createClient
 from DIRAC.Core.Utilities.List import breakListIntoChunks
-from DIRAC.Core.Utilities.JEncode import decode as jdecode
+from DIRAC.Core.Utilities.ReturnValues import convertToReturnValue, returnValueOrRaise
+from DIRAC.Core.Utilities.JEncode import decode as jdecode, strToIntDict
 from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
 from DIRAC.TransformationSystem.Client import TransformationStatus
 from DIRAC.TransformationSystem.Client import TransformationFilesStatus
@@ -47,13 +48,6 @@ class TransformationClient(Client):
 
         getFileSummary(lfns)
         exists(lfns)
-
-    Web monitoring tools
-
-        getDistinctAttributeValues(attribute, selectDict)
-        getTransformationStatusCounters()
-        getTransformationSummary()
-        getTransformationSummaryWeb(selectDict, sortList, startItem, maxItems)
     """
 
     def __init__(self, **kwargs):
@@ -62,8 +56,8 @@ class TransformationClient(Client):
         super().__init__(**kwargs)
         opsH = Operations()
         self.maxResetCounter = opsH.getValue("Transformations/FilesMaxResetCounter", 10)
-
-        self.setServer("Transformation/TransformationManager")
+        if "url" not in kwargs:
+            self.setServer("Transformation/TransformationManager")
 
     def setServer(self, url):
         self.serverURL = url
@@ -591,3 +585,19 @@ class TransformationClient(Client):
     def addDirectory(self, path, force=False):
         rpcClient = self._getRPC()
         return rpcClient.addDirectory(path, force)
+
+    @convertToReturnValue
+    def getTransformationFilesCount(self, transName, field, selection=None, **kwargs):
+        if not selection:
+            selection = {}
+        rpcClient = self._getRPC(**kwargs)
+
+        files_count = returnValueOrRaise(rpcClient.getTransformationFilesCount(transName, field, selection))
+        # Some of these fields are LHCb specific, however:
+        # * it seems only LHCb uses this method
+        # * it will not be necessary with diracx & pydantic anymore
+        if field in ["TransformationID", "FileID", "TaskID", "ErrorCount", "RunNumber", "Size"]:
+            total = files_count.pop("Total")
+            files_count = strToIntDict(files_count)
+            files_count["Total"] = total
+        return files_count

DIRAC/TransformationSystem/Client/Utilities.py

@@ -9,6 +9,8 @@ Utilities for Transformation system
 import ast
 import random
 
+from cachetools import LRUCache, cachedmethod
+from cachetools.keys import hashkey
 from DIRAC import S_OK, S_ERROR, gLogger
 
 from DIRAC.Core.Utilities.List import breakListIntoChunks
@@ -22,6 +24,9 @@ from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
 from DIRAC.Resources.Storage.StorageElement import StorageElement
 from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
 
+# Module-level cache for isSameSEInList method (shared across PluginUtilities instances)
+_is_same_se_in_list_cache = LRUCache(maxsize=1024)
+
 
 class PluginUtilities:
     """
@@ -400,6 +405,10 @@ class PluginUtilities:
 
         return StorageElement(se1).isSameSE(StorageElement(se2))
 
+    @cachedmethod(
+        lambda self: _is_same_se_in_list_cache,
+        key=lambda _, a, b: hashkey(a, *sorted(b)),
+    )
     def isSameSEInList(self, se1, seList):
         """Check if an SE is the same as any in a list"""
         if se1 in seList:

DIRAC/TransformationSystem/DB/TransformationDB.py

@@ -1,9 +1,9 @@
-""" DIRAC Transformation DB
+"""DIRAC Transformation DB
 
-    Transformation database is used to collect and serve the necessary information
-    in order to automate the task of job preparation for high level transformations.
-    This class is typically used as a base class for more specific data processing
-    databases
+Transformation database is used to collect and serve the necessary information
+in order to automate the task of job preparation for high level transformations.
+This class is typically used as a base class for more specific data processing
+databases
 """
 
 import re
@@ -397,14 +397,11 @@ class TransformationDB(DB):
     def __updateTransformationParameter(self, transID, paramName, paramValue, connection=False):
         if paramName not in self.mutable:
             return S_ERROR(f"Can not update the '{paramName}' transformation parameter")
-        if paramName == "Body":
-            res = self._escapeString(paramValue)
-            if not res["OK"]:
-                return S_ERROR("Failed to parse parameter value")
-            paramValue = res["Value"]
-            req = f"UPDATE Transformations SET {paramName}='{paramValue}', LastUpdate=UTC_TIMESTAMP() WHERE TransformationID={transID}"
-            return self._update(req, conn=connection)
-        req = f"UPDATE Transformations SET {paramName}='{paramValue}', LastUpdate=UTC_TIMESTAMP() WHERE TransformationID={transID}"
+        res = self._escapeString(paramValue)
+        if not res["OK"]:
+            return S_ERROR("Failed to parse parameter value")
+        paramValue = res["Value"]
+        req = f"UPDATE Transformations SET {paramName}={paramValue}, LastUpdate=UTC_TIMESTAMP() WHERE TransformationID={transID}"
         return self._update(req, conn=connection)
 
     def _getTransformationID(self, transName, connection=False):
@@ -709,7 +706,7 @@ class TransformationDB(DB):
         res = self.getCounters("TransformationFiles", ["TransformationID", field], selection)
         if not res["OK"]:
             return res
-        countDict = {attrDict[field]: count for attrDict, count in res["Value"]}
+        countDict = {str(attrDict[field]): count for attrDict, count in res["Value"]}
         countDict["Total"] = sum(countDict.values())
         return S_OK(countDict)
 

DIRAC/TransformationSystem/DB/TransformationDB.sql

@@ -48,7 +48,7 @@ CREATE TABLE Transformations(
     INDEX(TransformationFamily),
     INDEX(Status),
     INDEX(Type)
-) ENGINE = InnoDB DEFAULT CHARSET = utf8;
+) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4;
 
 -- -------------------------------------------------------------------------------
 DROP TABLE IF EXISTS DataFiles;
@@ -59,7 +59,7 @@ CREATE TABLE DataFiles(
     INDEX(Status),
     INDEX(LFN),
     PRIMARY KEY(FileID)
-) ENGINE = InnoDB DEFAULT CHARSET = utf8;
+) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4;
 
 -- -------------------------------------------------------------------------------
 DROP TABLE IF EXISTS AdditionalParameters;
@@ -70,7 +70,7 @@ CREATE TABLE AdditionalParameters(
     ParameterType VARCHAR(32) DEFAULT 'StringType',
     PRIMARY KEY(TransformationID, ParameterName),
     FOREIGN KEY(TransformationID) REFERENCES Transformations(TransformationID)
-) ENGINE = InnoDB DEFAULT CHARSET = utf8;
+) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4;
 
 -- -------------------------------------------------------------------------------
 DROP TABLE IF EXISTS TransformationLog;
@@ -84,7 +84,7 @@ CREATE TABLE TransformationLog(
     INDEX(TransformationID),
     INDEX(MessageDate),
     FOREIGN KEY(TransformationID) REFERENCES Transformations(TransformationID)
-) ENGINE = InnoDB DEFAULT CHARSET = utf8;
+) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4;
 
 -- -------------------------------------------------------------------------------
 DROP TABLE IF EXISTS TransformationTasks;
@@ -100,7 +100,7 @@ CREATE TABLE TransformationTasks(
     INDEX(ExternalStatus),
     INDEX(TransformationID,ExternalStatus),
     FOREIGN KEY(TransformationID) REFERENCES Transformations(TransformationID)
-) ENGINE = InnoDB DEFAULT CHARSET = utf8;
+) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4;
 
 -- This is required to mimic the AUTO_INCREMENT behavior of TaskID which was possible with MyISAM:
 CREATE TRIGGER `TaskID_Generator` BEFORE INSERT ON TransformationTasks
@@ -125,7 +125,7 @@ CREATE TABLE TransformationFiles(
     INDEX(TransformationID,Status),
     FOREIGN KEY(TransformationID) REFERENCES Transformations(TransformationID),
     FOREIGN KEY(FileID) REFERENCES DataFiles(FileID)
-) ENGINE = InnoDB DEFAULT CHARSET = utf8;
+) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4;
 
 -- -------------------------------------------------------------------------------
 DROP TABLE IF EXISTS TransformationFileTasks;
@@ -137,7 +137,7 @@ CREATE TABLE TransformationFileTasks(
     FOREIGN KEY(TransformationID) REFERENCES Transformations(TransformationID),
     FOREIGN KEY(TransformationID, FileID) REFERENCES TransformationFiles(TransformationID, FileID),
     FOREIGN KEY(TransformationID, TaskID) REFERENCES TransformationTasks(TransformationID, TaskID)
-) ENGINE = InnoDB DEFAULT CHARSET = utf8;
+) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4;
 
 -- -------------------------------------------------------------------------------
 DROP TABLE IF EXISTS TaskInputs;
@@ -147,7 +147,7 @@ CREATE TABLE TaskInputs(
     InputVector MEDIUMTEXT,
     PRIMARY KEY(TransformationID, TaskID),
     FOREIGN KEY(TransformationID, TaskID) REFERENCES TransformationTasks(TransformationID, TaskID)
-) ENGINE = InnoDB DEFAULT CHARSET = utf8;
+) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4;
 
 -- -------------------------------------------------------------------------------
 DROP TABLE IF EXISTS TransformationMetaQueries;
@@ -159,6 +159,6 @@ CREATE TABLE TransformationMetaQueries(
     QueryType ENUM('Input', 'Output') DEFAULT 'Input',
     PRIMARY KEY(TransformationID, MetaDataName, QueryType),
     FOREIGN KEY(TransformationID) REFERENCES Transformations(TransformationID)
-) ENGINE = InnoDB DEFAULT CHARSET = utf8;
+) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4;
 
 SET FOREIGN_KEY_CHECKS = 1;