DIRAC 9.0.0a54__py3-none-any.whl → 9.0.7__py3-none-any.whl

DIRAC/Interfaces/API/DiracAdmin.py

@@ -4,7 +4,9 @@ All administrative functionality is exposed through the DIRAC Admin API.  Exampl
 site banning and unbanning, WMS proxy uploading etc.
 
 """
+
 import os
+from datetime import datetime, timedelta
 
 from DIRAC import S_ERROR, S_OK, gConfig, gLogger
 from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI
@@ -19,6 +21,9 @@ from DIRAC.ResourceStatusSystem.Client.SiteStatus import SiteStatus
 from DIRAC.WorkloadManagementSystem.Client.JobManagerClient import JobManagerClient
 from DIRAC.WorkloadManagementSystem.Client.PilotManagerClient import PilotManagerClient
 from DIRAC.WorkloadManagementSystem.Client.WMSAdministratorClient import WMSAdministratorClient
+from DIRAC.WorkloadManagementSystem.Service.WMSUtilities import (
+    killPilotsInQueues,
+)
 
 voName = ""
 ret = getProxyInfo(disableVOMS=True)
@@ -147,7 +152,7 @@ class DiracAdmin(API):
         return result
 
     #############################################################################
-    def allowSite(self, site, comment, printOutput=False):
+    def allowSite(self, site, comment, printOutput=False, days=1):
         """Adds the site to the site mask. The site must be a valid DIRAC site name
 
         Example usage:
@@ -171,7 +176,13 @@ class DiracAdmin(API):
                 gLogger.notice(f"Site {site} is already Active")
             return S_OK(f"Site {site} is already Active")
 
-        if not (result := self.sitestatus.setSiteStatus(site, "Active", comment))["OK"]:
+        tokenLifetime = int(days)
+        if tokenLifetime <= 0:
+            tokenExpiration = datetime.max
+        else:
+            tokenExpiration = datetime.utcnow().replace(microsecond=0) + timedelta(days=tokenLifetime)
+
+        if not (result := self.sitestatus.setSiteStatus(site, "Active", comment, expiry=tokenExpiration))["OK"]:
             return result
 
         if printOutput:
@@ -220,7 +231,7 @@ class DiracAdmin(API):
         return S_OK()
 
     #############################################################################
-    def banSite(self, site, comment, printOutput=False):
+    def banSite(self, site, comment, printOutput=False, days=1):
         """Removes the site from the site mask.
 
         Example usage:
@@ -233,7 +244,6 @@ class DiracAdmin(API):
         """
         if not (result := self._checkSiteIsValid(site))["OK"]:
             return result
-
         mask = self.getSiteMask(status="Banned")
         if not mask["OK"]:
             return mask
@@ -243,7 +253,12 @@ class DiracAdmin(API):
                 gLogger.notice(f"Site {site} is already Banned")
             return S_OK(f"Site {site} is already Banned")
 
-        if not (result := self.sitestatus.setSiteStatus(site, "Banned", comment))["OK"]:
+        tokenLifetime = int(days)
+        if tokenLifetime <= 0:
+            tokenExpiration = datetime.max
+        else:
+            tokenExpiration = datetime.utcnow().replace(microsecond=0) + timedelta(days=tokenLifetime)
+        if not (result := self.sitestatus.setSiteStatus(site, "Banned", comment, expiry=tokenExpiration))["OK"]:
             return result
 
         if printOutput:
@@ -452,8 +467,28 @@ class DiracAdmin(API):
         if not isinstance(gridReference, (str, list)):
             return self._errorReport("Expected string or list of strings for pilot reference")
 
-        result = PilotManagerClient().killPilot(gridReference)
-        return result
+        # Make a list if it is not yet
+        if isinstance(gridReference, str):
+            gridReference = [gridReference]
+
+        # Regroup pilots per site
+        pilotRefDict = {}
+        for pilotReference in gridReference:
+            result = PilotManagerClient().getPilotInfo(pilotReference)
+            if not result["OK"] or not result["Value"]:
+                return S_ERROR(f"Failed to get info for pilot {pilotReference}")
+
+            pilotDict = result["Value"][pilotReference]
+            queue = "@@@".join(
+                [pilotDict["VO"], pilotDict["GridSite"], pilotDict["DestinationSite"], pilotDict["Queue"]]
+            )
+            gridType = pilotDict["GridType"]
+            pilotRefDict.setdefault(queue, {})
+            pilotRefDict[queue].setdefault("PilotList", [])
+            pilotRefDict[queue]["PilotList"].append(pilotReference)
+            pilotRefDict[queue]["GridType"] = gridType
+
+        return killPilotsInQueues(pilotRefDict)
 
     #############################################################################
     def getPilotLoggingInfo(self, gridReference):

DIRAC/Interfaces/API/Job.py

@@ -1105,6 +1105,7 @@ class Job(API):
                 uniqueInputSandbox = uniqueElements(finalInputSandbox.split(";"))
                 paramsDict["InputSandbox"]["value"] = ";".join(uniqueInputSandbox)
                 self.log.verbose(f"Final unique Input Sandbox {';'.join(uniqueInputSandbox)}")
+                paramsDict["InputSandbox"]["type"] = "JDL"
             else:
                 paramsDict["InputSandbox"] = {}
                 paramsDict["InputSandbox"]["value"] = extraFiles

DIRAC/Interfaces/scripts/dirac_admin_allow_site.py

@@ -13,6 +13,9 @@ from DIRAC.Core.Base.Script import Script
 @Script()
 def main():
     Script.registerSwitch("E:", "email=", "Boolean True/False (True by default)")
+    Script.registerSwitch(
+        "", "days=", "Number of days the token is valid for. Default is 1 day. 0 or less days denotes forever."
+    )
     # Registering arguments will automatically add their description to the help menu
     Script.registerArgument("Site:     Name of the Site")
     Script.registerArgument("Comment:  Reason of the action")
@@ -32,9 +35,12 @@ def main():
             Script.showHelp()
 
     email = True
+    days = 1
     for switch in Script.getUnprocessedSwitches():
         if switch[0] == "email":
             email = getBoolean(switch[1])
+        if switch[0] == "days":
+            days = int(switch[1])
 
     diracAdmin = DiracAdmin()
     exitCode = 0
@@ -42,7 +48,7 @@ def main():
 
     # parseCommandLine show help when mandatory arguments are not specified or incorrect argument
     site, comment = Script.getPositionalArgs(group=True)
-    result = diracAdmin.allowSite(site, comment, printOutput=True)
+    result = diracAdmin.allowSite(site, comment, printOutput=True, days=days)
     if not result["OK"]:
         errorList.append((site, result["Message"]))
         exitCode = 2

DIRAC/Interfaces/scripts/dirac_admin_ban_site.py

@@ -13,6 +13,9 @@ from DIRAC.Core.Base.Script import Script
 @Script()
 def main():
     Script.registerSwitch("E:", "email=", "Boolean True/False (True by default)")
+    Script.registerSwitch(
+        "", "days=", "Number of days the token is valid for. Default is 1 day. 0 or less days denotes forever."
+    )
     # Registering arguments will automatically add their description to the help menu
     Script.registerArgument("Site:     Name of the Site")
     Script.registerArgument("Comment:  Reason of the action")
@@ -32,9 +35,12 @@ def main():
             Script.showHelp()
 
     email = True
+    days = 1
     for switch in Script.getUnprocessedSwitches():
         if switch[0] == "email":
             email = getBoolean(switch[1])
+        if switch[0] == "days":
+            days = int(switch[1])
 
     diracAdmin = DiracAdmin()
     exitCode = 0
@@ -50,7 +56,7 @@ def main():
 
     # parseCommandLine show help when mandatory arguments are not specified or incorrect argument
     site, comment = Script.getPositionalArgs(group=True)
-    result = diracAdmin.banSite(site, comment, printOutput=True)
+    result = diracAdmin.banSite(site, comment, printOutput=True, days=days)
     if not result["OK"]:
         errorList.append((site, result["Message"]))
         exitCode = 2

DIRAC/Interfaces/scripts/dirac_wms_job_parameters.py

@@ -17,7 +17,6 @@ Example:
    'LocalAccount': 'prod006',
    'LocalBatchID': '',
    'LocalJobID': '277821.ce.labmc.inf.utfsm.cl',
-   'MatcherServiceTime': '2.27646398544',
    'Memory(kB)': '858540kB',
    'ModelName': 'Intel(R)Xeon(R)CPU5110@1.60GHz',
    'NormCPUTime(s)': '1.02',

DIRAC/MonitoringSystem/Client/Types/WMSHistory.py

@@ -30,6 +30,8 @@ class WMSHistory(BaseType):
             "MinorStatus",
             "ApplicationStatus",
             "JobSplitType",
+            "Tier",
+            "Type",
         ]
 
         self.monitoringFields = ["Jobs", "Reschedules"]
@@ -46,6 +48,8 @@ class WMSHistory(BaseType):
                 "User": {"type": "keyword"},
                 "JobGroup": {"type": "keyword"},
                 "UserGroup": {"type": "keyword"},
+                "Tier": {"type": "keyword"},
+                "Type": {"type": "keyword"},
             }
         )
         # {'timestamp': {'type': 'date'}} will be added for all monitoring types

DIRAC/MonitoringSystem/Client/WebAppClient.py

@@ -0,0 +1,26 @@
+""" Module that contains client access to the WebApp handler.
+"""
+
+from DIRAC.Core.Base.Client import Client, createClient
+
+
+@createClient("Monitoring/WebApp")
+class WebAppClient(Client):
+    """WebAppClient sets url for the WebAppHandler."""
+
+    def __init__(self, url=None, **kwargs):
+        """
+        Sets URL for WebApp handler
+
+        :param self: self reference
+        :param url: url of the WebAppHandler
+        :param kwargs: forwarded to the Base Client class
+        """
+
+        super().__init__(**kwargs)
+
+        if not url:
+            self.serverURL = "Monitoring/WebApp"
+
+        else:
+            self.serverURL = url

DIRAC/MonitoringSystem/ConfigTemplate.cfg

@@ -28,4 +28,13 @@ Services
     }
   }
   ##END
+  ##BEGIN WebApp
+  WebApp
+  {
+    Port = 9199
+    Authorization
+    {
+      Default = authenticated
+    }
+  }
 }

DIRAC/MonitoringSystem/DB/MonitoringDB.py

@@ -5,8 +5,7 @@ Wrapper on top of ElasticDB. It is used to manage the DIRAC monitoring types.
 
 The following option can be set in `Systems/Monitoring/Databases/MonitoringDB`
 
-* *IndexPrefix*:  Prefix used to prepend to indexes created in the ES instance. If this
-                  is not present in the CS, the indexes are prefixed with the setup name.
+* *IndexPrefix*:  Prefix used to prepend to indexes created in the OpenSearch instance.
 
 For each monitoring types managed, the Period (how often a new index is created)
 can be defined with::
@@ -33,7 +32,6 @@ import time
 
 from DIRAC import S_ERROR, S_OK
 from DIRAC.ConfigurationSystem.Client.Config import gConfig
-from DIRAC.ConfigurationSystem.Client.Helpers import CSGlobals
 from DIRAC.ConfigurationSystem.Client.PathFinder import getDatabaseSection
 from DIRAC.Core.Base.ElasticDB import ElasticDB
 from DIRAC.Core.Utilities.Plotting.TypeLoader import TypeLoader
@@ -48,7 +46,7 @@ class MonitoringDB(ElasticDB):
 
         try:
             section = getDatabaseSection("Monitoring/MonitoringDB")
-            indexPrefix = gConfig.getValue(f"{section}/IndexPrefix", CSGlobals.getSetup()).lower()
+            indexPrefix = gConfig.getValue(f"{section}/IndexPrefix", "").lower()
             # Connecting to the ES cluster
             super().__init__(fullName=name, indexPrefix=indexPrefix)
         except RuntimeError as ex:
@@ -192,7 +190,10 @@ class MonitoringDB(ElasticDB):
         # and now we group with bucket aggregation
         groupingAggregation = self._A("terms", field=grouping, size=self.RESULT_SIZE)
         groupingAggregation.bucket(
-            "end_data", "date_histogram", field="timestamp", interval=interval  # name  # type
+            "end_data",
+            "date_histogram",
+            field="timestamp",
+            interval=interval,
         ).metric("timeAggregation", timeAggregation).pipeline(
             "timeAggregation_avg_bucket", "avg_bucket", buckets_path="timeAggregation>total", gap_policy="insert_zeros"
         )
@@ -447,26 +448,6 @@ class MonitoringDB(ElasticDB):
                 records.append({paramName: getattr(resObj["_source"], paramName) for paramName in paramNames})
             return S_OK(records)
 
-    def getLastDayData(self, typeName, condDict):
-        """
-        It returns the last day data for a given monitoring type.
-
-        :returns: for example
-
-          .. code-block:: python
-
-           {'sort': [{'timestamp': {'order': 'desc'}}],
-            'query': {'bool': {'must': [{'match': {'host': 'dzmathe.cern.ch'}},
-                                        {'match': {'component': 'Bookkeeping_BookkeepingManager'}}]}}}
-
-        :param str typeName: name of the monitoring type
-        :param dict condDict: conditions for the query
-
-                      * key -> name of the field
-                      * value -> list of possible values
-        """
-        return self.__getRawData(typeName, condDict)
-
     def getLimitedData(self, typeName, condDict, size=10):
         """
         Returns a list of records for a given selection.

DIRAC/MonitoringSystem/Service/MonitoringHandler.py

@@ -244,24 +244,6 @@ class MonitoringHandlerMixin:
         reportRequest["generatePlot"] = False
         return reporter.generate(reportRequest)
 
-    types_addMonitoringRecords = [str, list]
-
-    def export_addMonitoringRecords(self, monitoringtype, data):
-        """
-        Bulk insert data directly to the given monitoring type.
-
-        :param str monitoringtype: monitoring type name
-        :param list data: list of documents
-        :returns: S_OK or S_ERROR
-        """
-
-        retVal = self.__db.getIndexName(monitoringtype)
-        if not retVal["OK"]:
-            return retVal
-        prefix = retVal["Value"]
-        gLogger.debug("addMonitoringRecords:", prefix)
-        return self.__db.bulk_index(prefix, data)
-
     types_addRecords = [str, str, list]
 
     def export_addRecords(self, indexname, monitoringType, data):
@@ -290,21 +272,6 @@ class MonitoringHandlerMixin:
         gLogger.debug("delete index:", indexName)
         return self.__db.deleteIndex(indexName)
 
-    types_getLastDayData = [str, dict]
-
-    def export_getLastDayData(self, typeName, condDict):
-        """
-        It returns the data from the last day index. Note: we create daily indexes.
-
-        :param str typeName: name of the monitoring type
-        :param dict condDict: conditions for the query
-
-                       * key -> name of the field
-                       * value -> list of possible values
-        """
-
-        return self.__db.getLastDayData(typeName, condDict)
-
     types_getLimitedDat = [str, dict, int]
 
     def export_getLimitedData(self, typeName, condDict, size):