DIRAC 9.0.0a64__py3-none-any.whl → 9.0.0a67__py3-none-any.whl

DIRAC/Core/Utilities/TimeUtilities.py

@@ -1,260 +1,17 @@
-"""
-DIRAC TimeUtilities module
-Support for basic Date and Time operations
-based on system datetime module.
-
-It provides common interface to UTC timestamps,
-converter to string types and back.
-
-Useful timedelta constant are also provided to
-define time intervals.
-
-Notice: datetime.timedelta objects allow multiplication and division by interger
-but not by float. Thus:
-
-  - DIRAC.TimeUtilities.second * 1.5             is not allowed
-  - DIRAC.TimeUtilities.second * 3 / 2           is allowed
+"""Backward compatibility wrapper - moved to DIRACCommon
 
-An timeInterval class provides a method to check
-if a give datetime is in the defined interval.
+This module has been moved to DIRACCommon.Core.Utilities.TimeUtilities to avoid
+circular dependencies and allow DiracX to use these utilities without
+triggering DIRAC's global state initialization.
 
+All exports are maintained for backward compatibility.
 """
-import datetime
-import sys
-import time
-
-from DIRAC import gLogger
-
-# Some useful constants for time operations
-microsecond = datetime.timedelta(microseconds=1)
-second = datetime.timedelta(seconds=1)
-minute = datetime.timedelta(minutes=1)
-hour = datetime.timedelta(hours=1)
-day = datetime.timedelta(days=1)
-week = datetime.timedelta(days=7)
-
-
-def timeThis(method):
-    """Function to be used as a decorator for timing other functions/methods"""
-
-    def timed(*args, **kw):
-        """What actually times"""
-        ts = time.time()
-        result = method(*args, **kw)
-        if sys.stdout.isatty():
-            return result
-        te = time.time()
-
-        pre = datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S UTC ")
-
-        try:
-            pre += args[0].log.getName() + "/" + args[0].log.getSubName() + "   TIME: " + args[0].transString
-        except AttributeError:
-            try:
-                pre += args[0].log.getName() + "    TIME: " + args[0].transString
-            except AttributeError:
-                try:
-                    pre += args[0].log.getName() + "/" + args[0].log.getSubName() + "   TIME: "
-                except AttributeError:
-                    pre += "TIME: "
-        except IndexError:
-            pre += "TIME: "
-
-        argsLen = ""
-        if args:
-            try:
-                if isinstance(args[1], (list, dict)):
-                    argsLen = f"arguments len: {len(args[1])}"
-            except IndexError:
-                if kw:
-                    try:
-                        if isinstance(list(list(kw.items())[0])[1], (list, dict)):
-                            argsLen = f"arguments len: {len(list(list(kw.items())[0])[1])}"
-                    except IndexError:
-                        argsLen = ""
-
-        gLogger.info(f"{pre} Exec time ===> function {method.__name__!r} {argsLen} -> {te - ts:2.2f} sec")
-        return result
-
-    return timed
-
-
-def toEpoch(dateTimeObject=None):
-    """
-    Get seconds since epoch. Accepts datetime or date objects
-    """
-    return toEpochMilliSeconds(dateTimeObject) // 1000
+from functools import partial
 
+# Re-export everything from DIRACCommon for backward compatibility
+from DIRACCommon.Core.Utilities.TimeUtilities import *  # noqa: F401, F403
 
-def toEpochMilliSeconds(dateTimeObject=None):
-    """
-    Get milliseconds since epoch
-    """
-    if dateTimeObject is None:
-        dateTimeObject = datetime.datetime.utcnow()
-    if dateTimeObject.resolution == datetime.timedelta(days=1):
-        # Add time information corresponding to midnight UTC if it's a datetime.date
-        dateTimeObject = datetime.datetime.combine(
-            dateTimeObject, datetime.time.min.replace(tzinfo=datetime.timezone.utc)
-        )
-    posixTime = dateTimeObject.replace(tzinfo=datetime.timezone.utc).timestamp()
-    return int(posixTime * 1000)
-
-
-def fromEpoch(epoch):
-    """
-    Get datetime object from epoch
-    """
-    # Check if the timestamp is in milliseconds
-    if epoch > 10**17:  # nanoseconds
-        epoch /= 1000**3
-    elif epoch > 10**14:  # microseconds
-        epoch /= 1000**2
-    elif epoch > 10**11:  # milliseconds
-        epoch /= 1000
-    return datetime.datetime.utcfromtimestamp(epoch)
-
-
-def toString(myDate=None):
-    """
-    Convert to String
-    if argument type is neither _dateTimeType, _dateType, nor _timeType
-    the current dateTime converted to String is returned instead
-
-    Notice: datetime.timedelta are converted to strings using the format:
-      [day] days [hour]:[min]:[sec]:[microsec]
-      where hour, min, sec, microsec are always positive integers,
-      and day carries the sign.
-    To keep internal consistency we are using:
-      [hour]:[min]:[sec]:[microsec]
-      where min, sec, microsec are always positive integers and hour carries the sign.
-    """
-    if isinstance(myDate, datetime.date):
-        return str(myDate)
-
-    elif isinstance(myDate, datetime.time):
-        return "%02d:%02d:%02d.%06d" % (
-            myDate.days * 24 + myDate.seconds / 3600,
-            myDate.seconds % 3600 / 60,
-            myDate.seconds % 60,
-            myDate.microseconds,
-        )
-    else:
-        return toString(datetime.datetime.utcnow())
-
-
-def fromString(myDate=None):
-    """
-    Convert date/time/datetime String back to appropriated objects
-
-    The format of the string it is assume to be that returned by toString method.
-    See notice on toString method
-    On Error, return None
-
-    :param myDate: the date string to be converted
-    :type myDate: str or datetime.datetime
-    """
-    if isinstance(myDate, datetime.datetime):
-        return myDate
-    if isinstance(myDate, str):
-        if myDate.find(" ") > 0:
-            dateTimeTuple = myDate.split(" ")
-            dateTuple = dateTimeTuple[0].split("-")
-            try:
-                return datetime.datetime(year=dateTuple[0], month=dateTuple[1], day=dateTuple[2]) + fromString(
-                    dateTimeTuple[1]
-                )
-                # return datetime.datetime.utcnow().combine( fromString( dateTimeTuple[0] ),
-                #                                   fromString( dateTimeTuple[1] ) )
-            except Exception:
-                try:
-                    return datetime.datetime(
-                        year=int(dateTuple[0]), month=int(dateTuple[1]), day=int(dateTuple[2])
-                    ) + fromString(dateTimeTuple[1])
-                except ValueError:
-                    return None
-                # return datetime.datetime.utcnow().combine( fromString( dateTimeTuple[0] ),
-                #                                   fromString( dateTimeTuple[1] ) )
-        elif myDate.find(":") > 0:
-            timeTuple = myDate.replace(".", ":").split(":")
-            try:
-                if len(timeTuple) == 4:
-                    return datetime.timedelta(
-                        hours=int(timeTuple[0]),
-                        minutes=int(timeTuple[1]),
-                        seconds=int(timeTuple[2]),
-                        microseconds=int(timeTuple[3]),
-                    )
-                elif len(timeTuple) == 3:
-                    try:
-                        return datetime.timedelta(
-                            hours=int(timeTuple[0]),
-                            minutes=int(timeTuple[1]),
-                            seconds=int(timeTuple[2]),
-                            microseconds=0,
-                        )
-                    except ValueError:
-                        return None
-                else:
-                    return None
-            except Exception:
-                return None
-        elif myDate.find("-") > 0:
-            dateTuple = myDate.split("-")
-            try:
-                return datetime.date(int(dateTuple[0]), int(dateTuple[1]), int(dateTuple[2]))
-            except Exception:
-                return None
-
-    return None
-
-
-class timeInterval:
-    """
-    Simple class to define a timeInterval object able to check if a given
-    dateTime is inside
-    """
-
-    def __init__(self, initialDateTime, intervalTimeDelta):
-        """
-        Initialization method, it requires the initial dateTime and the
-        timedelta that define the limits.
-        The upper limit is not included thus it is [begin,end)
-        If not properly initialized an error flag is set, and subsequent calls
-        to any method will return None
-        """
-        if not isinstance(initialDateTime, datetime.datetime) or not isinstance(intervalTimeDelta, datetime.timedelta):
-            self.__error = True
-            return None
-        self.__error = False
-        if intervalTimeDelta.days < 0:
-            self.__startDateTime = initialDateTime + intervalTimeDelta
-            self.__endDateTime = initialDateTime
-        else:
-            self.__startDateTime = initialDateTime
-            self.__endDateTime = initialDateTime + intervalTimeDelta
-
-    def includes(self, myDateTime):
-        """ """
-        if self.__error:
-            return None
-        if not isinstance(myDateTime, datetime.datetime):
-            return None
-        if myDateTime < self.__startDateTime:
-            return False
-        if myDateTime >= self.__endDateTime:
-            return False
-        return True
-
-
-def queryTime(f):
-    """Decorator to measure the function call time"""
+from DIRAC import gLogger
 
-    def measureQueryTime(*args, **kwargs):
-        start = time.time()
-        result = f(*args, **kwargs)
-        if result["OK"] and "QueryTime" not in result:
-            result["QueryTime"] = time.time() - start
-        return result
 
-    return measureQueryTime
+timeThis = partial(timeThis, logger_info=gLogger.info)

DIRAC/Core/Utilities/test/Test_JDL.py

@@ -19,9 +19,6 @@ def jdl_monkey_business(monkeypatch):
     monkeypatch.setattr("DIRAC.Core.Base.API.getSites", lambda: S_OK(["LCG.IN2P3.fr"]))
     monkeypatch.setattr("DIRAC.WorkloadManagementSystem.Utilities.JobModel.getSites", lambda: S_OK(["LCG.IN2P3.fr"]))
     monkeypatch.setattr("DIRAC.Interfaces.API.Job.getDIRACPlatforms", lambda: S_OK("x86_64-slc6-gcc49-opt"))
-    monkeypatch.setattr(
-        "DIRAC.WorkloadManagementSystem.Utilities.JobModel.getDIRACPlatforms", lambda: S_OK("x86_64-slc6-gcc49-opt")
-    )
     yield
 
 

DIRAC/Core/scripts/dirac_agent.py

@@ -25,7 +25,7 @@ def main():
     localCfg.setConfigurationForAgent(agentName)
     localCfg.addDefaultEntry("/DIRAC/Security/UseServerCertificate", "yes")
     localCfg.addDefaultEntry("LogColor", True)
-    resultDict = localCfg.loadUserData()
+    resultDict = localCfg.loadUserData(requireSuccessfulSync=True)
     if not resultDict["OK"]:
         gLogger.error("There were errors when loading configuration", resultDict["Message"])
         sys.exit(1)

DIRAC/DataManagementSystem/DB/FTS3DB.py

@@ -16,6 +16,7 @@ from sqlalchemy import (
     Enum,
     Float,
     ForeignKey,
+    Index,
     Integer,
     MetaData,
     SmallInteger,
@@ -85,6 +86,7 @@ fts3JobTable = Table(
     Column("error", String(2048)),
     Column("status", Enum(*FTS3Job.ALL_STATES), server_default=FTS3Job.INIT_STATE, index=True),
     Column("assignment", String(255), server_default=None),
+    Index("idx_jobs_lastupdate_assignment", "lastUpdate", "assignment"),
     mysql_engine="InnoDB",
 )
 
@@ -110,6 +112,7 @@ fts3OperationTable = Table(
     Column("error", String(1024)),
     Column("type", String(255)),
     Column("assignment", String(255), server_default=None),
+    Index("idx_operations_lastupdate_assignment", "lastUpdate", "assignment"),
     mysql_engine="InnoDB",
 )
 

DIRAC/RequestManagementSystem/DB/test/RMSTestScenari.py

@@ -6,6 +6,7 @@ integration tests (Test_ReqDB.py)
 # pylint: disable=invalid-name,wrong-import-position
 import time
 
+import pytest
 
 from DIRAC.RequestManagementSystem.Client.Request import Request
 from DIRAC.RequestManagementSystem.Client.Operation import Operation
@@ -42,6 +43,7 @@ def test_stress(reqDB):
         assert delete["OK"], delete
 
 
+@pytest.mark.slow
 def test_stressBulk(reqDB):
     """stress test bulk"""
 

DIRAC/Resources/Catalog/RucioFileCatalogClient.py

@@ -151,7 +151,7 @@ class RucioFileCatalogClient(FileCatalogClientBase):
             self.authHost = options.get("AuthHost", None)
             self.caCertPath = Locations.getCAsLocation()
             try:
-                sLog.info(f"Logging in with a proxy located at: {self.proxyPath}")
+                sLog.debug(f"Logging in with a proxy located at: {self.proxyPath}")
                 sLog.debug("account: ", self.username)
                 sLog.debug("rucio host: ", self.rucioHost)
                 sLog.debug("auth  host: ", self.authHost)

DIRAC/Resources/Computing/test/Test_PoolComputingElement.py

@@ -25,7 +25,7 @@ while True:
   if os.path.isfile(stopFile):
     os.remove(stopFile)
     break
-  if (time.time() - start) > 30:
+  if (time.time() - start) > 5:
     break
 print("End job", jobNumber, time.time())
 """
@@ -262,6 +262,7 @@ def test_executeJob_wholeNode8(createAndDelete):
             assert "Not enough processors" in submissionResult["Message"]
 
 
+@pytest.mark.slow
 def test_executeJob_submitAndStop(createAndDelete):
     time.sleep(0.5)
 

DIRAC/TransformationSystem/Agent/TransformationCleaningAgent.py

@@ -37,7 +37,7 @@ from DIRAC.WorkloadManagementSystem.Service.JobPolicy import (
     RIGHT_DELETE,
     RIGHT_KILL,
 )
-from DIRAC.WorkloadManagementSystem.Utilities.jobAdministration import kill_delete_jobs
+from DIRAC.WorkloadManagementSystem.DB.StatusUtils import kill_delete_jobs
 
 # # agent's name
 AGENT_NAME = "Transformation/TransformationCleaningAgent"

DIRAC/Workflow/Modules/test/Test_Modules.py

@@ -5,6 +5,8 @@ import os
 import copy
 import shutil
 
+import pytest
+
 from unittest.mock import MagicMock as Mock
 
 from DIRAC import gLogger
@@ -742,6 +744,7 @@ class FailoverRequestSuccess(ModulesTestCase):
 class ScriptSuccess(ModulesTestCase):
     #################################################
 
+    @pytest.mark.slow
     def test_execute(self):
         self.script.jobType = "merge"
         self.script.stepInputData = ["foo", "bar"]
@@ -770,6 +773,7 @@ class ScriptSuccess(ModulesTestCase):
 class ScriptUnicode(ModulesTestCase):
     #################################################
 
+    @pytest.mark.slow
     def test_execute(self):
         self.script.jobType = "merge"
         self.script.stepInputData = ["foo", "bar"]
@@ -799,6 +803,7 @@ class ScriptUnicode(ModulesTestCase):
 class ScriptFailure(ModulesTestCase):
     #################################################
 
+    @pytest.mark.slow
     def test_execute(self):
         self.script.jobType = "merge"
         self.script.stepInputData = ["foo", "bar"]

DIRAC/WorkloadManagementSystem/Agent/JobCleaningAgent.py

@@ -39,7 +39,7 @@ from DIRAC.WorkloadManagementSystem.Client.WMSClient import WMSClient
 from DIRAC.WorkloadManagementSystem.DB.JobDB import JobDB
 from DIRAC.WorkloadManagementSystem.DB.SandboxMetadataDB import SandboxMetadataDB
 from DIRAC.WorkloadManagementSystem.Service.JobPolicy import RIGHT_DELETE
-from DIRAC.WorkloadManagementSystem.Utilities.jobAdministration import kill_delete_jobs
+from DIRAC.WorkloadManagementSystem.DB.StatusUtils import kill_delete_jobs
 from DIRAC.WorkloadManagementSystem.Utilities.JobParameters import getJobParameters
 
 

DIRAC/WorkloadManagementSystem/Agent/StalledJobAgent.py

@@ -23,7 +23,7 @@ from DIRAC.WorkloadManagementSystem.DB.JobDB import JobDB
 from DIRAC.WorkloadManagementSystem.DB.JobLoggingDB import JobLoggingDB
 from DIRAC.WorkloadManagementSystem.DB.PilotAgentsDB import PilotAgentsDB
 from DIRAC.WorkloadManagementSystem.Service.JobPolicy import RIGHT_KILL
-from DIRAC.WorkloadManagementSystem.Utilities.jobAdministration import kill_delete_jobs
+from DIRAC.WorkloadManagementSystem.DB.StatusUtils import kill_delete_jobs
 from DIRAC.WorkloadManagementSystem.Utilities.JobParameters import getJobParameters
 from DIRAC.WorkloadManagementSystem.Utilities.Utils import rescheduleJobs
 

DIRAC/WorkloadManagementSystem/Agent/test/Test_Agent_JobAgent.py

@@ -260,6 +260,7 @@ def test__checkMatcherInfo(mocker, matcherInfo, matcherParams, expectedResult):
 #############################################################################
 
 
+@pytest.mark.slow
 @pytest.mark.parametrize(
     "mockGCReply, mockPMReply, expected",
     [
@@ -308,6 +309,7 @@ def test__setupProxy(mocker, mockGCReply, mockPMReply, expected):
         assert result["Message"] == expected["Message"]
 
 
+@pytest.mark.slow
 @pytest.mark.parametrize(
     "mockGCReply, mockPMReply, expected",
     [

DIRAC/WorkloadManagementSystem/Agent/test/Test_Agent_PushJobAgent.py

@@ -178,6 +178,7 @@ def jobID():
     shutil.rmtree(jobID)
 
 
+@pytest.mark.slow
 def test_submitJobWrapper(mocker, jobID):
     """Test JobAgent._submitJobWrapper()"""
     mocker.patch("DIRAC.WorkloadManagementSystem.Agent.JobAgent.AgentModule.__init__")