DIRAC 9.0.0a66__py3-none-any.whl → 9.0.0a67__py3-none-any.whl

DIRAC/Core/Utilities/StateMachine.py

@@ -1,185 +1,19 @@
-""" StateMachine
+"""Backward compatibility wrapper - moved to DIRACCommon
 
-  This module contains the basic blocks to build a state machine (State and StateMachine)
-"""
-from DIRAC import S_OK, S_ERROR, gLogger
-
-
-class State:
-    """
-    State class that represents a single step on a StateMachine, with all the
-    possible transitions, the default transition and an ordering level.
-
-
-    examples:
-      >>> s0 = State(100)
-      >>> s1 = State(0, ['StateName1', 'StateName2'], defState='StateName1')
-      >>> s2 = State(0, ['StateName1', 'StateName2'])
-      # this example is tricky. The transition rule says that will go to
-      # nextState, e.g. 'StateNext'. But, it is not on the stateMap, and there
-      # is no default defined, so it will end up going to StateNext anyway. You
-      # must be careful while defining states and their stateMaps and defaults.
-    """
-
-    def __init__(self, level, stateMap=None, defState=None):
-        """
-        :param int level: each state is mapped to an integer, which is used to sort the states according to that integer.
-        :param list stateMap: it is a list (of strings) with the reachable states from this particular status.
-                              If not defined, we assume there are no restrictions.
-        :param str defState: default state used in case the next state is not in stateMap (not defined or simply not there).
-        """
-
-        self.level = level
-        self.stateMap = stateMap if stateMap else []
-        self.default = defState
-
-    def transitionRule(self, nextState):
-        """
-        Method that selects next state, knowing the default and the transitions
-        map, and the proposed next state. If <nextState> is in stateMap, goes there.
-        If not, then goes to <self.default> if any. Otherwise, goes to <nextState>
-        anyway.
-
-        examples:
-          >>> s0.transitionRule('nextState')
-              'nextState'
-          >>> s1.transitionRule('StateName2')
-              'StateName2'
-          >>> s1.transitionRule('StateNameNotInMap')
-              'StateName1'
-          >>> s2.transitionRule('StateNameNotInMap')
-              'StateNameNotInMap'
-
-        :param str nextState: name of the state in the stateMap
-        :return: state name
-        :rtype: str
-        """
-
-        # If next state is on the list of next states, go ahead.
-        if nextState in self.stateMap:
-            return nextState
-
-        # If not, calculate defaultState:
-        # if there is a default, that one
-        # otherwise is nextState (states with empty list have no movement restrictions)
-        defaultNext = self.default if self.default else nextState
-        return defaultNext
-
-
-class StateMachine:
-    """
-    StateMachine class that represents the whole state machine with all transitions.
-
-    examples:
-      >>> sm0 = StateMachine()
-      >>> sm1 = StateMachine(state = 'Active')
-
-    :param state: current state of the StateMachine, could be None if we do not use the
-        StateMachine to calculate transitions. Beware, it is not checked if the
-        state is on the states map !
-    :type state: None or str
-
-    """
+This module has been moved to DIRACCommon.Core.Utilities.StateMachine to avoid
+circular dependencies and allow DiracX to use these utilities without
+triggering DIRAC's global state initialization.
 
-    def __init__(self, state=None):
-        """
-        Constructor.
-        """
-
-        self.state = state
-        # To be overwritten by child classes, unless you like Nirvana state that much.
-        self.states = {"Nirvana": State(100)}
-
-    def getLevelOfState(self, state):
-        """
-        Given a state name, it returns its level (integer), which defines the hierarchy.
+All exports are maintained for backward compatibility.
+"""
+# Re-export everything from DIRACCommon for backward compatibility
+from DIRACCommon.Core.Utilities.StateMachine import *  # noqa: F401, F403
 
-        >>> sm0.getLevelOfState('Nirvana')
-            100
-        >>> sm0.getLevelOfState('AnotherState')
-            -1
+from DIRAC import gLogger
 
-        :param str state: name of the state, it should be on <self.states> key set
-        :return: `int` || -1 (if not in <self.states>)
-        """
 
-        if state not in self.states:
-            return -1
-        return self.states[state].level
+class StateMachine(StateMachine):  # noqa: F405 pylint: disable=function-redefined
+    """Backward compatibility wrapper - moved to DIRACCommon"""
 
     def setState(self, candidateState, noWarn=False):
-        """Makes sure the state is either None or known to the machine, and that it is a valid state to move into.
-            Final states are also checked.
-
-        examples:
-          >>> sm0.setState(None)['OK']
-              True
-          >>> sm0.setState('Nirvana')['OK']
-              True
-          >>> sm0.setState('AnotherState')['OK']
-              False
-
-        :param state: state which will be set as current state of the StateMachine
-        :type state: None or str
-        :return: S_OK || S_ERROR
-        """
-        if candidateState == self.state:
-            return S_OK(candidateState)
-
-        if not candidateState:
-            self.state = candidateState
-        elif candidateState in self.states:
-            if not self.states[self.state].stateMap:
-                if not noWarn:
-                    gLogger.warn("Final state, won't move", f"({self.state}, asked to move to {candidateState})")
-                return S_OK(self.state)
-            if candidateState not in self.states[self.state].stateMap:
-                gLogger.warn(f"Can't move from {self.state} to {candidateState}, choosing a good one")
-            result = self.getNextState(candidateState)
-            if not result["OK"]:
-                return result
-            self.state = result["Value"]
-            # If the StateMachine does not accept the candidate, return error message
-        else:
-            return S_ERROR(f"setState: {candidateState!r} is not a valid state")
-
-        return S_OK(self.state)
-
-    def getStates(self):
-        """
-        Returns all possible states in the state map
-
-        examples:
-          >>> sm0.getStates()
-              [ 'Nirvana' ]
-
-        :return: list(stateNames)
-        """
-
-        return list(self.states)
-
-    def getNextState(self, candidateState):
-        """
-        Method that gets the next state, given the proposed transition to candidateState.
-        If candidateState is not on the state map <self.states>, it is rejected. If it is
-        not the case, we have two options: if <self.state> is None, then the next state
-        will be <candidateState>. Otherwise, the current state is using its own
-        transition rule to decide.
-
-        examples:
-          >>> sm0.getNextState(None)
-              S_OK(None)
-          >>> sm0.getNextState('NextState')
-              S_OK('NextState')
-
-        :param str candidateState: name of the next state
-        :return: S_OK(nextState) || S_ERROR
-        """
-        if candidateState not in self.states:
-            return S_ERROR(f"getNextState: {candidateState!r} is not a valid state")
-
-        # FIXME: do we need this anymore ?
-        if self.state is None:
-            return S_OK(candidateState)
-
-        return S_OK(self.states[self.state].transitionRule(candidateState))
+        return super().setState(candidateState, noWarn, logger_warn=gLogger.warn)

DIRAC/Core/Utilities/TimeUtilities.py

@@ -1,260 +1,17 @@
-"""
-DIRAC TimeUtilities module
-Support for basic Date and Time operations
-based on system datetime module.
-
-It provides common interface to UTC timestamps,
-converter to string types and back.
-
-Useful timedelta constant are also provided to
-define time intervals.
-
-Notice: datetime.timedelta objects allow multiplication and division by interger
-but not by float. Thus:
-
-  - DIRAC.TimeUtilities.second * 1.5             is not allowed
-  - DIRAC.TimeUtilities.second * 3 / 2           is allowed
+"""Backward compatibility wrapper - moved to DIRACCommon
 
-An timeInterval class provides a method to check
-if a give datetime is in the defined interval.
+This module has been moved to DIRACCommon.Core.Utilities.TimeUtilities to avoid
+circular dependencies and allow DiracX to use these utilities without
+triggering DIRAC's global state initialization.
 
+All exports are maintained for backward compatibility.
 """
-import datetime
-import sys
-import time
-
-from DIRAC import gLogger
-
-# Some useful constants for time operations
-microsecond = datetime.timedelta(microseconds=1)
-second = datetime.timedelta(seconds=1)
-minute = datetime.timedelta(minutes=1)
-hour = datetime.timedelta(hours=1)
-day = datetime.timedelta(days=1)
-week = datetime.timedelta(days=7)
-
-
-def timeThis(method):
-    """Function to be used as a decorator for timing other functions/methods"""
-
-    def timed(*args, **kw):
-        """What actually times"""
-        ts = time.time()
-        result = method(*args, **kw)
-        if sys.stdout.isatty():
-            return result
-        te = time.time()
-
-        pre = datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S UTC ")
-
-        try:
-            pre += args[0].log.getName() + "/" + args[0].log.getSubName() + "   TIME: " + args[0].transString
-        except AttributeError:
-            try:
-                pre += args[0].log.getName() + "    TIME: " + args[0].transString
-            except AttributeError:
-                try:
-                    pre += args[0].log.getName() + "/" + args[0].log.getSubName() + "   TIME: "
-                except AttributeError:
-                    pre += "TIME: "
-        except IndexError:
-            pre += "TIME: "
-
-        argsLen = ""
-        if args:
-            try:
-                if isinstance(args[1], (list, dict)):
-                    argsLen = f"arguments len: {len(args[1])}"
-            except IndexError:
-                if kw:
-                    try:
-                        if isinstance(list(list(kw.items())[0])[1], (list, dict)):
-                            argsLen = f"arguments len: {len(list(list(kw.items())[0])[1])}"
-                    except IndexError:
-                        argsLen = ""
-
-        gLogger.info(f"{pre} Exec time ===> function {method.__name__!r} {argsLen} -> {te - ts:2.2f} sec")
-        return result
-
-    return timed
-
-
-def toEpoch(dateTimeObject=None):
-    """
-    Get seconds since epoch. Accepts datetime or date objects
-    """
-    return toEpochMilliSeconds(dateTimeObject) // 1000
+from functools import partial
 
+# Re-export everything from DIRACCommon for backward compatibility
+from DIRACCommon.Core.Utilities.TimeUtilities import *  # noqa: F401, F403
 
-def toEpochMilliSeconds(dateTimeObject=None):
-    """
-    Get milliseconds since epoch
-    """
-    if dateTimeObject is None:
-        dateTimeObject = datetime.datetime.utcnow()
-    if dateTimeObject.resolution == datetime.timedelta(days=1):
-        # Add time information corresponding to midnight UTC if it's a datetime.date
-        dateTimeObject = datetime.datetime.combine(
-            dateTimeObject, datetime.time.min.replace(tzinfo=datetime.timezone.utc)
-        )
-    posixTime = dateTimeObject.replace(tzinfo=datetime.timezone.utc).timestamp()
-    return int(posixTime * 1000)
-
-
-def fromEpoch(epoch):
-    """
-    Get datetime object from epoch
-    """
-    # Check if the timestamp is in milliseconds
-    if epoch > 10**17:  # nanoseconds
-        epoch /= 1000**3
-    elif epoch > 10**14:  # microseconds
-        epoch /= 1000**2
-    elif epoch > 10**11:  # milliseconds
-        epoch /= 1000
-    return datetime.datetime.utcfromtimestamp(epoch)
-
-
-def toString(myDate=None):
-    """
-    Convert to String
-    if argument type is neither _dateTimeType, _dateType, nor _timeType
-    the current dateTime converted to String is returned instead
-
-    Notice: datetime.timedelta are converted to strings using the format:
-      [day] days [hour]:[min]:[sec]:[microsec]
-      where hour, min, sec, microsec are always positive integers,
-      and day carries the sign.
-    To keep internal consistency we are using:
-      [hour]:[min]:[sec]:[microsec]
-      where min, sec, microsec are always positive integers and hour carries the sign.
-    """
-    if isinstance(myDate, datetime.date):
-        return str(myDate)
-
-    elif isinstance(myDate, datetime.time):
-        return "%02d:%02d:%02d.%06d" % (
-            myDate.days * 24 + myDate.seconds / 3600,
-            myDate.seconds % 3600 / 60,
-            myDate.seconds % 60,
-            myDate.microseconds,
-        )
-    else:
-        return toString(datetime.datetime.utcnow())
-
-
-def fromString(myDate=None):
-    """
-    Convert date/time/datetime String back to appropriated objects
-
-    The format of the string it is assume to be that returned by toString method.
-    See notice on toString method
-    On Error, return None
-
-    :param myDate: the date string to be converted
-    :type myDate: str or datetime.datetime
-    """
-    if isinstance(myDate, datetime.datetime):
-        return myDate
-    if isinstance(myDate, str):
-        if myDate.find(" ") > 0:
-            dateTimeTuple = myDate.split(" ")
-            dateTuple = dateTimeTuple[0].split("-")
-            try:
-                return datetime.datetime(year=dateTuple[0], month=dateTuple[1], day=dateTuple[2]) + fromString(
-                    dateTimeTuple[1]
-                )
-                # return datetime.datetime.utcnow().combine( fromString( dateTimeTuple[0] ),
-                #                                   fromString( dateTimeTuple[1] ) )
-            except Exception:
-                try:
-                    return datetime.datetime(
-                        year=int(dateTuple[0]), month=int(dateTuple[1]), day=int(dateTuple[2])
-                    ) + fromString(dateTimeTuple[1])
-                except ValueError:
-                    return None
-                # return datetime.datetime.utcnow().combine( fromString( dateTimeTuple[0] ),
-                #                                   fromString( dateTimeTuple[1] ) )
-        elif myDate.find(":") > 0:
-            timeTuple = myDate.replace(".", ":").split(":")
-            try:
-                if len(timeTuple) == 4:
-                    return datetime.timedelta(
-                        hours=int(timeTuple[0]),
-                        minutes=int(timeTuple[1]),
-                        seconds=int(timeTuple[2]),
-                        microseconds=int(timeTuple[3]),
-                    )
-                elif len(timeTuple) == 3:
-                    try:
-                        return datetime.timedelta(
-                            hours=int(timeTuple[0]),
-                            minutes=int(timeTuple[1]),
-                            seconds=int(timeTuple[2]),
-                            microseconds=0,
-                        )
-                    except ValueError:
-                        return None
-                else:
-                    return None
-            except Exception:
-                return None
-        elif myDate.find("-") > 0:
-            dateTuple = myDate.split("-")
-            try:
-                return datetime.date(int(dateTuple[0]), int(dateTuple[1]), int(dateTuple[2]))
-            except Exception:
-                return None
-
-    return None
-
-
-class timeInterval:
-    """
-    Simple class to define a timeInterval object able to check if a given
-    dateTime is inside
-    """
-
-    def __init__(self, initialDateTime, intervalTimeDelta):
-        """
-        Initialization method, it requires the initial dateTime and the
-        timedelta that define the limits.
-        The upper limit is not included thus it is [begin,end)
-        If not properly initialized an error flag is set, and subsequent calls
-        to any method will return None
-        """
-        if not isinstance(initialDateTime, datetime.datetime) or not isinstance(intervalTimeDelta, datetime.timedelta):
-            self.__error = True
-            return None
-        self.__error = False
-        if intervalTimeDelta.days < 0:
-            self.__startDateTime = initialDateTime + intervalTimeDelta
-            self.__endDateTime = initialDateTime
-        else:
-            self.__startDateTime = initialDateTime
-            self.__endDateTime = initialDateTime + intervalTimeDelta
-
-    def includes(self, myDateTime):
-        """ """
-        if self.__error:
-            return None
-        if not isinstance(myDateTime, datetime.datetime):
-            return None
-        if myDateTime < self.__startDateTime:
-            return False
-        if myDateTime >= self.__endDateTime:
-            return False
-        return True
-
-
-def queryTime(f):
-    """Decorator to measure the function call time"""
+from DIRAC import gLogger
 
-    def measureQueryTime(*args, **kwargs):
-        start = time.time()
-        result = f(*args, **kwargs)
-        if result["OK"] and "QueryTime" not in result:
-            result["QueryTime"] = time.time() - start
-        return result
 
-    return measureQueryTime
+timeThis = partial(timeThis, logger_info=gLogger.info)

DIRAC/Core/Utilities/test/Test_JDL.py

@@ -19,9 +19,6 @@ def jdl_monkey_business(monkeypatch):
     monkeypatch.setattr("DIRAC.Core.Base.API.getSites", lambda: S_OK(["LCG.IN2P3.fr"]))
     monkeypatch.setattr("DIRAC.WorkloadManagementSystem.Utilities.JobModel.getSites", lambda: S_OK(["LCG.IN2P3.fr"]))
     monkeypatch.setattr("DIRAC.Interfaces.API.Job.getDIRACPlatforms", lambda: S_OK("x86_64-slc6-gcc49-opt"))
-    monkeypatch.setattr(
-        "DIRAC.WorkloadManagementSystem.Utilities.JobModel.getDIRACPlatforms", lambda: S_OK("x86_64-slc6-gcc49-opt")
-    )
     yield
 
 

DIRAC/DataManagementSystem/DB/FTS3DB.py

@@ -16,6 +16,7 @@ from sqlalchemy import (
     Enum,
     Float,
     ForeignKey,
+    Index,
     Integer,
     MetaData,
     SmallInteger,
@@ -85,6 +86,7 @@ fts3JobTable = Table(
     Column("error", String(2048)),
     Column("status", Enum(*FTS3Job.ALL_STATES), server_default=FTS3Job.INIT_STATE, index=True),
     Column("assignment", String(255), server_default=None),
+    Index("idx_jobs_lastupdate_assignment", "lastUpdate", "assignment"),
     mysql_engine="InnoDB",
 )
 
@@ -110,6 +112,7 @@ fts3OperationTable = Table(
     Column("error", String(1024)),
     Column("type", String(255)),
     Column("assignment", String(255), server_default=None),
+    Index("idx_operations_lastupdate_assignment", "lastUpdate", "assignment"),
     mysql_engine="InnoDB",
 )
 

DIRAC/RequestManagementSystem/DB/test/RMSTestScenari.py

@@ -6,6 +6,7 @@ integration tests (Test_ReqDB.py)
 # pylint: disable=invalid-name,wrong-import-position
 import time
 
+import pytest
 
 from DIRAC.RequestManagementSystem.Client.Request import Request
 from DIRAC.RequestManagementSystem.Client.Operation import Operation
@@ -42,6 +43,7 @@ def test_stress(reqDB):
         assert delete["OK"], delete
 
 
+@pytest.mark.slow
 def test_stressBulk(reqDB):
     """stress test bulk"""
 

DIRAC/Resources/Catalog/RucioFileCatalogClient.py

@@ -151,7 +151,7 @@ class RucioFileCatalogClient(FileCatalogClientBase):
             self.authHost = options.get("AuthHost", None)
             self.caCertPath = Locations.getCAsLocation()
             try:
-                sLog.info(f"Logging in with a proxy located at: {self.proxyPath}")
+                sLog.debug(f"Logging in with a proxy located at: {self.proxyPath}")
                 sLog.debug("account: ", self.username)
                 sLog.debug("rucio host: ", self.rucioHost)
                 sLog.debug("auth  host: ", self.authHost)

DIRAC/Resources/Computing/test/Test_PoolComputingElement.py

@@ -25,7 +25,7 @@ while True:
   if os.path.isfile(stopFile):
     os.remove(stopFile)
     break
-  if (time.time() - start) > 30:
+  if (time.time() - start) > 5:
     break
 print("End job", jobNumber, time.time())
 """
@@ -262,6 +262,7 @@ def test_executeJob_wholeNode8(createAndDelete):
             assert "Not enough processors" in submissionResult["Message"]
 
 
+@pytest.mark.slow
 def test_executeJob_submitAndStop(createAndDelete):
     time.sleep(0.5)
 

DIRAC/Workflow/Modules/test/Test_Modules.py

@@ -5,6 +5,8 @@ import os
 import copy
 import shutil
 
+import pytest
+
 from unittest.mock import MagicMock as Mock
 
 from DIRAC import gLogger
@@ -742,6 +744,7 @@ class FailoverRequestSuccess(ModulesTestCase):
 class ScriptSuccess(ModulesTestCase):
     #################################################
 
+    @pytest.mark.slow
     def test_execute(self):
         self.script.jobType = "merge"
         self.script.stepInputData = ["foo", "bar"]
@@ -770,6 +773,7 @@ class ScriptSuccess(ModulesTestCase):
 class ScriptUnicode(ModulesTestCase):
     #################################################
 
+    @pytest.mark.slow
     def test_execute(self):
         self.script.jobType = "merge"
         self.script.stepInputData = ["foo", "bar"]
@@ -799,6 +803,7 @@ class ScriptUnicode(ModulesTestCase):
 class ScriptFailure(ModulesTestCase):
     #################################################
 
+    @pytest.mark.slow
     def test_execute(self):
         self.script.jobType = "merge"
         self.script.stepInputData = ["foo", "bar"]

DIRAC/WorkloadManagementSystem/Agent/test/Test_Agent_JobAgent.py

@@ -260,6 +260,7 @@ def test__checkMatcherInfo(mocker, matcherInfo, matcherParams, expectedResult):
 #############################################################################
 
 
+@pytest.mark.slow
 @pytest.mark.parametrize(
     "mockGCReply, mockPMReply, expected",
     [
@@ -308,6 +309,7 @@ def test__setupProxy(mocker, mockGCReply, mockPMReply, expected):
         assert result["Message"] == expected["Message"]
 
 
+@pytest.mark.slow
 @pytest.mark.parametrize(
     "mockGCReply, mockPMReply, expected",
     [

DIRAC/WorkloadManagementSystem/Agent/test/Test_Agent_PushJobAgent.py

@@ -178,6 +178,7 @@ def jobID():
     shutil.rmtree(jobID)
 
 
+@pytest.mark.slow
 def test_submitJobWrapper(mocker, jobID):
     """Test JobAgent._submitJobWrapper()"""
     mocker.patch("DIRAC.WorkloadManagementSystem.Agent.JobAgent.AgentModule.__init__")