DIRAC 9.0.0a66__py3-none-any.whl → 9.0.0a68__py3-none-any.whl

DIRAC/Core/Utilities/List.py

@@ -1,127 +1 @@
-"""Collection of DIRAC useful list related modules.
-   By default on Error they return None.
-"""
-import random
-import sys
-from typing import Any, TypeVar
-from collections.abc import Iterable
-
-T = TypeVar("T")
-
-
-def uniqueElements(aList: list) -> list:
-    """Utility to retrieve list of unique elements in a list (order is kept)."""
-
-    # Use dict.fromkeys instead of set ensure the order is preserved
-    return list(dict.fromkeys(aList))
-
-
-def appendUnique(aList: list, anObject: Any):
-    """Append to list if object does not exist.
-
-    :param aList: list of elements
-    :param anObject: object you want to append
-    """
-    if anObject not in aList:
-        aList.append(anObject)
-
-
-def fromChar(inputString: str, sepChar: str = ","):
-    """Generates a list splitting a string by the required character(s)
-    resulting string items are stripped and empty items are removed.
-
-    :param inputString: list serialised to string
-    :param sepChar: separator
-    :return: list of strings or None if sepChar has a wrong type
-    """
-    # to prevent getting an empty String as argument
-    if not (isinstance(inputString, str) and isinstance(sepChar, str) and sepChar):
-        return None
-    return [fieldString.strip() for fieldString in inputString.split(sepChar) if len(fieldString.strip()) > 0]
-
-
-def randomize(aList: Iterable[T]) -> list[T]:
-    """Return a randomly sorted list.
-
-    :param aList: list to permute
-    """
-    tmpList = list(aList)
-    random.shuffle(tmpList)
-    return tmpList
-
-
-def pop(aList, popElement):
-    """Pop the first element equal to popElement from the list.
-
-    :param aList: list
-    :type aList: python:list
-    :param popElement: element to pop
-    """
-    if popElement in aList:
-        return aList.pop(aList.index(popElement))
-
-
-def stringListToString(aList: list) -> str:
-    """This function is used for making MySQL queries with a list of string elements.
-
-    :param aList: list to be serialized to string for making queries
-    """
-    return ",".join(f"'{x}'" for x in aList)
-
-
-def intListToString(aList: list) -> str:
-    """This function is used for making MySQL queries with a list of int elements.
-
-    :param aList: list to be serialized to string for making queries
-    """
-    return ",".join(str(x) for x in aList)
-
-
-def getChunk(aList: list, chunkSize: int):
-    """Generator yielding chunk from a list of a size chunkSize.
-
-    :param aList: list to be splitted
-    :param chunkSize: lenght of one chunk
-    :raise: StopIteration
-
-    Usage:
-
-    >>> for chunk in getChunk( aList, chunkSize=10):
-          process( chunk )
-
-    """
-    chunkSize = int(chunkSize)
-    for i in range(0, len(aList), chunkSize):
-        yield aList[i : i + chunkSize]
-
-
-def breakListIntoChunks(aList: list, chunkSize: int):
-    """This function takes a list as input and breaks it into list of size 'chunkSize'.
-       It returns a list of lists.
-
-    :param aList: list of elements
-    :param chunkSize: len of a single chunk
-    :return: list of lists of length of chunkSize
-    :raise: RuntimeError if numberOfFilesInChunk is less than 1
-    """
-    if chunkSize < 1:
-        raise RuntimeError("chunkSize cannot be less than 1")
-    if isinstance(aList, (set, dict, tuple, {}.keys().__class__, {}.items().__class__, {}.values().__class__)):
-        aList = list(aList)
-    return [chunk for chunk in getChunk(aList, chunkSize)]
-
-
-def getIndexInList(anItem: Any, aList: list) -> int:
-    """Return the index of the element x in the list l
-    or sys.maxint if it does not exist
-
-    :param anItem: element to look for
-    :param aList: list to look into
-
-    :return: the index or sys.maxint
-    """
-    # try:
-    if anItem in aList:
-        return aList.index(anItem)
-    else:
-        return sys.maxsize
+from DIRACCommon.Core.Utilities.List import *  # noqa: F401,F403

DIRAC/Core/Utilities/ReturnValues.py

@@ -1,10 +1,10 @@
 """Backward compatibility wrapper - moved to DIRACCommon
 
-This module has been moved to DIRACCommon.Utils.ReturnValues to avoid
+This module has been moved to DIRACCommon.Core.Utilities.ReturnValues to avoid
 circular dependencies and allow DiracX to use these utilities without
 triggering DIRAC's global state initialization.
 
 All exports are maintained for backward compatibility.
 """
 # Re-export everything from DIRACCommon for backward compatibility
-from DIRACCommon.Utils.ReturnValues import *  # noqa: F401, F403
+from DIRACCommon.Core.Utilities.ReturnValues import *  # noqa: F401, F403

DIRAC/Core/Utilities/StateMachine.py

@@ -1,185 +1,19 @@
-""" StateMachine
+"""Backward compatibility wrapper - moved to DIRACCommon
 
-  This module contains the basic blocks to build a state machine (State and StateMachine)
-"""
-from DIRAC import S_OK, S_ERROR, gLogger
-
-
-class State:
-    """
-    State class that represents a single step on a StateMachine, with all the
-    possible transitions, the default transition and an ordering level.
-
-
-    examples:
-      >>> s0 = State(100)
-      >>> s1 = State(0, ['StateName1', 'StateName2'], defState='StateName1')
-      >>> s2 = State(0, ['StateName1', 'StateName2'])
-      # this example is tricky. The transition rule says that will go to
-      # nextState, e.g. 'StateNext'. But, it is not on the stateMap, and there
-      # is no default defined, so it will end up going to StateNext anyway. You
-      # must be careful while defining states and their stateMaps and defaults.
-    """
-
-    def __init__(self, level, stateMap=None, defState=None):
-        """
-        :param int level: each state is mapped to an integer, which is used to sort the states according to that integer.
-        :param list stateMap: it is a list (of strings) with the reachable states from this particular status.
-                              If not defined, we assume there are no restrictions.
-        :param str defState: default state used in case the next state is not in stateMap (not defined or simply not there).
-        """
-
-        self.level = level
-        self.stateMap = stateMap if stateMap else []
-        self.default = defState
-
-    def transitionRule(self, nextState):
-        """
-        Method that selects next state, knowing the default and the transitions
-        map, and the proposed next state. If <nextState> is in stateMap, goes there.
-        If not, then goes to <self.default> if any. Otherwise, goes to <nextState>
-        anyway.
-
-        examples:
-          >>> s0.transitionRule('nextState')
-              'nextState'
-          >>> s1.transitionRule('StateName2')
-              'StateName2'
-          >>> s1.transitionRule('StateNameNotInMap')
-              'StateName1'
-          >>> s2.transitionRule('StateNameNotInMap')
-              'StateNameNotInMap'
-
-        :param str nextState: name of the state in the stateMap
-        :return: state name
-        :rtype: str
-        """
-
-        # If next state is on the list of next states, go ahead.
-        if nextState in self.stateMap:
-            return nextState
-
-        # If not, calculate defaultState:
-        # if there is a default, that one
-        # otherwise is nextState (states with empty list have no movement restrictions)
-        defaultNext = self.default if self.default else nextState
-        return defaultNext
-
-
-class StateMachine:
-    """
-    StateMachine class that represents the whole state machine with all transitions.
-
-    examples:
-      >>> sm0 = StateMachine()
-      >>> sm1 = StateMachine(state = 'Active')
-
-    :param state: current state of the StateMachine, could be None if we do not use the
-        StateMachine to calculate transitions. Beware, it is not checked if the
-        state is on the states map !
-    :type state: None or str
-
-    """
+This module has been moved to DIRACCommon.Core.Utilities.StateMachine to avoid
+circular dependencies and allow DiracX to use these utilities without
+triggering DIRAC's global state initialization.
 
-    def __init__(self, state=None):
-        """
-        Constructor.
-        """
-
-        self.state = state
-        # To be overwritten by child classes, unless you like Nirvana state that much.
-        self.states = {"Nirvana": State(100)}
-
-    def getLevelOfState(self, state):
-        """
-        Given a state name, it returns its level (integer), which defines the hierarchy.
+All exports are maintained for backward compatibility.
+"""
+# Re-export everything from DIRACCommon for backward compatibility
+from DIRACCommon.Core.Utilities.StateMachine import *  # noqa: F401, F403
 
-        >>> sm0.getLevelOfState('Nirvana')
-            100
-        >>> sm0.getLevelOfState('AnotherState')
-            -1
+from DIRAC import gLogger
 
-        :param str state: name of the state, it should be on <self.states> key set
-        :return: `int` || -1 (if not in <self.states>)
-        """
 
-        if state not in self.states:
-            return -1
-        return self.states[state].level
+class StateMachine(StateMachine):  # noqa: F405 pylint: disable=function-redefined
+    """Backward compatibility wrapper - moved to DIRACCommon"""
 
     def setState(self, candidateState, noWarn=False):
-        """Makes sure the state is either None or known to the machine, and that it is a valid state to move into.
-            Final states are also checked.
-
-        examples:
-          >>> sm0.setState(None)['OK']
-              True
-          >>> sm0.setState('Nirvana')['OK']
-              True
-          >>> sm0.setState('AnotherState')['OK']
-              False
-
-        :param state: state which will be set as current state of the StateMachine
-        :type state: None or str
-        :return: S_OK || S_ERROR
-        """
-        if candidateState == self.state:
-            return S_OK(candidateState)
-
-        if not candidateState:
-            self.state = candidateState
-        elif candidateState in self.states:
-            if not self.states[self.state].stateMap:
-                if not noWarn:
-                    gLogger.warn("Final state, won't move", f"({self.state}, asked to move to {candidateState})")
-                return S_OK(self.state)
-            if candidateState not in self.states[self.state].stateMap:
-                gLogger.warn(f"Can't move from {self.state} to {candidateState}, choosing a good one")
-            result = self.getNextState(candidateState)
-            if not result["OK"]:
-                return result
-            self.state = result["Value"]
-            # If the StateMachine does not accept the candidate, return error message
-        else:
-            return S_ERROR(f"setState: {candidateState!r} is not a valid state")
-
-        return S_OK(self.state)
-
-    def getStates(self):
-        """
-        Returns all possible states in the state map
-
-        examples:
-          >>> sm0.getStates()
-              [ 'Nirvana' ]
-
-        :return: list(stateNames)
-        """
-
-        return list(self.states)
-
-    def getNextState(self, candidateState):
-        """
-        Method that gets the next state, given the proposed transition to candidateState.
-        If candidateState is not on the state map <self.states>, it is rejected. If it is
-        not the case, we have two options: if <self.state> is None, then the next state
-        will be <candidateState>. Otherwise, the current state is using its own
-        transition rule to decide.
-
-        examples:
-          >>> sm0.getNextState(None)
-              S_OK(None)
-          >>> sm0.getNextState('NextState')
-              S_OK('NextState')
-
-        :param str candidateState: name of the next state
-        :return: S_OK(nextState) || S_ERROR
-        """
-        if candidateState not in self.states:
-            return S_ERROR(f"getNextState: {candidateState!r} is not a valid state")
-
-        # FIXME: do we need this anymore ?
-        if self.state is None:
-            return S_OK(candidateState)
-
-        return S_OK(self.states[self.state].transitionRule(candidateState))
+        return super().setState(candidateState, noWarn, logger_warn=gLogger.warn)

DIRAC/Core/Utilities/TimeUtilities.py

@@ -1,260 +1,17 @@
-"""
-DIRAC TimeUtilities module
-Support for basic Date and Time operations
-based on system datetime module.
-
-It provides common interface to UTC timestamps,
-converter to string types and back.
-
-Useful timedelta constant are also provided to
-define time intervals.
-
-Notice: datetime.timedelta objects allow multiplication and division by interger
-but not by float. Thus:
-
-  - DIRAC.TimeUtilities.second * 1.5             is not allowed
-  - DIRAC.TimeUtilities.second * 3 / 2           is allowed
+"""Backward compatibility wrapper - moved to DIRACCommon
 
-An timeInterval class provides a method to check
-if a give datetime is in the defined interval.
+This module has been moved to DIRACCommon.Core.Utilities.TimeUtilities to avoid
+circular dependencies and allow DiracX to use these utilities without
+triggering DIRAC's global state initialization.
 
+All exports are maintained for backward compatibility.
 """
-import datetime
-import sys
-import time
-
-from DIRAC import gLogger
-
-# Some useful constants for time operations
-microsecond = datetime.timedelta(microseconds=1)
-second = datetime.timedelta(seconds=1)
-minute = datetime.timedelta(minutes=1)
-hour = datetime.timedelta(hours=1)
-day = datetime.timedelta(days=1)
-week = datetime.timedelta(days=7)
-
-
-def timeThis(method):
-    """Function to be used as a decorator for timing other functions/methods"""
-
-    def timed(*args, **kw):
-        """What actually times"""
-        ts = time.time()
-        result = method(*args, **kw)
-        if sys.stdout.isatty():
-            return result
-        te = time.time()
-
-        pre = datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S UTC ")
-
-        try:
-            pre += args[0].log.getName() + "/" + args[0].log.getSubName() + "   TIME: " + args[0].transString
-        except AttributeError:
-            try:
-                pre += args[0].log.getName() + "    TIME: " + args[0].transString
-            except AttributeError:
-                try:
-                    pre += args[0].log.getName() + "/" + args[0].log.getSubName() + "   TIME: "
-                except AttributeError:
-                    pre += "TIME: "
-        except IndexError:
-            pre += "TIME: "
-
-        argsLen = ""
-        if args:
-            try:
-                if isinstance(args[1], (list, dict)):
-                    argsLen = f"arguments len: {len(args[1])}"
-            except IndexError:
-                if kw:
-                    try:
-                        if isinstance(list(list(kw.items())[0])[1], (list, dict)):
-                            argsLen = f"arguments len: {len(list(list(kw.items())[0])[1])}"
-                    except IndexError:
-                        argsLen = ""
-
-        gLogger.info(f"{pre} Exec time ===> function {method.__name__!r} {argsLen} -> {te - ts:2.2f} sec")
-        return result
-
-    return timed
-
-
-def toEpoch(dateTimeObject=None):
-    """
-    Get seconds since epoch. Accepts datetime or date objects
-    """
-    return toEpochMilliSeconds(dateTimeObject) // 1000
+from functools import partial
 
+# Re-export everything from DIRACCommon for backward compatibility
+from DIRACCommon.Core.Utilities.TimeUtilities import *  # noqa: F401, F403
 
-def toEpochMilliSeconds(dateTimeObject=None):
-    """
-    Get milliseconds since epoch
-    """
-    if dateTimeObject is None:
-        dateTimeObject = datetime.datetime.utcnow()
-    if dateTimeObject.resolution == datetime.timedelta(days=1):
-        # Add time information corresponding to midnight UTC if it's a datetime.date
-        dateTimeObject = datetime.datetime.combine(
-            dateTimeObject, datetime.time.min.replace(tzinfo=datetime.timezone.utc)
-        )
-    posixTime = dateTimeObject.replace(tzinfo=datetime.timezone.utc).timestamp()
-    return int(posixTime * 1000)
-
-
-def fromEpoch(epoch):
-    """
-    Get datetime object from epoch
-    """
-    # Check if the timestamp is in milliseconds
-    if epoch > 10**17:  # nanoseconds
-        epoch /= 1000**3
-    elif epoch > 10**14:  # microseconds
-        epoch /= 1000**2
-    elif epoch > 10**11:  # milliseconds
-        epoch /= 1000
-    return datetime.datetime.utcfromtimestamp(epoch)
-
-
-def toString(myDate=None):
-    """
-    Convert to String
-    if argument type is neither _dateTimeType, _dateType, nor _timeType
-    the current dateTime converted to String is returned instead
-
-    Notice: datetime.timedelta are converted to strings using the format:
-      [day] days [hour]:[min]:[sec]:[microsec]
-      where hour, min, sec, microsec are always positive integers,
-      and day carries the sign.
-    To keep internal consistency we are using:
-      [hour]:[min]:[sec]:[microsec]
-      where min, sec, microsec are always positive integers and hour carries the sign.
-    """
-    if isinstance(myDate, datetime.date):
-        return str(myDate)
-
-    elif isinstance(myDate, datetime.time):
-        return "%02d:%02d:%02d.%06d" % (
-            myDate.days * 24 + myDate.seconds / 3600,
-            myDate.seconds % 3600 / 60,
-            myDate.seconds % 60,
-            myDate.microseconds,
-        )
-    else:
-        return toString(datetime.datetime.utcnow())
-
-
-def fromString(myDate=None):
-    """
-    Convert date/time/datetime String back to appropriated objects
-
-    The format of the string it is assume to be that returned by toString method.
-    See notice on toString method
-    On Error, return None
-
-    :param myDate: the date string to be converted
-    :type myDate: str or datetime.datetime
-    """
-    if isinstance(myDate, datetime.datetime):
-        return myDate
-    if isinstance(myDate, str):
-        if myDate.find(" ") > 0:
-            dateTimeTuple = myDate.split(" ")
-            dateTuple = dateTimeTuple[0].split("-")
-            try:
-                return datetime.datetime(year=dateTuple[0], month=dateTuple[1], day=dateTuple[2]) + fromString(
-                    dateTimeTuple[1]
-                )
-                # return datetime.datetime.utcnow().combine( fromString( dateTimeTuple[0] ),
-                #                                   fromString( dateTimeTuple[1] ) )
-            except Exception:
-                try:
-                    return datetime.datetime(
-                        year=int(dateTuple[0]), month=int(dateTuple[1]), day=int(dateTuple[2])
-                    ) + fromString(dateTimeTuple[1])
-                except ValueError:
-                    return None
-                # return datetime.datetime.utcnow().combine( fromString( dateTimeTuple[0] ),
-                #                                   fromString( dateTimeTuple[1] ) )
-        elif myDate.find(":") > 0:
-            timeTuple = myDate.replace(".", ":").split(":")
-            try:
-                if len(timeTuple) == 4:
-                    return datetime.timedelta(
-                        hours=int(timeTuple[0]),
-                        minutes=int(timeTuple[1]),
-                        seconds=int(timeTuple[2]),
-                        microseconds=int(timeTuple[3]),
-                    )
-                elif len(timeTuple) == 3:
-                    try:
-                        return datetime.timedelta(
-                            hours=int(timeTuple[0]),
-                            minutes=int(timeTuple[1]),
-                            seconds=int(timeTuple[2]),
-                            microseconds=0,
-                        )
-                    except ValueError:
-                        return None
-                else:
-                    return None
-            except Exception:
-                return None
-        elif myDate.find("-") > 0:
-            dateTuple = myDate.split("-")
-            try:
-                return datetime.date(int(dateTuple[0]), int(dateTuple[1]), int(dateTuple[2]))
-            except Exception:
-                return None
-
-    return None
-
-
-class timeInterval:
-    """
-    Simple class to define a timeInterval object able to check if a given
-    dateTime is inside
-    """
-
-    def __init__(self, initialDateTime, intervalTimeDelta):
-        """
-        Initialization method, it requires the initial dateTime and the
-        timedelta that define the limits.
-        The upper limit is not included thus it is [begin,end)
-        If not properly initialized an error flag is set, and subsequent calls
-        to any method will return None
-        """
-        if not isinstance(initialDateTime, datetime.datetime) or not isinstance(intervalTimeDelta, datetime.timedelta):
-            self.__error = True
-            return None
-        self.__error = False
-        if intervalTimeDelta.days < 0:
-            self.__startDateTime = initialDateTime + intervalTimeDelta
-            self.__endDateTime = initialDateTime
-        else:
-            self.__startDateTime = initialDateTime
-            self.__endDateTime = initialDateTime + intervalTimeDelta
-
-    def includes(self, myDateTime):
-        """ """
-        if self.__error:
-            return None
-        if not isinstance(myDateTime, datetime.datetime):
-            return None
-        if myDateTime < self.__startDateTime:
-            return False
-        if myDateTime >= self.__endDateTime:
-            return False
-        return True
-
-
-def queryTime(f):
-    """Decorator to measure the function call time"""
+from DIRAC import gLogger
 
-    def measureQueryTime(*args, **kwargs):
-        start = time.time()
-        result = f(*args, **kwargs)
-        if result["OK"] and "QueryTime" not in result:
-            result["QueryTime"] = time.time() - start
-        return result
 
-    return measureQueryTime
+timeThis = partial(timeThis, logger_info=gLogger.info)

DIRAC/Core/Utilities/test/Test_JDL.py

@@ -19,9 +19,6 @@ def jdl_monkey_business(monkeypatch):
     monkeypatch.setattr("DIRAC.Core.Base.API.getSites", lambda: S_OK(["LCG.IN2P3.fr"]))
     monkeypatch.setattr("DIRAC.WorkloadManagementSystem.Utilities.JobModel.getSites", lambda: S_OK(["LCG.IN2P3.fr"]))
     monkeypatch.setattr("DIRAC.Interfaces.API.Job.getDIRACPlatforms", lambda: S_OK("x86_64-slc6-gcc49-opt"))
-    monkeypatch.setattr(
-        "DIRAC.WorkloadManagementSystem.Utilities.JobModel.getDIRACPlatforms", lambda: S_OK("x86_64-slc6-gcc49-opt")
-    )
     yield
 
 

DIRAC/DataManagementSystem/DB/FTS3DB.py

@@ -16,6 +16,7 @@ from sqlalchemy import (
     Enum,
     Float,
     ForeignKey,
+    Index,
     Integer,
     MetaData,
     SmallInteger,
@@ -85,6 +86,7 @@ fts3JobTable = Table(
     Column("error", String(2048)),
     Column("status", Enum(*FTS3Job.ALL_STATES), server_default=FTS3Job.INIT_STATE, index=True),
     Column("assignment", String(255), server_default=None),
+    Index("idx_jobs_lastupdate_assignment", "lastUpdate", "assignment"),
     mysql_engine="InnoDB",
 )
 
@@ -110,6 +112,7 @@ fts3OperationTable = Table(
     Column("error", String(1024)),
     Column("type", String(255)),
     Column("assignment", String(255), server_default=None),
+    Index("idx_operations_lastupdate_assignment", "lastUpdate", "assignment"),
     mysql_engine="InnoDB",
 )
 

DIRAC/RequestManagementSystem/DB/test/RMSTestScenari.py

@@ -6,6 +6,7 @@ integration tests (Test_ReqDB.py)
 # pylint: disable=invalid-name,wrong-import-position
 import time
 
+import pytest
 
 from DIRAC.RequestManagementSystem.Client.Request import Request
 from DIRAC.RequestManagementSystem.Client.Operation import Operation
@@ -42,6 +43,7 @@ def test_stress(reqDB):
         assert delete["OK"], delete
 
 
+@pytest.mark.slow
 def test_stressBulk(reqDB):
     """stress test bulk"""
 

DIRAC/Resources/Catalog/RucioFileCatalogClient.py

@@ -151,7 +151,7 @@ class RucioFileCatalogClient(FileCatalogClientBase):
             self.authHost = options.get("AuthHost", None)
             self.caCertPath = Locations.getCAsLocation()
             try:
-                sLog.info(f"Logging in with a proxy located at: {self.proxyPath}")
+                sLog.debug(f"Logging in with a proxy located at: {self.proxyPath}")
                 sLog.debug("account: ", self.username)
                 sLog.debug("rucio host: ", self.rucioHost)
                 sLog.debug("auth  host: ", self.authHost)

DIRAC/Resources/Computing/test/Test_PoolComputingElement.py

@@ -25,7 +25,7 @@ while True:
   if os.path.isfile(stopFile):
     os.remove(stopFile)
     break
-  if (time.time() - start) > 30:
+  if (time.time() - start) > 5:
     break
 print("End job", jobNumber, time.time())
 """
@@ -262,6 +262,7 @@ def test_executeJob_wholeNode8(createAndDelete):
             assert "Not enough processors" in submissionResult["Message"]
 
 
+@pytest.mark.slow
 def test_executeJob_submitAndStop(createAndDelete):
     time.sleep(0.5)