DIRACCommon 9.0.0__py3-none-any.whl

DIRACCommon/Core/Utilities/StateMachine.py

@@ -0,0 +1,185 @@
+""" StateMachine
+
+  This module contains the basic blocks to build a state machine (State and StateMachine)
+"""
+from DIRACCommon.Core.Utilities.ReturnValues import S_OK, S_ERROR
+
+
+class State:
+    """
+    State class that represents a single step on a StateMachine, with all the
+    possible transitions, the default transition and an ordering level.
+
+
+    examples:
+      >>> s0 = State(100)
+      >>> s1 = State(0, ['StateName1', 'StateName2'], defState='StateName1')
+      >>> s2 = State(0, ['StateName1', 'StateName2'])
+      # this example is tricky. The transition rule says that will go to
+      # nextState, e.g. 'StateNext'. But, it is not on the stateMap, and there
+      # is no default defined, so it will end up going to StateNext anyway. You
+      # must be careful while defining states and their stateMaps and defaults.
+    """
+
+    def __init__(self, level, stateMap=None, defState=None):
+        """
+        :param int level: each state is mapped to an integer, which is used to sort the states according to that integer.
+        :param list stateMap: it is a list (of strings) with the reachable states from this particular status.
+                              If not defined, we assume there are no restrictions.
+        :param str defState: default state used in case the next state is not in stateMap (not defined or simply not there).
+        """
+
+        self.level = level
+        self.stateMap = stateMap if stateMap else []
+        self.default = defState
+
+    def transitionRule(self, nextState):
+        """
+        Method that selects next state, knowing the default and the transitions
+        map, and the proposed next state. If <nextState> is in stateMap, goes there.
+        If not, then goes to <self.default> if any. Otherwise, goes to <nextState>
+        anyway.
+
+        examples:
+          >>> s0.transitionRule('nextState')
+              'nextState'
+          >>> s1.transitionRule('StateName2')
+              'StateName2'
+          >>> s1.transitionRule('StateNameNotInMap')
+              'StateName1'
+          >>> s2.transitionRule('StateNameNotInMap')
+              'StateNameNotInMap'
+
+        :param str nextState: name of the state in the stateMap
+        :return: state name
+        :rtype: str
+        """
+
+        # If next state is on the list of next states, go ahead.
+        if nextState in self.stateMap:
+            return nextState
+
+        # If not, calculate defaultState:
+        # if there is a default, that one
+        # otherwise is nextState (states with empty list have no movement restrictions)
+        defaultNext = self.default if self.default else nextState
+        return defaultNext
+
+
+class StateMachine:
+    """
+    StateMachine class that represents the whole state machine with all transitions.
+
+    examples:
+      >>> sm0 = StateMachine()
+      >>> sm1 = StateMachine(state = 'Active')
+
+    :param state: current state of the StateMachine, could be None if we do not use the
+        StateMachine to calculate transitions. Beware, it is not checked if the
+        state is on the states map !
+    :type state: None or str
+
+    """
+
+    def __init__(self, state=None):
+        """
+        Constructor.
+        """
+
+        self.state = state
+        # To be overwritten by child classes, unless you like Nirvana state that much.
+        self.states = {"Nirvana": State(100)}
+
+    def getLevelOfState(self, state):
+        """
+        Given a state name, it returns its level (integer), which defines the hierarchy.
+
+        >>> sm0.getLevelOfState('Nirvana')
+            100
+        >>> sm0.getLevelOfState('AnotherState')
+            -1
+
+        :param str state: name of the state, it should be on <self.states> key set
+        :return: `int` || -1 (if not in <self.states>)
+        """
+
+        if state not in self.states:
+            return -1
+        return self.states[state].level
+
+    def setState(self, candidateState, noWarn=False, *, logger_warn=None):
+        """Makes sure the state is either None or known to the machine, and that it is a valid state to move into.
+            Final states are also checked.
+
+        examples:
+          >>> sm0.setState(None)['OK']
+              True
+          >>> sm0.setState('Nirvana')['OK']
+              True
+          >>> sm0.setState('AnotherState')['OK']
+              False
+
+        :param state: state which will be set as current state of the StateMachine
+        :type state: None or str
+        :return: S_OK || S_ERROR
+        """
+        if candidateState == self.state:
+            return S_OK(candidateState)
+
+        if not candidateState:
+            self.state = candidateState
+        elif candidateState in self.states:
+            if not self.states[self.state].stateMap:
+                if not noWarn and logger_warn:
+                    logger_warn("Final state, won't move", f"({self.state}, asked to move to {candidateState})")
+                return S_OK(self.state)
+            if candidateState not in self.states[self.state].stateMap and logger_warn:
+                logger_warn(f"Can't move from {self.state} to {candidateState}, choosing a good one")
+            result = self.getNextState(candidateState)
+            if not result["OK"]:
+                return result
+            self.state = result["Value"]
+            # If the StateMachine does not accept the candidate, return error message
+        else:
+            return S_ERROR(f"setState: {candidateState!r} is not a valid state")
+
+        return S_OK(self.state)
+
+    def getStates(self):
+        """
+        Returns all possible states in the state map
+
+        examples:
+          >>> sm0.getStates()
+              [ 'Nirvana' ]
+
+        :return: list(stateNames)
+        """
+
+        return list(self.states)
+
+    def getNextState(self, candidateState):
+        """
+        Method that gets the next state, given the proposed transition to candidateState.
+        If candidateState is not on the state map <self.states>, it is rejected. If it is
+        not the case, we have two options: if <self.state> is None, then the next state
+        will be <candidateState>. Otherwise, the current state is using its own
+        transition rule to decide.
+
+        examples:
+          >>> sm0.getNextState(None)
+              S_OK(None)
+          >>> sm0.getNextState('NextState')
+              S_OK('NextState')
+
+        :param str candidateState: name of the next state
+        :return: S_OK(nextState) || S_ERROR
+        """
+        if candidateState not in self.states:
+            return S_ERROR(f"getNextState: {candidateState!r} is not a valid state")
+
+        # FIXME: do we need this anymore ?
+        if self.state is None:
+            return S_OK(candidateState)
+
+        return S_OK(self.states[self.state].transitionRule(candidateState))

DIRACCommon/Core/Utilities/TimeUtilities.py

@@ -0,0 +1,259 @@
+"""
+DIRAC TimeUtilities module
+Support for basic Date and Time operations
+based on system datetime module.
+
+It provides common interface to UTC timestamps,
+converter to string types and back.
+
+Useful timedelta constant are also provided to
+define time intervals.
+
+Notice: datetime.timedelta objects allow multiplication and division by interger
+but not by float. Thus:
+
+  - DIRAC.TimeUtilities.second * 1.5             is not allowed
+  - DIRAC.TimeUtilities.second * 3 / 2           is allowed
+
+An timeInterval class provides a method to check
+if a give datetime is in the defined interval.
+
+"""
+import datetime
+import sys
+import time
+
+# Some useful constants for time operations
+microsecond = datetime.timedelta(microseconds=1)
+second = datetime.timedelta(seconds=1)
+minute = datetime.timedelta(minutes=1)
+hour = datetime.timedelta(hours=1)
+day = datetime.timedelta(days=1)
+week = datetime.timedelta(days=7)
+
+
+def timeThis(method, *, logger_info=None):
+    """Function to be used as a decorator for timing other functions/methods"""
+
+    def timed(*args, **kw):
+        """What actually times"""
+        ts = time.time()
+        result = method(*args, **kw)
+        if sys.stdout.isatty():
+            return result
+        te = time.time()
+
+        pre = datetime.datetime.now(datetime.UTC).strftime("%Y-%m-%d %H:%M:%S UTC ")
+
+        try:
+            pre += args[0].log.getName() + "/" + args[0].log.getSubName() + "   TIME: " + args[0].transString
+        except AttributeError:
+            try:
+                pre += args[0].log.getName() + "    TIME: " + args[0].transString
+            except AttributeError:
+                try:
+                    pre += args[0].log.getName() + "/" + args[0].log.getSubName() + "   TIME: "
+                except AttributeError:
+                    pre += "TIME: "
+        except IndexError:
+            pre += "TIME: "
+
+        argsLen = ""
+        if args:
+            try:
+                if isinstance(args[1], (list, dict)):
+                    argsLen = f"arguments len: {len(args[1])}"
+            except IndexError:
+                if kw:
+                    try:
+                        if isinstance(list(list(kw.items())[0])[1], (list, dict)):
+                            argsLen = f"arguments len: {len(list(list(kw.items())[0])[1])}"
+                    except IndexError:
+                        argsLen = ""
+
+        if logger_info is not None:
+            logger_info(f"{pre} Exec time ===> function {method.__name__!r} {argsLen} -> {te - ts:2.2f} sec")
+        return result
+
+    return timed
+
+
+def toEpoch(dateTimeObject=None):
+    """
+    Get seconds since epoch. Accepts datetime or date objects
+    """
+    return toEpochMilliSeconds(dateTimeObject) // 1000
+
+
+def toEpochMilliSeconds(dateTimeObject=None):
+    """
+    Get milliseconds since epoch
+    """
+    if dateTimeObject is None:
+        dateTimeObject = datetime.datetime.utcnow()
+    if dateTimeObject.resolution == datetime.timedelta(days=1):
+        # Add time information corresponding to midnight UTC if it's a datetime.date
+        dateTimeObject = datetime.datetime.combine(
+            dateTimeObject, datetime.time.min.replace(tzinfo=datetime.timezone.utc)
+        )
+    posixTime = dateTimeObject.replace(tzinfo=datetime.timezone.utc).timestamp()
+    return int(posixTime * 1000)
+
+
+def fromEpoch(epoch):
+    """
+    Get datetime object from epoch
+    """
+    # Check if the timestamp is in milliseconds
+    if epoch > 10**17:  # nanoseconds
+        epoch /= 1000**3
+    elif epoch > 10**14:  # microseconds
+        epoch /= 1000**2
+    elif epoch > 10**11:  # milliseconds
+        epoch /= 1000
+    return datetime.datetime.utcfromtimestamp(epoch)
+
+
+def toString(myDate=None):
+    """
+    Convert to String
+    if argument type is neither _dateTimeType, _dateType, nor _timeType
+    the current dateTime converted to String is returned instead
+
+    Notice: datetime.timedelta are converted to strings using the format:
+      [day] days [hour]:[min]:[sec]:[microsec]
+      where hour, min, sec, microsec are always positive integers,
+      and day carries the sign.
+    To keep internal consistency we are using:
+      [hour]:[min]:[sec]:[microsec]
+      where min, sec, microsec are always positive integers and hour carries the sign.
+    """
+    if isinstance(myDate, datetime.date):
+        return str(myDate)
+
+    elif isinstance(myDate, datetime.time):
+        return "%02d:%02d:%02d.%06d" % (
+            myDate.days * 24 + myDate.seconds / 3600,
+            myDate.seconds % 3600 / 60,
+            myDate.seconds % 60,
+            myDate.microseconds,
+        )
+    else:
+        return toString(datetime.datetime.utcnow())
+
+
+def fromString(myDate=None):
+    """
+    Convert date/time/datetime String back to appropriated objects
+
+    The format of the string it is assume to be that returned by toString method.
+    See notice on toString method
+    On Error, return None
+
+    :param myDate: the date string to be converted
+    :type myDate: str or datetime.datetime
+    """
+    if isinstance(myDate, datetime.datetime):
+        return myDate
+    if isinstance(myDate, str):
+        if myDate.find(" ") > 0:
+            dateTimeTuple = myDate.split(" ")
+            dateTuple = dateTimeTuple[0].split("-")
+            try:
+                return datetime.datetime(year=dateTuple[0], month=dateTuple[1], day=dateTuple[2]) + fromString(
+                    dateTimeTuple[1]
+                )
+                # return datetime.datetime.utcnow().combine( fromString( dateTimeTuple[0] ),
+                #                                   fromString( dateTimeTuple[1] ) )
+            except Exception:
+                try:
+                    return datetime.datetime(
+                        year=int(dateTuple[0]), month=int(dateTuple[1]), day=int(dateTuple[2])
+                    ) + fromString(dateTimeTuple[1])
+                except ValueError:
+                    return None
+                # return datetime.datetime.utcnow().combine( fromString( dateTimeTuple[0] ),
+                #                                   fromString( dateTimeTuple[1] ) )
+        elif myDate.find(":") > 0:
+            timeTuple = myDate.replace(".", ":").split(":")
+            try:
+                if len(timeTuple) == 4:
+                    return datetime.timedelta(
+                        hours=int(timeTuple[0]),
+                        minutes=int(timeTuple[1]),
+                        seconds=int(timeTuple[2]),
+                        microseconds=int(timeTuple[3]),
+                    )
+                elif len(timeTuple) == 3:
+                    try:
+                        return datetime.timedelta(
+                            hours=int(timeTuple[0]),
+                            minutes=int(timeTuple[1]),
+                            seconds=int(timeTuple[2]),
+                            microseconds=0,
+                        )
+                    except ValueError:
+                        return None
+                else:
+                    return None
+            except Exception:
+                return None
+        elif myDate.find("-") > 0:
+            dateTuple = myDate.split("-")
+            try:
+                return datetime.date(int(dateTuple[0]), int(dateTuple[1]), int(dateTuple[2]))
+            except Exception:
+                return None
+
+    return None
+
+
+class timeInterval:
+    """
+    Simple class to define a timeInterval object able to check if a given
+    dateTime is inside
+    """
+
+    def __init__(self, initialDateTime, intervalTimeDelta):
+        """
+        Initialization method, it requires the initial dateTime and the
+        timedelta that define the limits.
+        The upper limit is not included thus it is [begin,end)
+        If not properly initialized an error flag is set, and subsequent calls
+        to any method will return None
+        """
+        if not isinstance(initialDateTime, datetime.datetime) or not isinstance(intervalTimeDelta, datetime.timedelta):
+            self.__error = True
+            return None
+        self.__error = False
+        if intervalTimeDelta.days < 0:
+            self.__startDateTime = initialDateTime + intervalTimeDelta
+            self.__endDateTime = initialDateTime
+        else:
+            self.__startDateTime = initialDateTime
+            self.__endDateTime = initialDateTime + intervalTimeDelta
+
+    def includes(self, myDateTime):
+        """ """
+        if self.__error:
+            return None
+        if not isinstance(myDateTime, datetime.datetime):
+            return None
+        if myDateTime < self.__startDateTime:
+            return False
+        if myDateTime >= self.__endDateTime:
+            return False
+        return True
+
+
+def queryTime(f):
+    """Decorator to measure the function call time"""
+
+    def measureQueryTime(*args, **kwargs):
+        start = time.time()
+        result = f(*args, **kwargs)
+        if result["OK"] and "QueryTime" not in result:
+            result["QueryTime"] = time.time() - start
+        return result
+
+    return measureQueryTime

DIRACCommon/Core/Utilities/__init__.py

@@ -0,0 +1,3 @@
+"""
+DIRACCommon.Utils - Stateless utility functions
+"""

DIRACCommon/Core/__init__.py

@@ -0,0 +1 @@
+"""DIRACCommon Core utilities"""

DIRACCommon/WorkloadManagementSystem/Client/JobState/JobManifest.py

@@ -0,0 +1,235 @@
+from __future__ import annotations
+
+from typing import Literal, TypedDict
+
+from diraccfg import CFG
+
+from DIRACCommon.Core.Utilities import List
+from DIRACCommon.Core.Utilities.JDL import dumpCFGAsJDL, loadJDLAsCFG
+from DIRACCommon.Core.Utilities.ReturnValues import S_ERROR, S_OK
+
+
+class JobManifestNumericalVar(TypedDict):
+    CPUTime: int
+    Priority: int
+
+
+class JobManifestConfig(TypedDict):
+    """Dictionary type for defining the information JobManifest needs from the CS"""
+
+    defaultForGroup: JobManifestNumericalVar
+    minForGroup: JobManifestNumericalVar
+    maxForGroup: JobManifestNumericalVar
+    allowedJobTypesForGroup: list[str]
+
+    maxInputData: int
+
+
+class JobManifest:
+    def __init__(self, manifest=""):
+        self.__manifest = CFG()
+        self.__dirty = False
+        if manifest:
+            result = self.load(manifest)
+            if not result["OK"]:
+                raise Exception(result["Message"])
+
+    def isDirty(self):
+        return self.__dirty
+
+    def setDirty(self):
+        self.__dirty = True
+
+    def clearDirty(self):
+        self.__dirty = False
+
+    def load(self, dataString):
+        """
+        Auto discover format type based on [ .. ] of JDL
+        """
+        dataString = dataString.strip()
+        if dataString[0] == "[" and dataString[-1] == "]":
+            return self.loadJDL(dataString)
+        else:
+            return self.loadCFG(dataString)
+
+    def loadJDL(self, jdlString):
+        """
+        Load job manifest from JDL format
+        """
+        result = loadJDLAsCFG(jdlString.strip())
+        if not result["OK"]:
+            self.__manifest = CFG()
+            return result
+        self.__manifest = result["Value"][0]
+        return S_OK()
+
+    def loadCFG(self, cfgString):
+        """
+        Load job manifest from CFG format
+        """
+        try:
+            self.__manifest.loadFromBuffer(cfgString)
+        except Exception as e:
+            return S_ERROR(f"Can't load manifest from cfg: {str(e)}")
+        return S_OK()
+
+    def dumpAsCFG(self):
+        return str(self.__manifest)
+
+    def getAsCFG(self):
+        return self.__manifest.clone()
+
+    def dumpAsJDL(self):
+        return dumpCFGAsJDL(self.__manifest)
+
+    def _checkNumericalVar(
+        self,
+        varName: Literal["CPUTime", "Priority"],
+        defaultVal: int,
+        minVal: int,
+        maxVal: int,
+        config: JobManifestConfig,
+    ):
+        """
+        Check a numerical var
+        """
+        if varName in self.__manifest:
+            varValue = self.__manifest[varName]
+        else:
+            varValue = config["defaultForGroup"].get(varName, defaultVal)
+        try:
+            varValue = int(varValue)
+        except ValueError:
+            return S_ERROR(f"{varName} must be a number")
+        minVal = config["minForGroup"][varName]
+        maxVal = config["maxForGroup"][varName]
+        varValue = max(minVal, min(varValue, maxVal))
+        if varName not in self.__manifest:
+            self.__manifest.setOption(varName, varValue)
+        return S_OK(varValue)
+
+    def __contains__(self, key):
+        """Check if the manifest has the required key"""
+        return key in self.__manifest
+
+    def setOptionsFromDict(self, varDict):
+        for k in sorted(varDict):
+            self.setOption(k, varDict[k])
+
+    def check(self, *, config: JobManifestConfig):
+        """
+        Check that the manifest is OK
+        """
+        for k in ["Owner", "OwnerGroup"]:
+            if k not in self.__manifest:
+                return S_ERROR(f"Missing var {k} in manifest")
+
+        # Check CPUTime
+        result = self._checkNumericalVar("CPUTime", 86400, 100, 500000, config=config)
+        if not result["OK"]:
+            return result
+
+        result = self._checkNumericalVar("Priority", 1, 0, 10, config=config)
+        if not result["OK"]:
+            return result
+
+        if "InputData" in self.__manifest:
+            nInput = len(List.fromChar(self.__manifest["InputData"]))
+            if nInput > config["maxInputData"]:
+                return S_ERROR(
+                    f"Number of Input Data Files ({nInput}) greater than current limit: {config['maxInputData']}"
+                )
+
+        if "JobType" in self.__manifest:
+            varValue = self.__manifest["JobType"]
+            for v in List.fromChar(varValue):
+                if v not in config["allowedJobTypesForGroup"]:
+                    return S_ERROR(f"{v} is not a valid value for JobType")
+
+        return S_OK()
+
+    def createSection(self, secName, contents=False):
+        if secName not in self.__manifest:
+            if contents and not isinstance(contents, CFG):
+                return S_ERROR(f"Contents for section {secName} is not a cfg object")
+            self.__dirty = True
+            return S_OK(self.__manifest.createNewSection(secName, contents=contents))
+        return S_ERROR(f"Section {secName} already exists")
+
+    def getSection(self, secName):
+        self.__dirty = True
+        if secName not in self.__manifest:
+            return S_ERROR(f"{secName} does not exist")
+        sec = self.__manifest[secName]
+        if not sec:
+            return S_ERROR(f"{secName} section empty")
+        return S_OK(sec)
+
+    def setSectionContents(self, secName, contents):
+        if contents and not isinstance(contents, CFG):
+            return S_ERROR(f"Contents for section {secName} is not a cfg object")
+        self.__dirty = True
+        if secName in self.__manifest:
+            self.__manifest[secName].reset()
+            self.__manifest[secName].mergeWith(contents)
+        else:
+            self.__manifest.createNewSection(secName, contents=contents)
+
+    def setOption(self, varName, varValue):
+        """
+        Set a var in job manifest
+        """
+        self.__dirty = True
+        levels = List.fromChar(varName, "/")
+        cfg = self.__manifest
+        for l in levels[:-1]:
+            if l not in cfg:
+                cfg.createNewSection(l)
+            cfg = cfg[l]
+        cfg.setOption(levels[-1], varValue)
+
+    def remove(self, opName):
+        levels = List.fromChar(opName, "/")
+        cfg = self.__manifest
+        for l in levels[:-1]:
+            if l not in cfg:
+                return S_ERROR(f"{opName} does not exist")
+            cfg = cfg[l]
+        if cfg.deleteKey(levels[-1]):
+            self.__dirty = True
+            return S_OK()
+        return S_ERROR(f"{opName} does not exist")
+
+    def getOption(self, varName, defaultValue=None):
+        """
+        Get a variable from the job manifest
+        """
+        cfg = self.__manifest
+        return cfg.getOption(varName, defaultValue)
+
+    def getOptionList(self, section=""):
+        """
+        Get a list of variables in a section of the job manifest
+        """
+        cfg = self.__manifest.getRecursive(section)
+        if not cfg or "value" not in cfg:
+            return []
+        cfg = cfg["value"]
+        return cfg.listOptions()
+
+    def isOption(self, opName):
+        """
+        Check if it is a valid option
+        """
+        return self.__manifest.isOption(opName)
+
+    def getSectionList(self, section=""):
+        """
+        Get a list of sections in the job manifest
+        """
+        cfg = self.__manifest.getRecursive(section)
+        if not cfg or "value" not in cfg:
+            return []
+        cfg = cfg["value"]
+        return cfg.listSections()