DIRAC 9.0.0a54__py3-none-any.whl → 9.0.7__py3-none-any.whl

DIRAC/Core/Utilities/JDL.py

@@ -1,203 +1,9 @@
 """Transformation classes around the JDL format."""
 
-from diraccfg import CFG
-from pydantic import ValidationError
+from DIRACCommon.Core.Utilities.JDL import *  # noqa: F403,F401
 
-from DIRAC import S_OK, S_ERROR
-from DIRAC.Core.Utilities import List
-from DIRAC.Core.Utilities.ClassAd.ClassAdLight import ClassAd
 from DIRAC.WorkloadManagementSystem.Utilities.JobModel import BaseJobDescriptionModel
 
-ARGUMENTS = "Arguments"
-BANNED_SITES = "BannedSites"
-CPU_TIME = "CPUTime"
-EXECUTABLE = "Executable"
-EXECUTION_ENVIRONMENT = "ExecutionEnvironment"
-GRID_CE = "GridCE"
-INPUT_DATA = "InputData"
-INPUT_DATA_POLICY = "InputDataPolicy"
-INPUT_SANDBOX = "InputSandbox"
-JOB_CONFIG_ARGS = "JobConfigArgs"
-JOB_TYPE = "JobType"
-JOB_GROUP = "JobGroup"
-LOG_LEVEL = "LogLevel"
-NUMBER_OF_PROCESSORS = "NumberOfProcessors"
-MAX_NUMBER_OF_PROCESSORS = "MaxNumberOfProcessors"
-MIN_NUMBER_OF_PROCESSORS = "MinNumberOfProcessors"
-OUTPUT_DATA = "OutputData"
-OUTPUT_PATH = "OutputPath"
-OUTPUT_SE = "OutputSE"
-PLATFORM = "Platform"
-PRIORITY = "Priority"
-STD_ERROR = "StdError"
-STD_OUTPUT = "StdOutput"
-OUTPUT_SANDBOX = "OutputSandbox"
-JOB_NAME = "JobName"
-SITE = "Site"
-TAGS = "Tags"
-
-OWNER = "Owner"
-OWNER_GROUP = "OwnerGroup"
-VO = "VirtualOrganization"
-
-CREDENTIALS_FIELDS = {OWNER, OWNER_GROUP, VO}
-
-
-def loadJDLAsCFG(jdl):
-    """
-    Load a JDL as CFG
-    """
-
-    def cleanValue(value):
-        value = value.strip()
-        if value[0] == '"':
-            entries = []
-            iPos = 1
-            current = ""
-            state = "in"
-            while iPos < len(value):
-                if value[iPos] == '"':
-                    if state == "in":
-                        entries.append(current)
-                        current = ""
-                        state = "out"
-                    elif state == "out":
-                        current = current.strip()
-                        if current not in (",",):
-                            return S_ERROR("value seems a list but is not separated in commas")
-                        current = ""
-                        state = "in"
-                else:
-                    current += value[iPos]
-                iPos += 1
-            if state == "in":
-                return S_ERROR('value is opened with " but is not closed')
-            return S_OK(", ".join(entries))
-        else:
-            return S_OK(value.replace('"', ""))
-
-    def assignValue(key, value, cfg):
-        key = key.strip()
-        if len(key) == 0:
-            return S_ERROR("Invalid key name")
-        value = value.strip()
-        if not value:
-            return S_ERROR(f"No value for key {key}")
-        if value[0] == "{":
-            if value[-1] != "}":
-                return S_ERROR("Value '%s' seems a list but does not end in '}'" % (value))
-            valList = List.fromChar(value[1:-1])
-            for i in range(len(valList)):
-                result = cleanValue(valList[i])
-                if not result["OK"]:
-                    return S_ERROR(f"Var {key} : {result['Message']}")
-                valList[i] = result["Value"]
-                if valList[i] is None:
-                    return S_ERROR(f"List value '{value}' seems invalid for item {i}")
-            value = ", ".join(valList)
-        else:
-            result = cleanValue(value)
-            if not result["OK"]:
-                return S_ERROR(f"Var {key} : {result['Message']}")
-            nV = result["Value"]
-            if nV is None:
-                return S_ERROR(f"Value '{value} seems invalid")
-            value = nV
-        cfg.setOption(key, value)
-        return S_OK()
-
-    if jdl[0] == "[":
-        iPos = 1
-    else:
-        iPos = 0
-    key = ""
-    value = ""
-    action = "key"
-    insideLiteral = False
-    cfg = CFG()
-    while iPos < len(jdl):
-        char = jdl[iPos]
-        if char == ";" and not insideLiteral:
-            if key.strip():
-                result = assignValue(key, value, cfg)
-                if not result["OK"]:
-                    return result
-            key = ""
-            value = ""
-            action = "key"
-        elif char == "[" and not insideLiteral:
-            key = key.strip()
-            if not key:
-                return S_ERROR("Invalid key in JDL")
-            if value.strip():
-                return S_ERROR(f"Key {key} seems to have a value and open a sub JDL at the same time")
-            result = loadJDLAsCFG(jdl[iPos:])
-            if not result["OK"]:
-                return result
-            subCfg, subPos = result["Value"]
-            cfg.createNewSection(key, contents=subCfg)
-            key = ""
-            value = ""
-            action = "key"
-            insideLiteral = False
-            iPos += subPos
-        elif char == "=" and not insideLiteral:
-            if action == "key":
-                action = "value"
-                insideLiteral = False
-            else:
-                value += char
-        elif char == "]" and not insideLiteral:
-            key = key.strip()
-            if len(key) > 0:
-                result = assignValue(key, value, cfg)
-                if not result["OK"]:
-                    return result
-            return S_OK((cfg, iPos))
-        else:
-            if action == "key":
-                key += char
-            else:
-                value += char
-                if char == '"':
-                    insideLiteral = not insideLiteral
-        iPos += 1
-
-    return S_OK((cfg, iPos))
-
-
-def dumpCFGAsJDL(cfg, level=1, tab="  "):
-    indent = tab * level
-    contents = [f"{tab * (level - 1)}["]
-    sections = cfg.listSections()
-
-    for key in cfg:
-        if key in sections:
-            contents.append(f"{indent}{key} =")
-            contents.append(f"{dumpCFGAsJDL(cfg[key], level + 1, tab)};")
-        else:
-            val = List.fromChar(cfg[key])
-            # Some attributes are never lists
-            if len(val) < 2 or key in [ARGUMENTS, EXECUTABLE, STD_OUTPUT, STD_ERROR]:
-                value = cfg[key]
-                try:
-                    try_value = float(value)
-                    contents.append(f"{tab * level}{key} = {value};")
-                except Exception:
-                    contents.append(f'{tab * level}{key} = "{value}";')
-            else:
-                contents.append(f"{indent}{key} =")
-                contents.append("%s{" % indent)
-                for iPos in range(len(val)):
-                    try:
-                        value = float(val[iPos])
-                    except Exception:
-                        val[iPos] = f'"{val[iPos]}"'
-                contents.append(",\n".join([f"{tab * (level + 1)}{value}" for value in val]))
-                contents.append("%s};" % indent)
-    contents.append(f"{tab * (level - 1)}]")
-    return "\n".join(contents)
-
 
 def jdlToBaseJobDescriptionModel(classAd: ClassAd):
     """

DIRAC/Core/Utilities/List.py

@@ -1,124 +1 @@
-"""Collection of DIRAC useful list related modules.
-   By default on Error they return None.
-"""
-import random
-import sys
-from typing import Any
-
-
-def uniqueElements(aList: list) -> list:
-    """Utility to retrieve list of unique elements in a list (order is kept)."""
-
-    # Use dict.fromkeys instead of set ensure the order is preserved
-    return list(dict.fromkeys(aList))
-
-
-def appendUnique(aList: list, anObject: Any):
-    """Append to list if object does not exist.
-
-    :param aList: list of elements
-    :param anObject: object you want to append
-    """
-    if anObject not in aList:
-        aList.append(anObject)
-
-
-def fromChar(inputString: str, sepChar: str = ","):
-    """Generates a list splitting a string by the required character(s)
-    resulting string items are stripped and empty items are removed.
-
-    :param inputString: list serialised to string
-    :param sepChar: separator
-    :return: list of strings or None if sepChar has a wrong type
-    """
-    # to prevent getting an empty String as argument
-    if not (isinstance(inputString, str) and isinstance(sepChar, str) and sepChar):
-        return None
-    return [fieldString.strip() for fieldString in inputString.split(sepChar) if len(fieldString.strip()) > 0]
-
-
-def randomize(aList: list) -> list:
-    """Return a randomly sorted list.
-
-    :param aList: list to permute
-    """
-    tmpList = list(aList)
-    random.shuffle(tmpList)
-    return tmpList
-
-
-def pop(aList, popElement):
-    """Pop the first element equal to popElement from the list.
-
-    :param aList: list
-    :type aList: python:list
-    :param popElement: element to pop
-    """
-    if popElement in aList:
-        return aList.pop(aList.index(popElement))
-
-
-def stringListToString(aList: list) -> str:
-    """This function is used for making MySQL queries with a list of string elements.
-
-    :param aList: list to be serialized to string for making queries
-    """
-    return ",".join(f"'{x}'" for x in aList)
-
-
-def intListToString(aList: list) -> str:
-    """This function is used for making MySQL queries with a list of int elements.
-
-    :param aList: list to be serialized to string for making queries
-    """
-    return ",".join(str(x) for x in aList)
-
-
-def getChunk(aList: list, chunkSize: int):
-    """Generator yielding chunk from a list of a size chunkSize.
-
-    :param aList: list to be splitted
-    :param chunkSize: lenght of one chunk
-    :raise: StopIteration
-
-    Usage:
-
-    >>> for chunk in getChunk( aList, chunkSize=10):
-          process( chunk )
-
-    """
-    chunkSize = int(chunkSize)
-    for i in range(0, len(aList), chunkSize):
-        yield aList[i : i + chunkSize]
-
-
-def breakListIntoChunks(aList: list, chunkSize: int):
-    """This function takes a list as input and breaks it into list of size 'chunkSize'.
-       It returns a list of lists.
-
-    :param aList: list of elements
-    :param chunkSize: len of a single chunk
-    :return: list of lists of length of chunkSize
-    :raise: RuntimeError if numberOfFilesInChunk is less than 1
-    """
-    if chunkSize < 1:
-        raise RuntimeError("chunkSize cannot be less than 1")
-    if isinstance(aList, (set, dict, tuple, {}.keys().__class__, {}.items().__class__, {}.values().__class__)):
-        aList = list(aList)
-    return [chunk for chunk in getChunk(aList, chunkSize)]
-
-
-def getIndexInList(anItem: Any, aList: list) -> int:
-    """Return the index of the element x in the list l
-    or sys.maxint if it does not exist
-
-    :param anItem: element to look for
-    :param aList: list to look into
-
-    :return: the index or sys.maxint
-    """
-    # try:
-    if anItem in aList:
-        return aList.index(anItem)
-    else:
-        return sys.maxsize
+from DIRACCommon.Core.Utilities.List import *  # noqa: F401,F403

DIRAC/Core/Utilities/MySQL.py

@@ -1,151 +1,152 @@
-""" DIRAC Basic MySQL Class
-    It provides access to the basic MySQL methods in a multithread-safe mode
-    keeping used connections in a python Queue for further reuse.
+"""DIRAC Basic MySQL Class
+It provides access to the basic MySQL methods in a multithread-safe mode
+keeping used connections in a python Queue for further reuse.
 
-    These are the coded methods:
+These are the coded methods:
 
 
-    __init__( host, user, passwd, name, [maxConnsInQueue=10] )
+__init__( host, user, passwd, name, [maxConnsInQueue=10] )
 
-    Initializes the Queue and tries to connect to the DB server,
-    using the _connect method.
-    "maxConnsInQueue" defines the size of the Queue of open connections
-    that are kept for reuse. It also defined the maximum number of open
-    connections available from the object.
-    maxConnsInQueue = 0 means unlimited and it is not supported.
+Initializes the Queue and tries to connect to the DB server,
+using the _connect method.
+"maxConnsInQueue" defines the size of the Queue of open connections
+that are kept for reuse. It also defined the maximum number of open
+connections available from the object.
+maxConnsInQueue = 0 means unlimited and it is not supported.
 
 
-    _except( methodName, exception, errorMessage )
+_except( methodName, exception, errorMessage )
 
-    Helper method for exceptions: the "methodName" and the "errorMessage"
-    are printed with ERROR level, then the "exception" is printed (with
-    full description if it is a MySQL Exception) and S_ERROR is returned
-    with the errorMessage and the exception.
+Helper method for exceptions: the "methodName" and the "errorMessage"
+are printed with ERROR level, then the "exception" is printed (with
+full description if it is a MySQL Exception) and S_ERROR is returned
+with the errorMessage and the exception.
 
 
-    _connect()
+_connect()
 
-    Attempts connection to DB and sets the _connected flag to True upon success.
-    Returns S_OK or S_ERROR.
+Attempts connection to DB and sets the _connected flag to True upon success.
+Returns S_OK or S_ERROR.
 
 
-    _query( cmd, [conn=conn] )
+_query( cmd, [conn=conn] )
 
-    Executes SQL command "cmd".
-    Gets a connection from the Queue (or open a new one if none is available),
-    the used connection is  back into the Queue.
-    If a connection to the the DB is passed as second argument this connection
-    is used and is not  in the Queue.
-    Returns S_OK with fetchall() out in Value or S_ERROR upon failure.
+Executes SQL command "cmd".
+Gets a connection from the Queue (or open a new one if none is available),
+the used connection is  back into the Queue.
+If a connection to the the DB is passed as second argument this connection
+is used and is not  in the Queue.
+Returns S_OK with fetchall() out in Value or S_ERROR upon failure.
 
 
-    _update( cmd, [conn=conn] )
+_update( cmd, [conn=conn] )
 
-    Executes SQL command "cmd" and issue a commit
-    Gets a connection from the Queue (or open a new one if none is available),
-    the used connection is  back into the Queue.
-    If a connection to the the DB is passed as second argument this connection
-    is used and is not  in the Queue
-    Returns S_OK with number of updated registers in Value or S_ERROR upon failure.
+Executes SQL command "cmd" and issue a commit
+Gets a connection from the Queue (or open a new one if none is available),
+the used connection is  back into the Queue.
+If a connection to the the DB is passed as second argument this connection
+is used and is not  in the Queue
+Returns S_OK with number of updated registers in Value or S_ERROR upon failure.
 
 
-    _createTables( tableDict )
+_createTables( tableDict )
 
-    Create a new Table in the DB
+Create a new Table in the DB
 
 
-    _getConnection()
+_getConnection()
 
-    Gets a connection from the Queue (or open a new one if none is available)
-    Returns S_OK with connection in Value or S_ERROR
-    the calling method is responsible for closing this connection once it is no
-    longer needed.
+Gets a connection from the Queue (or open a new one if none is available)
+Returns S_OK with connection in Value or S_ERROR
+the calling method is responsible for closing this connection once it is no
+longer needed.
 
 
 
 
-    Some high level methods have been added to avoid the need to write SQL
-    statement in most common cases. They should be used instead of low level
-    _insert, _update methods when ever possible.
+Some high level methods have been added to avoid the need to write SQL
+statement in most common cases. They should be used instead of low level
+_insert, _update methods when ever possible.
 
-    buildCondition( self, condDict = None, older = None, newer = None,
-                      timeStamp = None, orderAttribute = None, limit = False,
-                      greater = None, smaller = None ):
+buildCondition( self, condDict = None, older = None, newer = None,
+                  timeStamp = None, orderAttribute = None, limit = False,
+                  greater = None, smaller = None ):
 
-      Build SQL condition statement from provided condDict and other extra check on
-      a specified time stamp.
-      The conditions dictionary specifies for each attribute one or a List of possible
-      values
-      greater and smaller are dictionaries in which the keys are the names of the fields,
-      that are requested to be >= or < than the corresponding value.
-      For compatibility with current usage it uses Exceptions to exit in case of
-      invalid arguments
+  Build SQL condition statement from provided condDict and other extra check on
+  a specified time stamp.
+  The conditions dictionary specifies for each attribute one or a List of possible
+  values
+  greater and smaller are dictionaries in which the keys are the names of the fields,
+  that are requested to be >= or < than the corresponding value.
+  For compatibility with current usage it uses Exceptions to exit in case of
+  invalid arguments
 
 
-    insertFields( self, tableName, inFields = None, inValues = None, conn = None, inDict = None ):
+insertFields( self, tableName, inFields = None, inValues = None, conn = None, inDict = None ):
 
-      Insert a new row in "tableName" assigning the values "inValues" to the
-      fields "inFields".
-      Alternatively inDict can be used
-      String type values will be appropriately escaped.
+  Insert a new row in "tableName" assigning the values "inValues" to the
+  fields "inFields".
+  Alternatively inDict can be used
+  String type values will be appropriately escaped.
 
 
-    updateFields( self, tableName, updateFields = None, updateValues = None,
-                  condDict = None,
-                  limit = False, conn = None,
-                  updateDict = None,
-                  older = None, newer = None,
-                  timeStamp = None, orderAttribute = None ):
+updateFields( self, tableName, updateFields = None, updateValues = None,
+              condDict = None,
+              limit = False, conn = None,
+              updateDict = None,
+              older = None, newer = None,
+              timeStamp = None, orderAttribute = None ):
 
-      Update "updateFields" from "tableName" with "updateValues".
-      updateDict alternative way to provide the updateFields and updateValues
-      N records can match the condition
-      return S_OK( number of updated rows )
-      if limit is not False, the given limit is set
-      String type values will be appropriately escaped.
+  Update "updateFields" from "tableName" with "updateValues".
+  updateDict alternative way to provide the updateFields and updateValues
+  N records can match the condition
+  return S_OK( number of updated rows )
+  if limit is not False, the given limit is set
+  String type values will be appropriately escaped.
 
 
-    deleteEntries( self, tableName,
-                   condDict = None,
-                   limit = False, conn = None,
-                   older = None, newer = None,
-                   timeStamp = None, orderAttribute = None ):
-
-      Delete rows from "tableName" with
-      N records can match the condition
-      if limit is not False, the given limit is set
-      String type values will be appropriately escaped, they can be single values or lists of values.
-
-
-    getFields( self, tableName, outFields = None,
+deleteEntries( self, tableName,
                condDict = None,
                limit = False, conn = None,
                older = None, newer = None,
                timeStamp = None, orderAttribute = None ):
 
-      Select "outFields" from "tableName" with condDict
-      N records can match the condition
-      return S_OK( tuple(Field,Value) )
-      if limit is not False, the given limit is set
-      String type values will be appropriately escaped, they can be single values or lists of values.
+  Delete rows from "tableName" with
+  N records can match the condition
+  if limit is not False, the given limit is set
+  String type values will be appropriately escaped, they can be single values or lists of values.
+
+
+getFields( self, tableName, outFields = None,
+           condDict = None,
+           limit = False, conn = None,
+           older = None, newer = None,
+           timeStamp = None, orderAttribute = None ):
 
-      for compatibility with other methods condDict keyed argument is added
+  Select "outFields" from "tableName" with condDict
+  N records can match the condition
+  return S_OK( tuple(Field,Value) )
+  if limit is not False, the given limit is set
+  String type values will be appropriately escaped, they can be single values or lists of values.
 
+  for compatibility with other methods condDict keyed argument is added
 
-    getCounters( self, table, attrList, condDict = None, older = None,
-                 newer = None, timeStamp = None, connection = False ):
 
-      Count the number of records on each distinct combination of AttrList, selected
-      with condition defined by condDict and time stamps
+getCounters( self, table, attrList, condDict = None, older = None,
+             newer = None, timeStamp = None, connection = False ):
 
+  Count the number of records on each distinct combination of AttrList, selected
+  with condition defined by condDict and time stamps
 
-    getDistinctAttributeValues( self, table, attribute, condDict = None, older = None,
-                                newer = None, timeStamp = None, connection = False ):
 
-      Get distinct values of a table attribute under specified conditions
+getDistinctAttributeValues( self, table, attribute, condDict = None, older = None,
+                            newer = None, timeStamp = None, connection = False ):
+
+  Get distinct values of a table attribute under specified conditions
 
 
 """
+
 import collections
 import functools
 import json
@@ -760,6 +761,7 @@ class MySQL:
 
         :return: S_OK with number of updated registers upon success.
                  S_ERROR upon error.
+                 lastRowId: if set, added to the returned dictionary
         """
 
         self.log.debug(f"_update: {self._safeCmd(cmd)}")
@@ -774,9 +776,11 @@ class MySQL:
         try:
             cursor = connection.cursor()
             res = cursor.execute(cmd, args=args)
+
             retDict = S_OK(res)
             if cursor.lastrowid:
                 retDict["lastRowId"] = cursor.lastrowid
+
         except Exception as x:
             retDict = self._except("_update", x, "Execution failed.", cmd, debug)
 

DIRAC/Core/Utilities/Os.py

@@ -3,10 +3,11 @@
    by default on Error they return None
 """
 import os
+import threading
 
 import DIRAC
-from DIRAC.Core.Utilities.Subprocess import shellCall, systemCall
 from DIRAC.Core.Utilities import List
+from DIRAC.Core.Utilities.Subprocess import shellCall, systemCall
 
 DEBUG = 0
 
@@ -128,3 +129,33 @@ def sourceEnv(timeout, cmdTuple, inputEnv=None):
     result["stderr"] = stderr
 
     return result
+
+
+def safe_listdir(directory, timeout=60):
+    """This is a "safe" list directory,
+    for lazily-loaded File Systems like CVMFS.
+    There's by default a 60 seconds timeout.
+
+    .. warning::
+        There is no distinction between an empty directory, and a non existent one.
+        It will return `[]` in both cases.
+
+    :param str directory: directory to list
+    :param int timeout: optional timeout, in seconds. Defaults to 60.
+    """
+
+    def listdir(directory):
+        try:
+            return os.listdir(directory)
+        except FileNotFoundError:
+            print(f"{directory} not found")
+            return []
+
+    contents = []
+    t = threading.Thread(target=lambda: contents.extend(listdir(directory)))
+    t.daemon = True  # don't delay program's exit
+    t.start()
+    t.join(timeout)
+    if t.is_alive():
+        return None  # timeout
+    return contents