DIRAC 9.0.0a54__py3-none-any.whl → 9.0.7__py3-none-any.whl

DIRAC/Core/Utilities/TimeUtilities.py

@@ -1,260 +1,17 @@
-"""
-DIRAC TimeUtilities module
-Support for basic Date and Time operations
-based on system datetime module.
-
-It provides common interface to UTC timestamps,
-converter to string types and back.
-
-Useful timedelta constant are also provided to
-define time intervals.
-
-Notice: datetime.timedelta objects allow multiplication and division by interger
-but not by float. Thus:
-
-  - DIRAC.TimeUtilities.second * 1.5             is not allowed
-  - DIRAC.TimeUtilities.second * 3 / 2           is allowed
+"""Backward compatibility wrapper - moved to DIRACCommon
 
-An timeInterval class provides a method to check
-if a give datetime is in the defined interval.
+This module has been moved to DIRACCommon.Core.Utilities.TimeUtilities to avoid
+circular dependencies and allow DiracX to use these utilities without
+triggering DIRAC's global state initialization.
 
+All exports are maintained for backward compatibility.
 """
-import datetime
-import sys
-import time
-
-from DIRAC import gLogger
-
-# Some useful constants for time operations
-microsecond = datetime.timedelta(microseconds=1)
-second = datetime.timedelta(seconds=1)
-minute = datetime.timedelta(minutes=1)
-hour = datetime.timedelta(hours=1)
-day = datetime.timedelta(days=1)
-week = datetime.timedelta(days=7)
-
-
-def timeThis(method):
-    """Function to be used as a decorator for timing other functions/methods"""
-
-    def timed(*args, **kw):
-        """What actually times"""
-        ts = time.time()
-        result = method(*args, **kw)
-        if sys.stdout.isatty():
-            return result
-        te = time.time()
-
-        pre = datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S UTC ")
-
-        try:
-            pre += args[0].log.getName() + "/" + args[0].log.getSubName() + "   TIME: " + args[0].transString
-        except AttributeError:
-            try:
-                pre += args[0].log.getName() + "    TIME: " + args[0].transString
-            except AttributeError:
-                try:
-                    pre += args[0].log.getName() + "/" + args[0].log.getSubName() + "   TIME: "
-                except AttributeError:
-                    pre += "TIME: "
-        except IndexError:
-            pre += "TIME: "
-
-        argsLen = ""
-        if args:
-            try:
-                if isinstance(args[1], (list, dict)):
-                    argsLen = f"arguments len: {len(args[1])}"
-            except IndexError:
-                if kw:
-                    try:
-                        if isinstance(list(list(kw.items())[0])[1], (list, dict)):
-                            argsLen = f"arguments len: {len(list(list(kw.items())[0])[1])}"
-                    except IndexError:
-                        argsLen = ""
-
-        gLogger.info(f"{pre} Exec time ===> function {method.__name__!r} {argsLen} -> {te - ts:2.2f} sec")
-        return result
-
-    return timed
-
-
-def toEpoch(dateTimeObject=None):
-    """
-    Get seconds since epoch. Accepts datetime or date objects
-    """
-    return toEpochMilliSeconds(dateTimeObject) // 1000
+from functools import partial
 
+# Re-export everything from DIRACCommon for backward compatibility
+from DIRACCommon.Core.Utilities.TimeUtilities import *  # noqa: F401, F403
 
-def toEpochMilliSeconds(dateTimeObject=None):
-    """
-    Get milliseconds since epoch
-    """
-    if dateTimeObject is None:
-        dateTimeObject = datetime.datetime.utcnow()
-    if dateTimeObject.resolution == datetime.timedelta(days=1):
-        # Add time information corresponding to midnight UTC if it's a datetime.date
-        dateTimeObject = datetime.datetime.combine(
-            dateTimeObject, datetime.time.min.replace(tzinfo=datetime.timezone.utc)
-        )
-    posixTime = dateTimeObject.replace(tzinfo=datetime.timezone.utc).timestamp()
-    return int(posixTime * 1000)
-
-
-def fromEpoch(epoch):
-    """
-    Get datetime object from epoch
-    """
-    # Check if the timestamp is in milliseconds
-    if epoch > 10**17:  # nanoseconds
-        epoch /= 1000**3
-    elif epoch > 10**14:  # microseconds
-        epoch /= 1000**2
-    elif epoch > 10**11:  # milliseconds
-        epoch /= 1000
-    return datetime.datetime.utcfromtimestamp(epoch)
-
-
-def toString(myDate=None):
-    """
-    Convert to String
-    if argument type is neither _dateTimeType, _dateType, nor _timeType
-    the current dateTime converted to String is returned instead
-
-    Notice: datetime.timedelta are converted to strings using the format:
-      [day] days [hour]:[min]:[sec]:[microsec]
-      where hour, min, sec, microsec are always positive integers,
-      and day carries the sign.
-    To keep internal consistency we are using:
-      [hour]:[min]:[sec]:[microsec]
-      where min, sec, microsec are always positive integers and hour carries the sign.
-    """
-    if isinstance(myDate, datetime.date):
-        return str(myDate)
-
-    elif isinstance(myDate, datetime.time):
-        return "%02d:%02d:%02d.%06d" % (
-            myDate.days * 24 + myDate.seconds / 3600,
-            myDate.seconds % 3600 / 60,
-            myDate.seconds % 60,
-            myDate.microseconds,
-        )
-    else:
-        return toString(datetime.datetime.utcnow())
-
-
-def fromString(myDate=None):
-    """
-    Convert date/time/datetime String back to appropriated objects
-
-    The format of the string it is assume to be that returned by toString method.
-    See notice on toString method
-    On Error, return None
-
-    :param myDate: the date string to be converted
-    :type myDate: str or datetime.datetime
-    """
-    if isinstance(myDate, datetime.datetime):
-        return myDate
-    if isinstance(myDate, str):
-        if myDate.find(" ") > 0:
-            dateTimeTuple = myDate.split(" ")
-            dateTuple = dateTimeTuple[0].split("-")
-            try:
-                return datetime.datetime(year=dateTuple[0], month=dateTuple[1], day=dateTuple[2]) + fromString(
-                    dateTimeTuple[1]
-                )
-                # return datetime.datetime.utcnow().combine( fromString( dateTimeTuple[0] ),
-                #                                   fromString( dateTimeTuple[1] ) )
-            except Exception:
-                try:
-                    return datetime.datetime(
-                        year=int(dateTuple[0]), month=int(dateTuple[1]), day=int(dateTuple[2])
-                    ) + fromString(dateTimeTuple[1])
-                except ValueError:
-                    return None
-                # return datetime.datetime.utcnow().combine( fromString( dateTimeTuple[0] ),
-                #                                   fromString( dateTimeTuple[1] ) )
-        elif myDate.find(":") > 0:
-            timeTuple = myDate.replace(".", ":").split(":")
-            try:
-                if len(timeTuple) == 4:
-                    return datetime.timedelta(
-                        hours=int(timeTuple[0]),
-                        minutes=int(timeTuple[1]),
-                        seconds=int(timeTuple[2]),
-                        microseconds=int(timeTuple[3]),
-                    )
-                elif len(timeTuple) == 3:
-                    try:
-                        return datetime.timedelta(
-                            hours=int(timeTuple[0]),
-                            minutes=int(timeTuple[1]),
-                            seconds=int(timeTuple[2]),
-                            microseconds=0,
-                        )
-                    except ValueError:
-                        return None
-                else:
-                    return None
-            except Exception:
-                return None
-        elif myDate.find("-") > 0:
-            dateTuple = myDate.split("-")
-            try:
-                return datetime.date(int(dateTuple[0]), int(dateTuple[1]), int(dateTuple[2]))
-            except Exception:
-                return None
-
-    return None
-
-
-class timeInterval:
-    """
-    Simple class to define a timeInterval object able to check if a given
-    dateTime is inside
-    """
-
-    def __init__(self, initialDateTime, intervalTimeDelta):
-        """
-        Initialization method, it requires the initial dateTime and the
-        timedelta that define the limits.
-        The upper limit is not included thus it is [begin,end)
-        If not properly initialized an error flag is set, and subsequent calls
-        to any method will return None
-        """
-        if not isinstance(initialDateTime, datetime.datetime) or not isinstance(intervalTimeDelta, datetime.timedelta):
-            self.__error = True
-            return None
-        self.__error = False
-        if intervalTimeDelta.days < 0:
-            self.__startDateTime = initialDateTime + intervalTimeDelta
-            self.__endDateTime = initialDateTime
-        else:
-            self.__startDateTime = initialDateTime
-            self.__endDateTime = initialDateTime + intervalTimeDelta
-
-    def includes(self, myDateTime):
-        """ """
-        if self.__error:
-            return None
-        if not isinstance(myDateTime, datetime.datetime):
-            return None
-        if myDateTime < self.__startDateTime:
-            return False
-        if myDateTime >= self.__endDateTime:
-            return False
-        return True
-
-
-def queryTime(f):
-    """Decorator to measure the function call time"""
+from DIRAC import gLogger
 
-    def measureQueryTime(*args, **kwargs):
-        start = time.time()
-        result = f(*args, **kwargs)
-        if result["OK"] and "QueryTime" not in result:
-            result["QueryTime"] = time.time() - start
-        return result
 
-    return measureQueryTime
+timeThis = partial(timeThis, logger_info=gLogger.info)

DIRAC/Core/Utilities/test/Test_JDL.py

@@ -19,9 +19,6 @@ def jdl_monkey_business(monkeypatch):
     monkeypatch.setattr("DIRAC.Core.Base.API.getSites", lambda: S_OK(["LCG.IN2P3.fr"]))
     monkeypatch.setattr("DIRAC.WorkloadManagementSystem.Utilities.JobModel.getSites", lambda: S_OK(["LCG.IN2P3.fr"]))
     monkeypatch.setattr("DIRAC.Interfaces.API.Job.getDIRACPlatforms", lambda: S_OK("x86_64-slc6-gcc49-opt"))
-    monkeypatch.setattr(
-        "DIRAC.WorkloadManagementSystem.Utilities.JobModel.getDIRACPlatforms", lambda: S_OK("x86_64-slc6-gcc49-opt")
-    )
     yield
 
 

DIRAC/Core/Utilities/test/Test_Profiler.py

@@ -29,50 +29,50 @@ def test_base():
     time.sleep(1)
     p = Profiler(mainProcess.pid)
     res = p.pid()
-    assert res["OK"] is True
+    assert res["OK"] is True, res
     res = p.status()
-    assert res["OK"] is True
+    assert res["OK"] is True, res
     res = p.runningTime()
-    assert res["OK"] is True
+    assert res["OK"] is True, res
     assert res["Value"] > 0
 
     res = p.memoryUsage()
-    assert res["OK"] is True
+    assert res["OK"] is True, res
     assert res["Value"] > 0
     resWC = p.memoryUsage(withChildren=True)
-    assert resWC["OK"] is True
+    assert resWC["OK"] is True, res
     assert resWC["Value"] > 0
     assert resWC["Value"] >= res["Value"]
 
     res = p.vSizeUsage()
-    assert res["OK"] is True
+    assert res["OK"] is True, res
     assert res["Value"] > 0
     resWC = p.vSizeUsage(withChildren=True)
-    assert resWC["OK"] is True
+    assert resWC["OK"] is True, res
     assert resWC["Value"] > 0
     assert resWC["Value"] >= res["Value"]
 
     res = p.vSizeUsage()
-    assert res["OK"] is True
+    assert res["OK"] is True, res
     assert res["Value"] > 0
     resWC = p.vSizeUsage(withChildren=True)
-    assert resWC["OK"] is True
+    assert resWC["OK"] is True, res
     assert resWC["Value"] > 0
     assert resWC["Value"] >= res["Value"]
 
     res = p.numThreads()
-    assert res["OK"] is True
+    assert res["OK"] is True, res
     assert res["Value"] > 0
     resWC = p.numThreads(withChildren=True)
-    assert resWC["OK"] is True
+    assert resWC["OK"] is True, res
     assert resWC["Value"] > 0
     assert resWC["Value"] >= res["Value"]
 
     res = p.cpuPercentage()
-    assert res["OK"] is True
+    assert res["OK"] is True, res
     assert res["Value"] >= 0
     resWC = p.cpuPercentage(withChildren=True)
-    assert resWC["OK"] is True
+    assert resWC["OK"] is True, res
     assert resWC["Value"] >= 0
     assert resWC["Value"] >= res["Value"]
 
@@ -88,13 +88,13 @@ def test_cpuUsage():
     time.sleep(2)
     p = Profiler(mainProcess.pid)
     res = p.pid()
-    assert res["OK"] is True
+    assert res["OK"] is True, res
     res = p.status()
-    assert res["OK"] is True
+    assert res["OK"] is True, res
 
     # user
     res = p.cpuUsageUser()
-    assert res["OK"] is True
+    assert res["OK"] is True, res
     assert res["Value"] > 0
     resC = p.cpuUsageUser(withChildren=True)
     assert resC["OK"] is True
@@ -102,7 +102,7 @@ def test_cpuUsage():
     assert resC["Value"] >= res["Value"]
 
     res = p.cpuUsageUser()
-    assert res["OK"] is True
+    assert res["OK"] is True, res
     assert res["Value"] > 0
     resC = p.cpuUsageUser(withChildren=True)
     assert resC["OK"] is True
@@ -121,15 +121,15 @@ def test_cpuUsage():
 
     # system
     res = p.cpuUsageSystem()
-    assert res["OK"] is True
+    assert res["OK"] is True, res
     assert res["Value"] >= 0
     resWC = p.cpuUsageSystem(withChildren=True)
-    assert resWC["OK"] is True
+    assert resWC["OK"] is True, res
     assert resWC["Value"] >= 0
     assert resWC["Value"] >= res["Value"]
 
     res = p.cpuUsageSystem()
-    assert res["OK"] is True
+    assert res["OK"] is True, res
     assert res["Value"] > 0
     resC = p.cpuUsageSystem(withChildren=True)
     assert resC["OK"] is True

DIRAC/Core/scripts/dirac_agent.py

@@ -25,7 +25,7 @@ def main():
     localCfg.setConfigurationForAgent(agentName)
     localCfg.addDefaultEntry("/DIRAC/Security/UseServerCertificate", "yes")
     localCfg.addDefaultEntry("LogColor", True)
-    resultDict = localCfg.loadUserData()
+    resultDict = localCfg.loadUserData(requireSuccessfulSync=True)
     if not resultDict["OK"]:
         gLogger.error("There were errors when loading configuration", resultDict["Message"])
         sys.exit(1)

DIRAC/Core/scripts/dirac_apptainer_exec.py

@@ -1,6 +1,5 @@
 #!/usr/bin/env python
-""" Starts a DIRAC command inside an apptainer container.
-"""
+"""Starts a DIRAC command inside an apptainer container."""
 
 import os
 import sys
@@ -10,6 +9,7 @@ import DIRAC
 from DIRAC import S_ERROR, gConfig, gLogger
 from DIRAC.Core.Base.Script import Script
 from DIRAC.Core.Security.Locations import getCAsLocation, getProxyLocation, getVOMSLocation
+from DIRAC.Core.Utilities.Os import safe_listdir
 from DIRAC.Core.Utilities.Subprocess import systemCall
 
 
@@ -51,8 +51,9 @@ def main():
         if switch[0].lower() == "i" or switch[0].lower() == "image":
             user_image = switch[1]
 
-    dirac_env_var = os.environ.get("DIRAC", os.getcwd())
-    diracos_env_var = os.environ.get("DIRACOS", os.getcwd())
+    cwd = os.path.realpath(os.getcwd())
+    dirac_env_var = os.environ.get("DIRAC", cwd)
+    diracos_env_var = os.environ.get("DIRACOS", cwd)
     etc_dir = os.path.join(DIRAC.rootPath, "etc")
     rc_script = os.path.join(os.path.realpath(sys.base_prefix), "diracosrc")
 
@@ -73,7 +74,7 @@ def main():
     cmd.extend(["--contain"])  # use minimal /dev and empty other directories (e.g. /tmp and $HOME)
     cmd.extend(["--ipc"])  # run container in a new IPC namespace
     cmd.extend(["--pid"])  # run container in a new PID namespace
-    cmd.extend(["--bind", f"{os.getcwd()}:/mnt"])  # bind current directory for dirac_container.sh
+    cmd.extend(["--bind", cwd])  # bind current directory for dirac_container.sh
     if proxy_location:
         cmd.extend(["--bind", f"{proxy_location}:/etc/proxy"])  # bind proxy file
     cmd.extend(["--bind", f"{getCAsLocation()}:/etc/grid-security/certificates"])  # X509_CERT_DIR
@@ -83,11 +84,18 @@ def main():
     cmd.extend(["--bind", f"{vomses_location}:/etc/grid-security/vomses"])  # X509_VOMSES
     cmd.extend(["--bind", "{0}:{0}:ro".format(etc_dir)])  # etc dir for dirac.cfg
     cmd.extend(["--bind", "{0}:{0}:ro".format(os.path.join(os.path.realpath(sys.base_prefix)))])  # code dir
+    # here bind optional paths
+    for bind_path in gConfig.getValue("/Resources/Computing/Singularity/BindPaths", []):
+        if safe_listdir(bind_path):
+            cmd.extend(["--bind", f"{bind_path}:{bind_path}"])
+        else:
+            gLogger.warn(f"Bind path {bind_path} does not exist, skipping")
+    cmd.extend(["--cwd", cwd])  # set working directory
 
     rootImage = user_image or gConfig.getValue("/Resources/Computing/Singularity/ContainerRoot") or CONTAINER_DEFROOT
 
     if os.path.isdir(rootImage) or os.path.isfile(rootImage):
-        cmd.extend([rootImage, "/mnt/dirac_container.sh"])
+        cmd.extend([rootImage, f"{cwd}/dirac_container.sh"])
     else:
         # if we are here is because there's no image, or it is not accessible (e.g. not on CVMFS)
         gLogger.error("Apptainer image to exec not found: ", rootImage)
@@ -99,7 +107,8 @@ def main():
         gLogger.error(result["Message"])
         DIRAC.exit(1)
     if result["Value"][0] != 0:
-        gLogger.error(result["Value"][2])
+        gLogger.error("Apptainer command failed with exit code", result["Value"][0])
+        gLogger.error("Command output:", result["Value"])
         DIRAC.exit(2)
     gLogger.notice(result["Value"][1])
 

DIRAC/Core/scripts/dirac_platform.py

@@ -23,107 +23,16 @@ try:
 except Exception:
     import argparse
     import platform
-    import os
-    import sys
-    import re
-    import subprocess
 
     parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
     parser.parse_known_args()
 
-    # We need to patch python platform module. It does a string comparison for the libc versions.
-    # it fails when going from 2.9 to 2.10,
-    # the fix converts the version to a tuple and attempts a numeric comparison
-
-    _libc_search = re.compile(r"(__libc_init)" "|" "(GLIBC_([0-9.]+))" "|" r"(libc(_\w+)?\.so(?:\.(\d[0-9.]*))?)")
-
-    def libc_ver(executable=sys.executable, lib="", version="", chunksize=2048):
-        """Tries to determine the libc version that the file executable
-        (which defaults to the Python interpreter) is linked against.
-
-        Returns a tuple of strings (lib,version) which default to the
-        given parameters in case the lookup fails.
-
-        Note that the function has intimate knowledge of how different
-        libc versions add symbols to the executable and thus is probably
-        only useable for executables compiled using gcc.
-
-        The file is read and scanned in chunks of chunksize bytes.
-
-        """
-        with open(executable, "rb") as f:
-            binary = f.read(chunksize)
-            pos = 0
-            version = [0, 0, 0]
-            while True:
-                m = _libc_search.search(binary, pos)
-                if not m:
-                    binary = f.read(chunksize)
-                    if not binary:
-                        break
-                    pos = 0
-                    continue
-                libcinit, glibc, glibcversion, so, threads, soversion = m.groups()
-                if libcinit and not lib:
-                    lib = "libc"
-                elif glibc:
-                    glibcversion_parts = glibcversion.split(".")
-                    for i, part in enumerate(glibcversion_parts):
-                        try:
-                            glibcversion_parts[i] = int(part)
-                        except ValueError:
-                            glibcversion_parts[i] = 0
-                    if libcinit and not lib:
-                        lib = "libc"
-                    elif glibc:
-                        if lib != "glibc":
-                            lib = "glibc"
-                            version = glibcversion_parts
-                        elif glibcversion_parts > version:
-                            version = glibcversion_parts
-                elif so:
-                    if lib != "glibc":
-                        lib = "libc"
-                        version = max(version, soversion)
-                        if threads and version[-len(threads) :] != threads:
-                            version = version + threads
-                pos = m.end()
-        return lib, ".".join(map(str, version))
-
     # Command line interface
     def getPlatformString():
         # Modified to return our desired platform string, R. Graciani
         platformTuple = (platform.system(), platform.machine())
         if platformTuple[0] == "Linux":
-            sp = subprocess.Popen(["/sbin/ldconfig", "--print-cache"], stdout=subprocess.PIPE)
-            ldre = re.compile(r".*=> (.*/libc\.so\..*$)")
-            libs = []
-            for line in sp.stdout.readlines():
-                reM = ldre.match(line)
-                if reM:
-                    libs.append(reM.groups()[0])
-            if not libs:
-                # get version of higher libc installed
-                if platform.machine().find("64") != -1:
-                    lib = "/lib64"
-                else:
-                    lib = "/lib"
-                for libFile in os.listdir(lib):
-                    if libFile.find("libc-") == 0 or libFile.find("libc.so") == 0:
-                        libs.append(os.path.join(lib, libFile))
-            newest_lib = [0, 0, 0]
-            for lib in libs:
-                lib_parts = libc_ver(lib)[1].split(".")
-                for i, part in enumerate(lib_parts):
-                    try:
-                        lib_parts[i] = int(part)
-                    except ValueError:
-                        lib_parts[i] = 0
-                        # print "non integer version numbers"
-                if lib_parts > newest_lib:
-                    newest_lib = lib_parts
-
-            platformTuple += ("glibc-" + ".".join(map(str, newest_lib)),)
+            platformTuple += ("-".join(platform.libc_ver()),)
         elif platformTuple[0] == "Darwin":
             platformTuple += (".".join(platform.mac_ver()[0].split(".")[:2]),)
         else:

DIRAC/DataManagementSystem/Agent/FTS3Agent.py

@@ -719,10 +719,11 @@ class FTS3Agent(AgentModule):
         return self.dataOpSender.concludeSending()
 
     def __sendAccounting(self, ftsJob):
-        self.dataOpSender.sendData(
-            ftsJob.accountingDict,
-            commitFlag=True,
-            delayedCommit=True,
-            startTime=fromString(ftsJob.submitTime),
-            endTime=fromString(ftsJob.lastUpdate),
-        )
+        for accountingDict in ftsJob.accountingDicts:
+            self.dataOpSender.sendData(
+                accountingDict,
+                commitFlag=True,
+                delayedCommit=True,
+                startTime=fromString(ftsJob.submitTime),
+                endTime=fromString(ftsJob.lastUpdate),
+            )

DIRAC/DataManagementSystem/Agent/RequestOperations/RemoveFile.py

@@ -4,17 +4,17 @@
 # Date: 2013/03/25 07:44:19
 ########################################################################
 
-""" :mod: RemoveFile
+""":mod: RemoveFile
 
-    ================
+================
 
-    .. module: RemoveFile
+.. module: RemoveFile
 
-    :synopsis: removeFile operation handler
+:synopsis: removeFile operation handler
 
-    .. moduleauthor:: Krzysztof.Ciba@NOSPAMgmail.com
+.. moduleauthor:: Krzysztof.Ciba@NOSPAMgmail.com
 
-    removeFile operation handler
+removeFile operation handler
 """
 # #
 # @file RemoveFile.py
@@ -132,6 +132,7 @@ class RemoveFile(DMSRequestOperationsBase):
                     self.rmsMonitoringReporter.addRecord(
                         self.createRMSRecord("Successful", len(toRemoveDict) - len(bulkRemoval["Value"]))
                     )
+                toRemoveDict = bulkRemoval["Value"]
 
             # # 2nd step - single file removal
             for lfn, opFile in toRemoveDict.items():