DIRAC 9.0.14__tar.gz → 9.0.16__tar.gz

dirac-9.0.16/AGENTS.md

@@ -0,0 +1,17 @@
+# DIRAC Agent Guidelines
+
+## Build/Lint/Test Commands
+- **Build**: `pip install -e .`
+- **Lint**: `ruff check src/ && pylint src/`
+- **Test**: `pytest tests/`
+- **Single test**: `pytest src/DIRAC/path/to/test.py::test_function`
+
+## Code Style Guidelines
+- **Formatting**: Use `black` with line length 120 (configured in pyproject.toml)
+- **Imports**: Absolute imports only; sort with `isort` (black profile)
+- **Naming**: CamelCase for classes, snake_case for functions/variables
+- **Types**: Use type hints; run `mypy` for strict checking
+- **Error handling**: Return `S_OK(result)` or `S_ERROR(message)` from DIRAC.Core.Utilities.ReturnValues
+- **Logging**: Use `gLogger.info/warn/error` (from DIRAC import gLogger)
+- **Docstrings**: Follow Google/NumPy style where present
+- **Security**: Never log secrets; validate inputs

{dirac-9.0.14/src/DIRAC.egg-info → dirac-9.0.16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: DIRAC
-Version: 9.0.14
+Version: 9.0.16
 Summary: DIRAC is an interware, meaning a software framework for distributed computing.
 Home-page: https://github.com/DIRACGrid/DIRAC/
 License: GPL-3.0-only
@@ -19,7 +19,7 @@ Requires-Dist: cachetools
 Requires-Dist: certifi
 Requires-Dist: cwltool
 Requires-Dist: diraccfg
-Requires-Dist: DIRACCommon==v9.0.14
+Requires-Dist: DIRACCommon==v9.0.16
 Requires-Dist: diracx-client>=v0.0.1
 Requires-Dist: diracx-core>=v0.0.1
 Requires-Dist: diracx-cli>=v0.0.1

{dirac-9.0.14 → dirac-9.0.16}/dirac.cfg

@@ -1131,7 +1131,7 @@ Operations
         Remote = xroot, root
       }
       # Module used for InputData resolution if not specified in the JDL
-      InputDataModule = DIRAC.Core.Utilities.InputDataResolution
+      InputDataModule = DIRAC.WorkloadManagementSystem.Client.InputDataResolution
     }
     Logging
     {

{dirac-9.0.14 → dirac-9.0.16}/release.notes

@@ -1,3 +1,79 @@
+[v9.0.16]
+
+*Tornado
+
+FIX: (#8426) return when retrying another Tornado service
+
+*Resources
+
+FIX: (#8424) SingularityComputingElement: convert MemoryLimitMB to int before comparison with maxRAM
+
+*WorkloadManagementSystem
+
+CHANGE: (#8423) rescheduleJobs: allow reusing existing JobDB, TaskQueueDB, jobLoggingDB instances
+
+*WorkloadManagement
+
+NEW: (#8422) Support overriding configuration servers ins pilot JSON
+
+[v9.0.15]
+
+*DIRAC
+
+FIX: (#8421) Fix default InputDataModule path in dirac.cfg
+
+*Core
+
+CHANGE: (#8419) the weekly index for OpenSearch uses also the year as a prefix
+FIX: (#8405) rootPath for server installation can be a random string, followed by a timestamp
+
+*RequestManagementSystem
+
+CHANGE: (#8418) just ignore a shifter if we cannot get a matching proxy
+
+*Resources
+
+FIX: (#8417) SLURM getCEStatus last empty line
+FIX: (#8412) add further details when time left cannot be obtained
+FIX: (#8408) PoolComputingElement: fix: do not set a fictional ram limit, use 0 for no-limit
+FIX: (#8406) adding basic RAM capabilities to InProcessCE
+CHANGE: (#8404) improve getReplicas performances by sorting LFNs
+CHANGE: (#8332) set default time for gfal operations
+
+*WorkloadManagementSystem
+
+NEW: (#8414) added a new table to TaskQueueDB for RAM requirements and matching
+NEW: (#8366) InProcessComputingElement takes RAM requirements into consideration for jobs' matching
+NEW: (#8366) PoolComputingElement takes RAM requirements into consideration when subdividing the pool
+NEW: (#8366) SingularityComputingElement can enforce CG2 RAM limits
+
+*DataManagementSystem
+
+FIX: (#8413) check properly if an FTS error is recoverable
+FIX: (#8410) Only submit FTS job with tokens if all hop support tokens
+CHANGE: (#8402) batch getReplicas at the FileCatalogClient level instead of at the DM level for speedup
+
+*Interfaces
+
+FIX: (#8409) inputDataPolicy should not change the inputDataModule but set a parameter inside the arguments.
+CHANGE: (#8407) Dirac().getInputDataCatalog - allow changing the inputDataPolicy
+
+*TransformationSystem
+
+CHANGE: (#8404) TransformationAgent does not bulk calls to dm.getReplicas
+
+*FrameworkSystem
+
+CHANGE: (#8403) (SystemAdministrator): add the possibility to install non-released software versions
+
+*WorkloadManagement
+
+NEW: (#8399) ability to adjust job wrapper log level
+
+*CSAPI
+
+FIX: (#8398) Have CSAPI return AffiliationEnds if existing
+
 [v9.0.14]
 
 

{dirac-9.0.14 → dirac-9.0.16}/setup.cfg

@@ -26,7 +26,7 @@ install_requires =
 	certifi
 	cwltool
 	diraccfg
-	DIRACCommon==v9.0.14
+	DIRACCommon==v9.0.16
 	diracx-client >=v0.0.1
 	diracx-core >=v0.0.1
 	diracx-cli >=v0.0.1

{dirac-9.0.14 → dirac-9.0.16}/src/DIRAC/ConfigurationSystem/Client/CSAPI.py

@@ -299,8 +299,19 @@ class CSAPI:
                     if "Users" in groupsDict[group] and entity in groupsDict[group]["Users"]:
                         entitiesDict[entity]["Groups"].append(group)
                 entitiesDict[entity]["Groups"].sort()
+                entitiesDict[entity]["AffiliationEnds"] = self.getUserAffiliationEnds(entity)
         return S_OK(entitiesDict)
 
+    def getUserAffiliationEnds(self, nick):
+        affiliation_ends_current = {}
+        csSection = f"{self.__baseSecurity}/Users/{nick}/"
+        user_sections = self.__csMod.getSections(csSection)
+        if "AffiliationEnds" in user_sections:
+            affiliation_ends_opts = self.__csMod.getOptions(f"{csSection}/AffiliationEnds")
+            for vo_ in affiliation_ends_opts:
+                affiliation_ends_current[vo_] = self.__csMod.getValue(f"{csSection}/AffiliationEnds/{vo_}")
+        return affiliation_ends_current
+
     def listGroups(self):
         """
         List all groups

{dirac-9.0.14 → dirac-9.0.16}/src/DIRAC/Core/Tornado/Client/private/TornadoBaseClient.py

@@ -599,7 +599,7 @@ class TornadoBaseClient:
             if url not in self.__bannedUrls:
                 self.__bannedUrls += [url]
             if retry < self.__nbOfUrls - 1:
-                self._request(retry=retry + 1, outputFile=outputFile, **kwargs)
+                return self._request(retry=retry + 1, outputFile=outputFile, **kwargs)
 
             errStr = f"{str(e)}: {rawText}"
             return S_ERROR(errStr)

{dirac-9.0.14 → dirac-9.0.16}/src/DIRAC/Core/Utilities/CGroups2.py

@@ -300,6 +300,7 @@ class CG2Manager(metaclass=DIRACSingleton):
             if "ceParameters" in kwargs:
                 if cpuLimit := kwargs["ceParameters"].get("CPULimit", None):
                     cores = float(cpuLimit)
+                # MemoryLimitMB should be the job upper limit
                 if memoryMB := int(kwargs["ceParameters"].get("MemoryLimitMB", 0)):
                     memory = memoryMB * 1024 * 1024
                 if kwargs["ceParameters"].get("MemoryNoSwap", "no").lower() in ("yes", "true"):

{dirac-9.0.14 → dirac-9.0.16}/src/DIRAC/Core/Utilities/ElasticSearchDB.py

@@ -621,7 +621,7 @@ class ElasticSearchDB:
         if period.lower() == "day":
             suffix = todayUTC.strftime("%Y-%m-%d")
         elif period.lower() == "week":
-            suffix = todayUTC.isocalendar()[1]
+            suffix = f"{todayUTC.strftime('%Y')}-{todayUTC.isocalendar()[1]}"
         elif period.lower() == "month":
             suffix = todayUTC.strftime("%Y-%m")
         elif period.lower() == "year":

{dirac-9.0.14 → dirac-9.0.16}/src/DIRAC/Core/Utilities/MySQL.py

@@ -903,6 +903,21 @@ class MySQL:
                     return createView
         return S_OK()
 
+    def _parseForeignKeyReference(self, auxTable, defaultKey):
+        """
+        Parse foreign key reference in format 'Table' or 'Table.key'
+
+        :param str auxTable: Foreign key reference (e.g., 'MyTable' or 'MyTable.id')
+        :param str defaultKey: Default key name if not specified in auxTable
+        :return: tuple (table_name, key_name)
+        """
+        if "." in auxTable:
+            parts = auxTable.split(".", 1)
+            if len(parts) != 2:
+                raise ValueError(f"Invalid foreign key reference format: {auxTable}")
+            return parts[0], parts[1]
+        return auxTable, defaultKey
+
     def _createTables(self, tableDict, force=False):
         """
         tableDict:
@@ -957,30 +972,37 @@ class MySQL:
             if "Fields" not in thisTable:
                 return S_ERROR(DErrno.EMYSQL, f"Missing `Fields` key in `{table}` table dictionary")
 
-        tableCreationList = [[]]
-
+        # Build dependency-ordered list of tables to create
+        # Tables with foreign keys must be created after their referenced tables
+        tableCreationList = []
         auxiliaryTableList = []
 
-        i = 0
+        # Get list of existing tables in the database to handle migrations
+        existingTablesResult = self._query("SHOW TABLES")
+        if not existingTablesResult["OK"]:
+            return existingTablesResult
+        existingTables = [t[0] for t in existingTablesResult["Value"]]
+
         extracted = True
         while tableList and extracted:
             # iterate extracting tables from list if they only depend on
             # already extracted tables.
             extracted = False
-            auxiliaryTableList += tableCreationList[i]
-            i += 1
-            tableCreationList.append([])
+            currentLevelTables = []
+
             for table in list(tableList):
                 toBeExtracted = True
                 thisTable = tableDict[table]
                 if "ForeignKeys" in thisTable:
                     thisKeys = thisTable["ForeignKeys"]
                     for key, auxTable in thisKeys.items():
-                        forTable = auxTable.split(".")[0]
-                        forKey = key
-                        if forTable != auxTable:
-                            forKey = auxTable.split(".")[1]
-                        if forTable not in auxiliaryTableList:
+                        try:
+                            forTable, forKey = self._parseForeignKeyReference(auxTable, key)
+                        except ValueError as e:
+                            return S_ERROR(DErrno.EMYSQL, str(e))
+
+                        # Check if the referenced table is either being created or already exists
+                        if forTable not in auxiliaryTableList and forTable not in existingTables:
                             toBeExtracted = False
                             break
                         if key not in thisTable["Fields"]:
@@ -988,24 +1010,29 @@ class MySQL:
                                 DErrno.EMYSQL,
                                 f"ForeignKey `{key}` -> `{forKey}` not defined in Primary table `{table}`.",
                             )
-                        if forKey not in tableDict[forTable]["Fields"]:
+                        # Only validate field existence if the referenced table is in tableDict
+                        if forTable in tableDict and forKey not in tableDict[forTable]["Fields"]:
                             return S_ERROR(
                                 DErrno.EMYSQL,
-                                "ForeignKey `%s` -> `%s` not defined in Auxiliary table `%s`."
-                                % (key, forKey, forTable),
+                                f"ForeignKey `{key}` -> `{forKey}` not defined in Auxiliary table `{forTable}`.",
                             )
 
                 if toBeExtracted:
                     # self.log.debug('Table %s ready to be created' % table)
                     extracted = True
                     tableList.remove(table)
-                    tableCreationList[i].append(table)
+                    currentLevelTables.append(table)
+
+            if currentLevelTables:
+                tableCreationList.append(currentLevelTables)
+                auxiliaryTableList.extend(currentLevelTables)
 
         if tableList:
             return S_ERROR(DErrno.EMYSQL, f"Recursive Foreign Keys in {', '.join(tableList)}")
 
-        for tableList in tableCreationList:
-            for table in tableList:
+        # Create tables level by level
+        for levelTables in tableCreationList:
+            for table in levelTables:
                 # Check if Table exist
                 retDict = self.__checkTable(table, force=force)
                 if not retDict["OK"]:
@@ -1035,18 +1062,17 @@ class MySQL:
                     for index in indexDict:
                         indexedFields = "`, `".join(indexDict[index])
                         cmdList.append(f"UNIQUE INDEX `{index}` ( `{indexedFields}` )")
+
                 if "ForeignKeys" in thisTable:
                     thisKeys = thisTable["ForeignKeys"]
                     for key, auxTable in thisKeys.items():
-                        forTable = auxTable.split(".")[0]
-                        forKey = key
-                        if forTable != auxTable:
-                            forKey = auxTable.split(".")[1]
+                        try:
+                            forTable, forKey = self._parseForeignKeyReference(auxTable, key)
+                        except ValueError as e:
+                            return S_ERROR(DErrno.EMYSQL, str(e))
 
-                        # cmdList.append( '`%s` %s' % ( forTable, tableDict[forTable]['Fields'][forKey] )
                         cmdList.append(
-                            "FOREIGN KEY ( `%s` ) REFERENCES `%s` ( `%s` )"
-                            " ON DELETE RESTRICT" % (key, forTable, forKey)
+                            f"FOREIGN KEY ( `{key}` ) REFERENCES `{forTable}` ( `{forKey}` ) ON DELETE RESTRICT"
                         )
 
                 engine = thisTable.get("Engine", "InnoDB")
@@ -1058,7 +1084,7 @@ class MySQL:
                     engine,
                     charset,
                 )
-                retDict = self._transaction([cmd])
+                retDict = self._update(cmd)
                 if not retDict["OK"]:
                     return retDict
                 # self.log.debug('Table %s created' % table)

{dirac-9.0.14 → dirac-9.0.16}/src/DIRAC/DataManagementSystem/Client/DataManager.py

@@ -22,7 +22,7 @@ from DIRAC import S_OK, S_ERROR, gLogger, gConfig
 from DIRAC.Core.Utilities import DErrno
 from DIRAC.Core.Utilities.Adler import fileAdler, compareAdler
 from DIRAC.Core.Utilities.File import makeGuid, getSize
-from DIRAC.Core.Utilities.List import randomize, breakListIntoChunks
+from DIRAC.Core.Utilities.List import randomize
 from DIRAC.Core.Utilities.ReturnValues import returnSingleResult
 from DIRAC.Core.Security.ProxyInfo import getProxyInfo
 from DIRAC.Core.Security.ProxyInfo import getVOfromProxyGroup
@@ -1677,18 +1677,15 @@ class DataManager:
         """get replicas from catalogue and filter if requested
         Warning: all filters are independent, hence active and preferDisk should be set if using forJobs
         """
-        catalogReplicas = {}
-        failed = {}
+
         if not protocol:
             protocol = self.registrationProtocol
 
-        for lfnChunk in breakListIntoChunks(lfns, 1000):
-            res = self.fileCatalog.getReplicas(lfnChunk, allStatus=allStatus)
-            if res["OK"]:
-                catalogReplicas.update(res["Value"]["Successful"])
-                failed.update(res["Value"]["Failed"])
-            else:
-                return res
+        res = self.fileCatalog.getReplicas(lfns, allStatus=allStatus)
+        if not res["OK"]:
+            return res
+        catalogReplicas = res["Value"]["Successful"]
+        failed = res["Value"]["Failed"]
         if not getUrl:
             for lfn in catalogReplicas:
                 catalogReplicas[lfn] = dict.fromkeys(catalogReplicas[lfn], True)

{dirac-9.0.14 → dirac-9.0.16}/src/DIRAC/DataManagementSystem/Client/FTS3Job.py

@@ -272,7 +272,7 @@ class FTS3Job(JSerializable):
 
                 # If the file is failed, check if it is recoverable
                 if file_state in FTS3File.FTS_FAILED_STATES:
-                    if not fileDict.get("Recoverable", True):
+                    if not fileDict.get("recoverable", True):
                         filesStatus[file_id]["status"] = "Defunct"
 
             # If the file is not in a final state, but the job is, we return an error
@@ -400,7 +400,7 @@ class FTS3Job(JSerializable):
         :return: S_OK( (job object, list of ftsFileIDs in the job))
         """
 
-        log = gLogger.getSubLogger(f"constructTransferJob/{self.operationID}/{self.sourceSE}_{self.targetSE}")
+        log = gLogger.getLocalSubLogger(f"constructTransferJob/{self.operationID}/{self.sourceSE}_{self.targetSE}")
 
         isMultiHop = False
         useTokens = False
@@ -411,6 +411,15 @@ class FTS3Job(JSerializable):
                 log.debug(f"Multihop job has {len(allLFNs)} files while only 1 allowed")
                 return S_ERROR(errno.E2BIG, "Trying multihop job with more than one file !")
             allHops = [(self.sourceSE, self.multiHopSE), (self.multiHopSE, self.targetSE)]
+            if tokensEnabled:
+                tokensEnabled = all(
+                    [
+                        self.__seTokenSupport(StorageElement(seName))
+                        for seName in (self.sourceSE, self.multiHopSE, self.targetSE)
+                    ]
+                )
+                if not tokensEnabled:
+                    log.warn("Not using token because not all hop supports it")
             isMultiHop = True
         else:
             allHops = [(self.sourceSE, self.targetSE)]
@@ -736,7 +745,7 @@ class FTS3Job(JSerializable):
         :return: S_OK( (job object, list of ftsFileIDs in the job))
         """
 
-        log = gLogger.getSubLogger(f"constructStagingJob/{self.operationID}/{self.targetSE}")
+        log = gLogger.getLocalSubLogger(f"constructStagingJob/{self.operationID}/{self.targetSE}")
 
         transfers = []
         fileIDsInTheJob = set()

{dirac-9.0.14 → dirac-9.0.16}/src/DIRAC/FrameworkSystem/Service/SystemAdministratorHandler.py

@@ -1,5 +1,5 @@
-""" SystemAdministrator service is a tool to control and monitor the DIRAC services and agents
-"""
+"""SystemAdministrator service is a tool to control and monitor the DIRAC services and agents"""
+
 import socket
 import os
 import re
@@ -254,17 +254,38 @@ class SystemAdministratorHandler(RequestHandler):
     types_updateSoftware = [str]
 
     def export_updateSoftware(self, version):
+        """Update DIRAC, or its extension, to a version. Use extension_name==version for the DIRAC extension.
+
+        A version can be:
+        - a PEP440 valid version of DIRAC.
+        - a PEP440 valid version of a DIRAC extension.
+        - "integration" or "devel" or "master" or "main" would all be interpreted as git+https://github.com/DIRACGrid/DIRAC.git@integration#egg=DIRAC[server]
+        - a git tag/branch like git+https://github.com/fstagni/DIRAC.git@test_branch#egg=DIRAC[server]
+        """
         # Validate and normalise the requested version
         primaryExtension = None
         if "==" in version:
             primaryExtension, version = version.split("==")
-        try:
-            version = Version(version)
-        except InvalidVersion:
-            self.log.exception("Invalid version passed", version)
-            return S_ERROR(f"Invalid version passed {version!r}")
-        isPrerelease = version.is_prerelease
-        version = f"v{version}"
+
+        released_version = True
+        isPrerelease = False
+
+        # Special cases (e.g. installing the integration/main branch)
+        if version.lower() in ["integration", "devel", "master", "main"]:
+            released_version = False
+            version = "git+https://github.com/DIRACGrid/DIRAC.git@integration#egg=DIRAC[server]"
+
+        if released_version:
+            try:
+                version = Version(version)
+                isPrerelease = version.is_prerelease
+                version = f"v{version}"
+            except InvalidVersion:
+                if "https://" in version:
+                    released_version = False
+                else:
+                    self.log.exception("Invalid version passed", version)
+                    return S_ERROR(f"Invalid version passed {version!r}")
 
         # Find what to install
         otherExtensions = []
@@ -290,10 +311,11 @@ class SystemAdministratorHandler(RequestHandler):
             installer.flush()
             self.log.info("Downloaded DIRACOS installer to", installer.name)
 
+            directory = version if released_version else version.split("@")[1].split("#")[0]
             newProPrefix = os.path.join(
                 rootPath,
                 "versions",
-                f"{version}-{datetime.utcnow().strftime('%s')}",
+                f"{directory}-{datetime.utcnow().strftime('%s')}",
             )
             installPrefix = os.path.join(newProPrefix, f"{platform.system()}-{platform.machine()}")
             self.log.info("Running DIRACOS installer for prefix", installPrefix)
@@ -313,7 +335,15 @@ class SystemAdministratorHandler(RequestHandler):
         cmd = [f"{installPrefix}/bin/pip", "install", "--no-color", "-v"]
         if isPrerelease:
             cmd += ["--pre"]
-        cmd += [f"{primaryExtension}[server]=={version}"]
+        if released_version:
+            cmd += [f"{primaryExtension}[server]=={version}"]
+        else:
+            # from here on we assume a version like git+https://github.com/DIRACGrid/DIRAC.git@integration#egg=DIRAC[server]
+            # is specified, for the primaryExtension
+            if not version.startswith("git+"):
+                version = f"git+{version}"
+            cmd += [version]
+
         cmd += [f"{e}[server]" for e in otherExtensions]
         r = subprocess.run(
             cmd,

{dirac-9.0.14 → dirac-9.0.16}/src/DIRAC/Interfaces/API/Dirac.py

@@ -208,7 +208,9 @@ class Dirac(API):
         return S_OK("Nothing to do")
 
     #############################################################################
-    def getInputDataCatalog(self, lfns, siteName="", fileName="pool_xml_catalog.xml", ignoreMissing=False):
+    def getInputDataCatalog(
+        self, lfns, siteName="", fileName="pool_xml_catalog.xml", ignoreMissing=False, inputDataPolicy=None
+    ):
         """This utility will create a pool xml catalogue slice for the specified LFNs using
         the full input data resolution policy plugins for the VO.
 
@@ -228,6 +230,10 @@ class Dirac(API):
         :type siteName: string
         :param fileName: Catalogue name (can include path)
         :type fileName: string
+        :param ignoreMissing: Flag to ignore missing files
+        :type ignoreMissing: bool
+        :param inputDataPolicy: Input data policy module to use
+        :type inputDataPolicy: string or None
         :returns: S_OK,S_ERROR
 
         """
@@ -248,8 +254,8 @@ class Dirac(API):
 
         self.log.verbose(localSEList)
 
-        inputDataPolicy = Operations().getValue("InputDataPolicy/InputDataModule")
-        if not inputDataPolicy:
+        inputDataModule = Operations().getValue("InputDataPolicy/InputDataModule")
+        if not inputDataModule:
             return self._errorReport("Could not retrieve /DIRAC/Operations/InputDataPolicy/InputDataModule for VO")
 
         self.log.info(f"Attempting to resolve data for {siteName}")
@@ -279,11 +285,13 @@ class Dirac(API):
 
         self.log.verbose(configDict)
         argumentsDict = {"FileCatalog": resolvedData, "Configuration": configDict, "InputData": lfns}
+        if inputDataPolicy:
+            argumentsDict["Job"] = {"InputDataPolicy": inputDataPolicy}
         if ignoreMissing:
             argumentsDict["IgnoreMissing"] = True
         self.log.verbose(argumentsDict)
 
-        result = self.objectLoader.loadObject(inputDataPolicy)
+        result = self.objectLoader.loadObject(inputDataModule)
         if not result["OK"]:
             return result
         module = result["Value"](argumentsDict)

{dirac-9.0.14 → dirac-9.0.16}/src/DIRAC/Interfaces/API/Job.py

@@ -1,27 +1,28 @@
 """
-   Job Base Class
+Job Base Class
 
-   This class provides generic job definition functionality suitable for any VO.
+This class provides generic job definition functionality suitable for any VO.
 
-   Helper functions are documented with example usage for the DIRAC API.  An example
-   script (for a simple executable) would be::
+Helper functions are documented with example usage for the DIRAC API.  An example
+script (for a simple executable) would be::
 
-     from DIRAC.Interfaces.API.Dirac import Dirac
-     from DIRAC.Interfaces.API.Job import Job
+  from DIRAC.Interfaces.API.Dirac import Dirac
+  from DIRAC.Interfaces.API.Job import Job
 
-     j = Job()
-     j.setCPUTime(500)
-     j.setExecutable('/bin/echo hello')
-     j.setExecutable('yourPythonScript.py')
-     j.setExecutable('/bin/echo hello again')
-     j.setName('MyJobName')
+  j = Job()
+  j.setCPUTime(500)
+  j.setExecutable('/bin/echo hello')
+  j.setExecutable('yourPythonScript.py')
+  j.setExecutable('/bin/echo hello again')
+  j.setName('MyJobName')
 
-     dirac = Dirac()
-     jobID = dirac.submitJob(j)
-     print 'Submission Result: ',jobID
+  dirac = Dirac()
+  jobID = dirac.submitJob(j)
+  print 'Submission Result: ',jobID
 
-   Note that several executables can be provided and wil be executed sequentially.
+Note that several executables can be provided and wil be executed sequentially.
 """
+
 import os
 import re
 import shlex
@@ -517,6 +518,50 @@ class Job(API):
         return S_OK()
 
     #############################################################################
+    def setRAMRequirements(self, ramRequired: int = 0, maxRAM: int = 0):
+        """Helper function.
+        Specify the RAM requirements for the job, in MB. 0 (default) means no specific requirements.
+
+        Example usage:
+
+        >>> job = Job()
+        >>> job.setRAMRequirements(ramRequired=2000)
+        means that the job needs at least 2 GBs of RAM to work. This is taken into consideration at job's matching time.
+        The job definition does not specify an upper limit.
+        From a user's point of view this is fine (normally, not for admins).
+
+        >>> job.setRAMRequirements(ramRequired=500, maxRAM=3800)
+        means that the job needs 500 MBs of RAM to work. 3.8 GBs will then be the upper limit for CG2 limits.
+
+        >>> job.setRAMRequirements(ramRequired=3200, maxRAM=3200)
+        means that we should match this job if there is at least 3.2 available GBs of run. At the same time, CG2 will not allow to use more than that.
+
+        >>> job.setRAMRequirements(maxRAM=4000)
+        means that the job does not set a min amount of RAM (so can match--run "everywhere"), but the 4 GBs will then be the upper limit for CG2 limits.
+
+        >>> job.setRAMRequirements(ramRequired=8000, maxRAM=4000)
+        Makes no sense, an error will be raised
+        """
+        if ramRequired and maxRAM and ramRequired > maxRAM:
+            return self._reportError("Invalid settings, ramRequired is higher than maxRAM")
+
+        if ramRequired:
+            self._addParameter(
+                self.workflow,
+                "MinRAM",
+                "JDL",
+                ramRequired,
+                "MBs of RAM requested",
+            )
+        if maxRAM:
+            self._addParameter(
+                self.workflow,
+                "MaxRAM",
+                "JDL",
+                maxRAM,
+                "Max MBs of RAM to be used",
+            )
+
     def setNumberOfProcessors(self, numberOfProcessors=None, minNumberOfProcessors=None, maxNumberOfProcessors=None):
         """Helper function to set the number of processors required by the job.
 
@@ -709,7 +754,7 @@ class Job(API):
         Example usage:
 
         >>> job = Job()
-        >>> job.setTag( ['WholeNode','8GBMemory'] )
+        >>> job.setTag( ['WholeNode'] )
 
         :param tags: single tag string or a list of tags
         :type tags: str or python:list

{dirac-9.0.14 → dirac-9.0.16}/src/DIRAC/RequestManagementSystem/private/RequestTask.py

@@ -106,7 +106,8 @@ class RequestTask:
                     userDN, userGroup, requiredTimeLeft=1200, cacheTime=4 * 43200
                 )
             if not getProxy["OK"]:
-                return S_ERROR(f"unable to setup shifter proxy for {shifter}: {getProxy['Message']}")
+                self.log.error("unable to setup shifter proxy", f"{shifter}: {getProxy['Message']}")
+                continue
             chain = getProxy["chain"]
             fileName = getProxy["Value"]
             self.log.debug(f"got {shifter}: {userName} {userGroup}")