DIRACCommon 9.0.0__py3-none-any.whl

DIRACCommon/WorkloadManagementSystem/Utilities/ParametricJob.py

@@ -0,0 +1,179 @@
+""" Utilities to process parametric job definitions and generate
+    bunches of parametric jobs. It exposes the following functions:
+
+    getParameterVectorLength() - to get the total size of the bunch of parametric jobs
+    generateParametricJobs() - to get a list of expanded descriptions of all the jobs
+"""
+import re
+
+from DIRACCommon.Core.Utilities.ClassAd.ClassAdLight import ClassAd
+from DIRACCommon.Core.Utilities.ReturnValues import S_OK, S_ERROR
+from DIRACCommon.Core.Utilities.DErrno import EWMSJDL
+
+
+def __getParameterSequence(nPar, parList=[], parStart=1, parStep=0, parFactor=1):
+    if parList:
+        if nPar != len(parList):
+            return []
+        else:
+            parameterList = list(parList)
+    else:
+        # The first parameter must have the same type as the other ones even if not defined explicitly
+        parameterList = [parStart * type(parFactor)(1) + type(parStep)(0)]
+        for np in range(1, nPar):
+            parameterList.append(parameterList[np - 1] * parFactor + parStep)
+
+    return parameterList
+
+
+def getParameterVectorLength(jobClassAd):
+    """Get the length of parameter vector in the parametric job description
+
+    :param jobClassAd: ClassAd job description object
+    :return: result structure with the Value: int number of parameter values, None if not a parametric job
+    """
+
+    nParValues = None
+    attributes = jobClassAd.getAttributes()
+    for attribute in attributes:
+        if attribute.startswith("Parameters"):
+            if jobClassAd.isAttributeList(attribute):
+                parameterList = jobClassAd.getListFromExpression(attribute)
+                nThisParValues = len(parameterList)
+            else:
+                nThisParValues = jobClassAd.getAttributeInt(attribute)
+            if nParValues is not None and nParValues != nThisParValues:
+                return S_ERROR(
+                    EWMSJDL,
+                    "Different length of parameter vectors: for %s, %s != %d" % (attribute, nParValues, nThisParValues),
+                )
+            nParValues = nThisParValues
+    if nParValues is not None and nParValues <= 0:
+        return S_ERROR(EWMSJDL, "Illegal number of job parameters %d" % (nParValues))
+    return S_OK(nParValues)
+
+
+def __updateAttribute(classAd, attribute, parName, parValue):
+    # If there is something to do:
+    pattern = r"%%\(%s\)s" % parName
+    if parName == "0":
+        pattern = "%s"
+    expr = classAd.get_expression(attribute)
+    if not re.search(pattern, expr):
+        return False
+
+    pattern = "%%(%s)s" % parName
+    if parName == "0":
+        pattern = "%s"
+
+    parValue = parValue.strip()
+    if classAd.isAttributeList(attribute):
+        parValue = parValue.strip()
+        if parValue.startswith("{"):
+            parValue = parValue.lstrip("{").rstrip("}").strip()
+
+    expr = classAd.get_expression(attribute)
+    newexpr = expr.replace(pattern, str(parValue))
+    classAd.set_expression(attribute, newexpr)
+    return True
+
+
+def generateParametricJobs(jobClassAd):
+    """Generate a series of ClassAd job descriptions expanding
+        job parameters
+
+    :param jobClassAd: ClassAd job description object
+    :return: list of ClassAd job description objects
+    """
+    if not jobClassAd.lookupAttribute("Parameters"):
+        return S_OK([jobClassAd.asJDL()])
+
+    result = getParameterVectorLength(jobClassAd)
+    if not result["OK"]:
+        return result
+    nParValues = result["Value"]
+    if nParValues is None:
+        return S_ERROR(EWMSJDL, "Can not determine the number of job parameters")
+
+    parameterDict = {}
+    attributes = jobClassAd.getAttributes()
+    for attribute in attributes:
+        for key in ["Parameters", "ParameterStart", "ParameterStep", "ParameterFactor"]:
+            if attribute.startswith(key):
+                seqID = "0" if "." not in attribute else attribute.split(".")[1]
+                parameterDict.setdefault(seqID, {})
+                if key == "Parameters":
+                    if jobClassAd.isAttributeList(attribute):
+                        parList = jobClassAd.getListFromExpression(attribute)
+                        if len(parList) != nParValues:
+                            return S_ERROR(EWMSJDL, "Inconsistent parametric job description")
+                        parameterDict[seqID]["ParameterList"] = parList
+                    else:
+                        if attribute != "Parameters":
+                            return S_ERROR(EWMSJDL, "Inconsistent parametric job description")
+                        nPar = jobClassAd.getAttributeInt(attribute)
+                        if nPar is None:
+                            value = jobClassAd.get_expression(attribute)
+                            return S_ERROR(EWMSJDL, f"Inconsistent parametric job description: {attribute}={value}")
+                        parameterDict[seqID]["Parameters"] = nPar
+                else:
+                    value = jobClassAd.getAttributeInt(attribute)
+                    if value is None:
+                        value = jobClassAd.getAttributeFloat(attribute)
+                        if value is None:
+                            value = jobClassAd.get_expression(attribute)
+                            return S_ERROR(f"Illegal value for {attribute} JDL field: {value}")
+                    parameterDict[seqID][key] = value
+
+    if "0" in parameterDict and not parameterDict.get("0"):
+        parameterDict.pop("0")
+
+    parameterLists = {}
+    for seqID in parameterDict:
+        parList = __getParameterSequence(
+            nParValues,
+            parList=parameterDict[seqID].get("ParameterList", []),
+            parStart=parameterDict[seqID].get("ParameterStart", 1),
+            parStep=parameterDict[seqID].get("ParameterStep", 0),
+            parFactor=parameterDict[seqID].get("ParameterFactor", 1),
+        )
+        if not parList:
+            return S_ERROR(EWMSJDL, "Inconsistent parametric job description")
+
+        parameterLists[seqID] = parList
+
+    jobDescList = []
+    jobDesc = jobClassAd.asJDL()
+    # Width of the sequential parameter number
+    zLength = len(str(nParValues - 1))
+    for n in range(nParValues):
+        newJobDesc = jobDesc
+        newJobDesc = newJobDesc.replace("%n", str(n).zfill(zLength))
+        newClassAd = ClassAd(newJobDesc)
+        for seqID in parameterLists:
+            parameter = parameterLists[seqID][n]
+            for attribute in newClassAd.getAttributes():
+                __updateAttribute(newClassAd, attribute, seqID, str(parameter))
+
+        for seqID in parameterLists:
+            for attribute in ["Parameters", "ParameterStart", "ParameterStep", "ParameterFactor"]:
+                if seqID == "0":
+                    newClassAd.deleteAttribute(attribute)
+                else:
+                    newClassAd.deleteAttribute(f"{attribute}.{seqID}")
+
+            parameter = parameterLists[seqID][n]
+            if seqID == "0":
+                attribute = "Parameter"
+            else:
+                attribute = f"Parameter.{seqID}"
+            if isinstance(parameter, str) and parameter.startswith("{"):
+                newClassAd.insertAttributeInt(attribute, str(parameter))
+            else:
+                newClassAd.insertAttributeString(attribute, str(parameter))
+
+        newClassAd.insertAttributeInt("ParameterNumber", n)
+        newJDL = newClassAd.asJDL()
+        jobDescList.append(newJDL)
+
+    return S_OK(jobDescList)

DIRACCommon/WorkloadManagementSystem/Utilities/__init__.py

@@ -0,0 +1 @@
+"""DIRACCommon WorkloadManagementSystem utilities"""

DIRACCommon/WorkloadManagementSystem/__init__.py

@@ -0,0 +1 @@
+"""DIRACCommon WorkloadManagementSystem utilities"""

DIRACCommon/__init__.py

@@ -0,0 +1,21 @@
+"""
+DIRACCommon - Stateless utilities for DIRAC
+
+This package contains stateless utilities extracted from DIRAC that can be used
+by DiracX and other projects without triggering DIRAC's global state initialization.
+
+The utilities here should not depend on:
+- gConfig (Configuration system)
+- gLogger (Global logging)
+- gMonitor (Monitoring)
+- Database connections
+- Any other global state
+"""
+
+import importlib.metadata
+
+try:
+    __version__ = importlib.metadata.version(__name__)
+except importlib.metadata.PackageNotFoundError:
+    # package is not installed
+    __version__ = "Unknown"

diraccommon-9.0.0.dist-info/METADATA

@@ -0,0 +1,281 @@
+Metadata-Version: 2.4
+Name: DIRACCommon
+Version: 9.0.0
+Summary: Stateless utilities extracted from DIRAC for use by DiracX and other projects
+Project-URL: Homepage, https://github.com/DIRACGrid/DIRAC
+Project-URL: Documentation, https://dirac.readthedocs.io/
+Project-URL: Source Code, https://github.com/DIRACGrid/DIRAC
+Author-email: DIRAC Collaboration <dirac-dev@cern.ch>
+License: GPL-3.0-only
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Scientific/Engineering
+Classifier: Topic :: System :: Distributed Computing
+Requires-Python: >=3.11
+Requires-Dist: diraccfg
+Requires-Dist: pydantic>=2.0.0
+Requires-Dist: typing-extensions>=4.0.0
+Provides-Extra: testing
+Requires-Dist: pytest-cov>=4.0.0; extra == 'testing'
+Requires-Dist: pytest>=7.0.0; extra == 'testing'
+Description-Content-Type: text/markdown
+
+# DIRACCommon
+
+Stateless utilities extracted from DIRAC for use by DiracX and other projects without triggering DIRAC's global state initialization.
+
+## Purpose
+
+This package solves the circular dependency issue where DiracX needs DIRAC utilities but importing DIRAC triggers global state initialization. DIRACCommon contains only stateless utilities that can be safely imported without side effects.
+
+## Contents
+
+- `DIRACCommon.Core.Utilities.ReturnValues`: DIRAC's S_OK/S_ERROR return value system
+- `DIRACCommon.Core.Utilities.DErrno`: DIRAC error codes and utilities
+- `DIRACCommon.Core.Utilities.ClassAd.ClassAdLight`: JDL parsing utilities
+- `DIRACCommon.Core.Utilities.TimeUtilities`: Time and date utilities
+- `DIRACCommon.Core.Utilities.StateMachine`: State machine utilities
+- `DIRACCommon.Core.Utilities.JDL`: JDL parsing utilities
+- `DIRACCommon.Core.Utilities.List`: List manipulation utilities
+- `DIRACCommon.ConfigurationSystem.Client.Helpers.Resources`: Platform compatibility utilities
+- `DIRACCommon.WorkloadManagementSystem.Client.JobStatus`: Job status constants and state machines
+- `DIRACCommon.WorkloadManagementSystem.Client.JobState.JobManifest`: Job manifest utilities
+- `DIRACCommon.WorkloadManagementSystem.DB.JobDBUtils`: Job database utilities
+- `DIRACCommon.WorkloadManagementSystem.Utilities.JobModel`: Pydantic-based job models
+- `DIRACCommon.WorkloadManagementSystem.Utilities.JobStatusUtility`: Job status utilities
+- `DIRACCommon.WorkloadManagementSystem.Utilities.ParametricJob`: Parametric job utilities
+
+## Installation
+
+```bash
+pip install DIRACCommon
+```
+
+## Usage
+
+### Basic Usage
+
+```python
+from DIRACCommon.Core.Utilities.ReturnValues import S_OK, S_ERROR
+
+def my_function():
+    if success:
+        return S_OK("Operation successful")
+    else:
+        return S_ERROR("Operation failed")
+```
+
+## Development
+
+This package is part of the DIRAC project and shares its version number. When DIRAC is released, DIRACCommon is also released with the same version.
+
+```bash
+pixi install
+pixi run pytest
+```
+
+## Migrating Code to DIRACCommon
+
+This section documents the proper pattern for moving code from DIRAC to DIRACCommon to enable shared usage by DiracX and other projects.
+
+### Migration Pattern
+
+The migration follows a specific pattern to maintain backward compatibility while making code stateless:
+
+1. **Move core functionality to DIRACCommon** - Create the stateless version
+2. **Update DIRAC module** - Make it a backward compatibility wrapper
+3. **Move and update tests** - Ensure both versions are tested
+4. **Verify migration** - Test both import paths work correctly
+
+### Step-by-Step Migration Process
+
+#### 1. Create DIRACCommon Module
+
+Create the new module in DIRACCommon with the **exact same directory structure** as DIRAC:
+
+```bash
+# Example: Moving from src/DIRAC/ConfigurationSystem/Client/Helpers/Resources.py
+# Create: dirac-common/src/DIRACCommon/ConfigurationSystem/Client/Helpers/Resources.py
+```
+
+#### 2. Make Code Stateless
+
+**❌ Remove these dependencies:**
+```python
+# DON'T import these in DIRACCommon
+from DIRAC import gConfig, gLogger, gMonitor, Operations
+from DIRAC.Core.Security import getProxyInfo
+# Any other DIRAC global state
+```
+
+**✅ Use these instead:**
+```python
+# Use DIRACCommon's own utilities
+from DIRACCommon.Core.Utilities.ReturnValues import S_OK, S_ERROR
+from DIRACCommon.Core.Utilities.DErrno import strerror
+
+# Accept configuration data as parameters
+def my_function(data, config_dict):
+    # Use config_dict instead of gConfig.getOptionsDict()
+    pass
+```
+
+#### 3. Handle Configuration Data
+
+**❌ Don't do this:**
+```python
+# DIRACCommon function taking config object
+def getDIRACPlatform(OSList, config):
+    result = config.getOptionsDict("/Resources/Computing/OSCompatibility")
+    # ...
+```
+
+**✅ Do this instead:**
+```python
+# DIRACCommon function taking configuration data directly
+def getDIRACPlatform(osList: str | list[str], osCompatibilityDict: dict[str, set[str]]) -> DReturnType[list[str]]:
+    if not osCompatibilityDict:
+        return S_ERROR("OS compatibility info not found")
+    # Use osCompatibilityDict directly
+    # ...
+```
+
+#### 4. Update DIRAC Module for Backward Compatibility
+
+Transform the original DIRAC module into a backward compatibility wrapper:
+
+```python
+"""Backward compatibility wrapper - moved to DIRACCommon
+
+This module has been moved to DIRACCommon.ConfigurationSystem.Client.Helpers.Resources
+to avoid circular dependencies and allow DiracX to use these utilities without
+triggering DIRAC's global state initialization.
+
+All exports are maintained for backward compatibility.
+"""
+
+# Re-export everything from DIRACCommon for backward compatibility
+from DIRACCommon.ConfigurationSystem.Client.Helpers.Resources import (
+    getDIRACPlatform as _getDIRACPlatform,
+    _platformSortKey,
+)
+
+from DIRAC import S_ERROR, S_OK, gConfig
+
+def getDIRACPlatform(OSList):
+    """Get standard DIRAC platform(s) compatible with the argument.
+
+    Backward compatibility wrapper that uses gConfig.
+    """
+    result = gConfig.getOptionsDict("/Resources/Computing/OSCompatibility")
+    if not (result["OK"] and result["Value"]):
+        return S_ERROR("OS compatibility info not found")
+
+    # Convert string values to sets for DIRACCommon function
+    platformsDict = {k: set(v.replace(" ", "").split(",")) for k, v in result["Value"].items()}
+    for k, v in platformsDict.items():
+        if k not in v:
+            v.add(k)
+
+    return _getDIRACPlatform(OSList, platformsDict)
+
+# Re-export the helper function
+_platformSortKey = _platformSortKey
+```
+
+#### 5. Move and Update Tests
+
+**Create DIRACCommon tests:**
+```python
+# dirac-common/tests/ConfigurationSystem/Client/Helpers/test_Resources.py
+from DIRACCommon.ConfigurationSystem.Client.Helpers.Resources import getDIRACPlatform
+
+def test_getDIRACPlatform():
+    # Test with configuration data directly
+    osCompatibilityDict = {
+        "plat1": {"OS1", "OS2"},
+        "plat2": {"OS3", "OS4"}
+    }
+    result = getDIRACPlatform("OS1", osCompatibilityDict)
+    assert result["OK"]
+    assert "plat1" in result["Value"]
+```
+
+**Update DIRAC tests:**
+```python
+# src/DIRAC/ConfigurationSystem/Client/Helpers/test/Test_Helpers.py
+from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getDIRACPlatform
+
+def test_getDIRACPlatform():
+    # Test backward compatibility wrapper
+    # (existing tests should continue to work)
+    pass
+```
+
+#### 6. Create Directory Structure
+
+Ensure the DIRACCommon directory structure mirrors DIRAC exactly:
+
+```bash
+dirac-common/src/DIRACCommon/
+├── ConfigurationSystem/
+│   ├── __init__.py
+│   └── Client/
+│       ├── __init__.py
+│       └── Helpers/
+│           ├── __init__.py
+│           └── Resources.py
+└── tests/
+    └── ConfigurationSystem/
+        ├── __init__.py
+        └── Client/
+            ├── __init__.py
+            └── Helpers/
+                ├── __init__.py
+                └── test_Resources.py
+```
+
+### Requirements for DIRACCommon Code
+
+Code in DIRACCommon **MUST** be:
+
+- **Completely stateless** - No global variables or state
+- **No DIRAC dependencies** - Cannot import from DIRAC
+- **No global state access** - Cannot use `gConfig`, `gLogger`, `gMonitor`, etc.
+- **No database connections** - Cannot establish DB connections
+- **No side effects on import** - Importing should not trigger any initialization
+- **Pure functions** - Functions should be deterministic and side-effect free
+
+### Configuration Data Handling
+
+When DIRACCommon functions need configuration data:
+
+1. **Accept data as parameters** - Don't accept config objects
+2. **Use specific data types** - Pass dictionaries, not config objects
+3. **Let DIRAC wrapper handle gConfig** - DIRAC gets data and passes it to DIRACCommon
+
+### Example Migration
+
+See the migration of `getDIRACPlatform` in:
+- `dirac-common/src/DIRACCommon/ConfigurationSystem/Client/Helpers/Resources.py`
+- `src/DIRAC/ConfigurationSystem/Client/Helpers/Resources.py`
+
+This demonstrates the complete pattern from stateless DIRACCommon implementation to backward-compatible DIRAC wrapper.
+
+### Testing the Migration
+
+After migration, verify:
+
+1. **DIRACCommon tests pass** - `pixi run python -m pytest dirac-common/tests/`
+2. **DIRAC tests pass** - `pixi run python -m pytest src/DIRAC/`
+3. **Both import paths work**:
+   ```python
+   # DIRACCommon (stateless)
+   from DIRACCommon.ConfigurationSystem.Client.Helpers.Resources import getDIRACPlatform
+
+   # DIRAC (backward compatibility)
+   from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getDIRACPlatform
+   ```
+4. **No linting errors** - All code should pass linting checks

diraccommon-9.0.0.dist-info/RECORD

@@ -0,0 +1,29 @@
+DIRACCommon/__init__.py,sha256=o31FFvmIiU2Wsc9nfsPZVaXYK9d3KTN0vOa8mZS6x5E,587
+DIRACCommon/ConfigurationSystem/__init__.py,sha256=zD7hqGhEaKikUOFQ9fYd-T1nGE3A7hBQnSbZMNdde_Q,74
+DIRACCommon/ConfigurationSystem/Client/__init__.py,sha256=wc_AKSzmjk_56P_CMkihYaEX4RUiED3XZCLcwZ9NPoY,88
+DIRACCommon/ConfigurationSystem/Client/Helpers/Resources.py,sha256=N8gRrfv3Ai-tzloNTvCb3jOMHj7PkMzKiWd-MmWu2vA,1904
+DIRACCommon/ConfigurationSystem/Client/Helpers/__init__.py,sha256=gCIB_25s-6H77hwho9ex_nUnMX2vvzg1Rr33u1-mLkI,95
+DIRACCommon/Core/__init__.py,sha256=Kv_Rkbm9hOPmHSnTG0_i7QR5Tck8bldITONhsoT2syg,33
+DIRACCommon/Core/Utilities/DErrno.py,sha256=LUaIcKkjuioTMO3O7H9T9Pk5pHUS63Nzx3FcaP40n5c,7998
+DIRACCommon/Core/Utilities/JDL.py,sha256=z4RID730tbCmbgldKfhSFc9ADfXDhUfjaBG8RpuK2Bs,6724
+DIRACCommon/Core/Utilities/List.py,sha256=AahQrlQqfnh8tyA1GjGqYJfo_3v5N77IQreA8VCNXks,3758
+DIRACCommon/Core/Utilities/ReturnValues.py,sha256=YDZEZrYLtI_Aa8qTnHX633F1cRupQ_UTAnI7aSlf0sI,8733
+DIRACCommon/Core/Utilities/StateMachine.py,sha256=5Pwpc2wmCgb6PwyUH2qe9HZwBMyzkhFTBA7cixso1Sg,6767
+DIRACCommon/Core/Utilities/TimeUtilities.py,sha256=wXuYIwECbLSFdVoNM0iSYoCrz6_iL-JrMV1yVv3FloA,9026
+DIRACCommon/Core/Utilities/__init__.py,sha256=hwOqqYwQDINF-D3eKKBC1r-B_jEnQ5ZSdJvc7rIv_fY,56
+DIRACCommon/Core/Utilities/ClassAd/ClassAdLight.py,sha256=7450ECzxnJQpIjzbaE-uFp3ISE55iL6YM-TRqyebZTY,9597
+DIRACCommon/Core/Utilities/ClassAd/__init__.py,sha256=z0yRe-wRxIHE-Sp3tnyC37S6z38dZHCxuqf3ZV1OyzQ,40
+DIRACCommon/WorkloadManagementSystem/__init__.py,sha256=XMOUD0OcPVo-mjuA0vU-mbtf9K0kBMy5I5sELrX3HEs,53
+DIRACCommon/WorkloadManagementSystem/Client/JobStatus.py,sha256=lBfPYa_RRjJlWEU5eqrbWFaSl1081CqSDjhI6B7-qeA,2813
+DIRACCommon/WorkloadManagementSystem/Client/__init__.py,sha256=ssMppR0P3iXlUaqWPjl1Y5AlQ6LgUAsRa9-5lVkBsL0,60
+DIRACCommon/WorkloadManagementSystem/Client/JobState/JobManifest.py,sha256=E3DVRzAHUFAFa9baqEbAMr-4cu6tTXuVRVdl7XgUX7I,7380
+DIRACCommon/WorkloadManagementSystem/Client/JobState/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+DIRACCommon/WorkloadManagementSystem/DB/JobDBUtils.py,sha256=YU97uk0fjwYN6O8nkD6XjnZ7k9ovhSLYjytqI3oLRfU,5777
+DIRACCommon/WorkloadManagementSystem/DB/__init__.py,sha256=CXWy4bdKeG6ml9T5r8-E7EyfDBNE79I_bs13qYI7yY8,56
+DIRACCommon/WorkloadManagementSystem/Utilities/JobModel.py,sha256=kpvsxQnR-j9Y9I38qUWLD7coFRssLwtJCtSNAbjFh1Y,8973
+DIRACCommon/WorkloadManagementSystem/Utilities/JobStatusUtility.py,sha256=Kf4o1bp25JLsRb-Vsc4NuM6w1Io2hg_ALtIyqBk-Qhw,3628
+DIRACCommon/WorkloadManagementSystem/Utilities/ParametricJob.py,sha256=iYd29TgGXZk31pQLPw_frIhcmZB9X-RmuDmIRxzgiww,7405
+DIRACCommon/WorkloadManagementSystem/Utilities/__init__.py,sha256=XMOUD0OcPVo-mjuA0vU-mbtf9K0kBMy5I5sELrX3HEs,53
+diraccommon-9.0.0.dist-info/METADATA,sha256=diT9PkasG5d0zhaEJFMDKtVS5XsPDBV_XtJfQVKgkS8,9735
+diraccommon-9.0.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+diraccommon-9.0.0.dist-info/RECORD,,

diraccommon-9.0.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.27.0
+Root-Is-Purelib: true
+Tag: py3-none-any