DIRAC 9.0.11__tar.gz → 9.0.13__tar.gz

{dirac-9.0.11/src/DIRAC.egg-info → dirac-9.0.13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: DIRAC
-Version: 9.0.11
+Version: 9.0.13
 Summary: DIRAC is an interware, meaning a software framework for distributed computing.
 Home-page: https://github.com/DIRACGrid/DIRAC/
 License: GPL-3.0-only
@@ -19,7 +19,7 @@ Requires-Dist: cachetools
 Requires-Dist: certifi
 Requires-Dist: cwltool
 Requires-Dist: diraccfg
-Requires-Dist: DIRACCommon==v9.0.11
+Requires-Dist: DIRACCommon==v9.0.13
 Requires-Dist: diracx-client>=v0.0.1
 Requires-Dist: diracx-core>=v0.0.1
 Requires-Dist: diracx-cli>=v0.0.1
@@ -47,6 +47,7 @@ Requires-Dist: Authlib>=1.0.0.a2
 Requires-Dist: pyjwt
 Requires-Dist: dominate
 Requires-Dist: zstandard
+Requires-Dist: xattr
 Provides-Extra: server
 Requires-Dist: CMRESHandler; extra == "server"
 Requires-Dist: opensearch-py; extra == "server"

{dirac-9.0.11 → dirac-9.0.13}/release.notes

@@ -1,3 +1,27 @@
+[v9.0.13]
+
+*DataManagementSystem
+
+FIX: (#8396) submit unmanaged token for FTS3 replication
+
+*Resources
+
+NEW: (#8382) Attempt to set the xrd checksum attribute when copying files localy
+
+[v9.0.12]
+
+*Resources
+
+FIX: (#8393) make client_id and secret optional to avoid KeyError exceptions.
+
+*ConfigurationSystem
+
+NEW: (#8392) handle adding user AffiliationEnds section in the CS when modifying the user from CSAPI
+
+*MonitoringSystem
+
+FIX: (#8391) Addapt webApp to new getTransformations API
+
 [v9.0.11]
 
 *WorkloadManagement

{dirac-9.0.11 → dirac-9.0.13}/setup.cfg

@@ -26,7 +26,7 @@ install_requires =
 	certifi
 	cwltool
 	diraccfg
-	DIRACCommon==v9.0.11
+	DIRACCommon==v9.0.13
 	diracx-client >=v0.0.1
 	diracx-core >=v0.0.1
 	diracx-cli >=v0.0.1
@@ -54,6 +54,7 @@ install_requires =
 	pyjwt
 	dominate
 	zstandard
+	xattr
 zip_safe = False
 include_package_data = True
 

{dirac-9.0.11 → dirac-9.0.13}/src/DIRAC/AccountingSystem/Client/Types/Network.py

@@ -1,6 +1,6 @@
-""" Accounting class to stores network metrics gathered by perfSONARs.
+"""Accounting class to stores network metrics gathered by perfSONARs.
 
-    Filled by "Accounting/Network" agent
+Filled by "Accounting/Network" agent
 """
 
 from DIRAC.AccountingSystem.Client.Types.BaseAccountingType import BaseAccountingType
@@ -16,12 +16,12 @@ class Network(BaseAccountingType):
 
         # IPv6 address has up to 45 chars
         self.definitionKeyFields = [
-            ("SourceIP", "VARCHAR(50)"),
-            ("DestinationIP", "VARCHAR(50)"),
-            ("SourceHostName", "VARCHAR(50)"),
-            ("DestinationHostName", "VARCHAR(50)"),
-            ("Source", "VARCHAR(50)"),
-            ("Destination", "VARCHAR(50)"),
+            ("SourceIP", "VARCHAR(64)"),
+            ("DestinationIP", "VARCHAR(64)"),
+            ("SourceHostName", "VARCHAR(255)"),
+            ("DestinationHostName", "VARCHAR(255)"),
+            ("Source", "VARCHAR(255)"),
+            ("Destination", "VARCHAR(255)"),
         ]
 
         self.definitionAccountingFields = [

{dirac-9.0.11 → dirac-9.0.13}/src/DIRAC/AccountingSystem/Client/Types/PilotSubmission.py

@@ -1,6 +1,6 @@
-""" Accounting Type for Pilot Submission
+"""Accounting Type for Pilot Submission
 
-    Filled by the "WorkloadManagement/SiteDirector" agent(s)
+Filled by the "WorkloadManagement/SiteDirector" agent(s)
 """
 
 from DIRAC.AccountingSystem.Client.Types.BaseAccountingType import BaseAccountingType
@@ -13,7 +13,7 @@ class PilotSubmission(BaseAccountingType):
         super().__init__()
 
         self.definitionKeyFields = [
-            ("HostName", "VARCHAR(100)"),
+            ("HostName", "VARCHAR(255)"),
             ("SiteDirector", "VARCHAR(100)"),
             ("Site", "VARCHAR(100)"),
             ("CE", "VARCHAR(100)"),

{dirac-9.0.11 → dirac-9.0.13}/src/DIRAC/ConfigurationSystem/Client/CSAPI.py

@@ -456,13 +456,23 @@ class CSAPI:
             gLogger.error("User is not registered: ", repr(username))
             return S_OK(False)
         for prop in properties:
-            if prop == "Groups":
+            if prop in ["Groups", "AffiliationEnds"]:
                 continue
             prevVal = self.__csMod.getValue(f"{self.__baseSecurity}/Users/{username}/{prop}")
             if not prevVal or prevVal != properties[prop]:
                 gLogger.info(f"Setting {prop} property for user {username} to {properties[prop]}")
                 self.__csMod.setOptionValue(f"{self.__baseSecurity}/Users/{username}/{prop}", properties[prop])
                 modifiedUser = True
+        if properties.get("AffiliationEnds", None):
+            user_affiliationends_section = f"{self.__baseSecurity}/Users/{username}/AffiliationEnds"
+            # add the section for AffiliationEnds
+            res = gConfig.getSections(user_affiliationends_section)
+            if not res["OK"]:
+                self.__csMod.createSection(user_affiliationends_section)
+            # now set value VO = end date
+            for vo, end_date in properties["AffiliationEnds"].items():
+                self.__csMod.setOptionValue(f"{user_affiliationends_section}/{vo}", end_date)
+                modifiedUser = True
         if "Groups" in properties:
             result = self.listGroups()
             if not result["OK"]:

{dirac-9.0.11 → dirac-9.0.13}/src/DIRAC/ConfigurationSystem/ConfigTemplate.cfg

@@ -6,6 +6,9 @@ Services
   {
     HandlerPath = DIRAC/ConfigurationSystem/Service/TornadoConfigurationHandler.py
     Port = 9135
+    # Set to True to make a DiracX model validation on commits
+    # If validation fails, commit will also failed
+    VerifyDiracXSyncOnCommit = False
     # Subsection to configure authorization over the service
     Authorization
     {

{dirac-9.0.11 → dirac-9.0.13}/src/DIRAC/ConfigurationSystem/private/Modificator.py

@@ -1,13 +1,16 @@
 """ This is the guy that actually modifies the content of the CS
 """
-import zlib
-import difflib
 import datetime
+import difflib
+import zlib
 
 from diraccfg import CFG
-from DIRAC.Core.Utilities import List
+
+from DIRAC import S_ERROR
 from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData
 from DIRAC.Core.Security.ProxyInfo import getProxyInfo
+from DIRAC.Core.Utilities import List
+from DIRAC.FrameworkSystem.Utilities.diracx import diracxVerifyConfig
 
 
 class Modificator:
@@ -239,6 +242,11 @@ class Modificator:
         return str(self.cfgData)
 
     def commit(self):
+        retOpt = self.getValue("/Systems/Configuration/Services/Server/VerifyDiracXSyncOnCommit")
+        if retOpt == "True":
+            resVerif = diracxVerifyConfig(self.cfgData)
+            if not resVerif["OK"]:
+                return S_ERROR(resVerif["Message"])
         compressedData = zlib.compress(str(self.cfgData).encode(), 9)
         return self.rpcClient.commitNewData(compressedData)
 

{dirac-9.0.11 → dirac-9.0.13}/src/DIRAC/DataManagementSystem/Client/FTS3Job.py

@@ -660,6 +660,7 @@ class FTS3Job(JSerializable):
             multihop=isMultiHop,
             metadata=job_metadata,
             priority=self.priority,
+            unmanaged_tokens=True,
             archive_timeout=archive_timeout,
             **dest_spacetoken,
         )

{dirac-9.0.11 → dirac-9.0.13}/src/DIRAC/FrameworkSystem/DB/InstalledComponentsDB.py

@@ -1,6 +1,7 @@
 """
 Classes and functions for easier management of the InstalledComponents database
 """
+
 import re
 import datetime
 
@@ -90,7 +91,7 @@ class Host(componentsBase):
     __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8mb4"}
 
     hostID = Column("HostID", Integer, primary_key=True)
-    hostName = Column("HostName", String(64), nullable=False)
+    hostName = Column("HostName", String(255), nullable=False)
     cpu = Column("CPU", String(64), nullable=False)
     installationList = relationship("InstalledComponent", backref="installationHost")
 
@@ -219,7 +220,7 @@ class HostLogging(componentsBase):
     __tablename__ = "HostLogging"
     __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8mb4"}
 
-    hostName = Column("HostName", String(32), nullable=False, primary_key=True)
+    hostName = Column("HostName", String(255), nullable=False, primary_key=True)
     # status
     DIRAC = Column("DIRACVersion", String(64))
     Extension = Column("Extension", String(512))

{dirac-9.0.11 → dirac-9.0.13}/src/DIRAC/FrameworkSystem/Utilities/MonitoringUtilities.py

@@ -1,6 +1,7 @@
 """
 Utilities for ComponentMonitoring features
 """
+
 import datetime
 import socket
 
@@ -30,7 +31,6 @@ def monitorInstallation(componentType, system, component, module=None, cpu=None,
 
     if not hostname:
         hostname = socket.getfqdn()
-        hostname = hostname[0:64]
     instance = component[0:32]
 
     result = monitoringClient.installationExists(

{dirac-9.0.11 → dirac-9.0.13}/src/DIRAC/FrameworkSystem/Utilities/diracx.py

@@ -1,20 +1,25 @@
-import requests
-
-from cachetools import TTLCache, LRUCache, cached
-from cachetools.keys import hashkey
+import os
+import re
+import subprocess
+from collections.abc import Generator
+from contextlib import contextmanager
 from pathlib import Path
 from tempfile import NamedTemporaryFile
+import tempfile
 from typing import Any
-from collections.abc import Generator
-from DIRAC import gConfig
-from DIRAC.ConfigurationSystem.Client.Helpers import Registry
-from contextlib import contextmanager
 
+import requests
+from cachetools import LRUCache, TTLCache, cached
+from cachetools.keys import hashkey
+from diracx.cli.internal.legacy import _apply_fixes
+from diracx.core.config.schema import Config as DiracxConfig
+from diracx.core.models import TokenResponse
 from diracx.core.preferences import DiracxPreferences
-
 from diracx.core.utils import write_credentials
+from pydantic import ValidationError
 
-from diracx.core.models import TokenResponse
+from DIRAC import S_ERROR, S_OK, gConfig
+from DIRAC.ConfigurationSystem.Client.Helpers import Registry
 
 try:
     from diracx.client.sync import SyncDiracClient
@@ -104,3 +109,29 @@ def TheImpersonator(credDict: dict[str, Any], *, source: str = "") -> Generator[
         client.__enter__()
         diracx_client_cache[token_location] = client
     yield client
+
+
+def diracxVerifyConfig(cfgData):
+    """Verify CS config using DiracX config validation
+
+    Args:
+        cfgData: CFG data
+
+    Returns:
+        S_OK | S_ERROR: Value: diracx Config validation
+    """
+    os.environ["DIRAC_COMPAT_ENABLE_CS_CONVERSION"] = "true"
+    with tempfile.NamedTemporaryFile() as temp_cfg:
+        with tempfile.NamedTemporaryFile() as temp_diracx_cfg:
+            cfgData.writeToFile(temp_cfg.name)
+            cmd = ["dirac", "internal", "legacy", "cs-sync", temp_cfg.name, temp_diracx_cfg.name]
+            res = subprocess.run(cmd, capture_output=True, text=True, timeout=15)
+    os.environ.pop("DIRAC_COMPAT_ENABLE_CS_CONVERSION")
+    if res.returncode == 0:
+        return S_OK(res.stdout)
+    else:
+        err = res.stderr.strip()
+        match = re.search(r"(ValidationError:.*)", err, flags=re.DOTALL)
+        if match:
+            return S_ERROR(match.group(1))
+        return S_ERROR(err)

{dirac-9.0.11 → dirac-9.0.13}/src/DIRAC/FrameworkSystem/scripts/dirac_uninstall_component.py

@@ -2,6 +2,7 @@
 """
 Uninstallation of a DIRAC component
 """
+
 import socket
 
 from DIRAC import exit as DIRACexit

{dirac-9.0.11 → dirac-9.0.13}/src/DIRAC/MonitoringSystem/Service/WebAppHandler.py

@@ -2,6 +2,7 @@
 The WebAppHandler module provides a class to handle web requests from the DIRAC WebApp.
 It is not indented to be used in diracx
 """
+
 from DIRAC import S_ERROR, S_OK
 from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
 from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getSites
@@ -527,13 +528,21 @@ class WebAppHandler(RequestHandler):
         ops = Operations()
         # Prepare the standard structure now within the resultDict dictionary
         resultDict = {}
-        trList = res["Records"]
+        # Reconstruct just the values list
+        trList = [
+            [str(item) if not isinstance(item, int) else item for item in trans_dict.values()]
+            for trans_dict in res["Value"]
+        ]
+
         # Create the total records entry
         nTrans = len(trList)
         resultDict["TotalRecords"] = nTrans
         # Create the ParameterNames entry
-        # As this list is a reference to the list in the DB, we cannot extend it, therefore copy it
-        resultDict["ParameterNames"] = list(res["ParameterNames"])
+        try:
+            resultDict["ParameterNames"] = list(res["Value"][0].keys())
+        except IndexError:
+            # As this list is a reference to the list in the DB, we cannot extend it, therefore copy it
+            resultDict["ParameterNames"] = list(cls.transformationDB.TRANSPARAMS)
         # Add the job states to the ParameterNames entry
         taskStateNames = TASKS_STATE_NAMES + ops.getValue("Transformations/AdditionalTaskStates", [])
         resultDict["ParameterNames"] += ["Jobs_" + x for x in taskStateNames]

{dirac-9.0.11 → dirac-9.0.13}/src/DIRAC/Resources/IdProvider/IdProviderFactory.py

@@ -75,8 +75,10 @@ class IdProviderFactory:
 
             if client_name_prefix:
                 client_name_prefix = client_name_prefix + "_"
-            pDict["client_id"] = pDict[f"{client_name_prefix}client_id"]
-            pDict["client_secret"] = pDict[f"{client_name_prefix}client_secret"]
+            if f"{client_name_prefix}client_id" in pDict:
+                pDict["client_id"] = pDict[f"{client_name_prefix}client_id"]
+            if f"{client_name_prefix}client_secret" in pDict:
+                pDict["client_secret"] = pDict[f"{client_name_prefix}client_secret"]
 
         pDict.update(kwargs)
         pDict["ProviderName"] = name

{dirac-9.0.11 → dirac-9.0.13}/src/DIRAC/Resources/IdProvider/tests/Test_IdProviderFactory.py

@@ -63,6 +63,12 @@ Resources
       client_secret = IdP_client_secret
       scope = openid+profile+offline_access+eduperson_entitlement
     }
+    SomeIdP3.partial
+    {
+      ProviderType = OAuth2
+      issuer = https://and-another-idp.url/
+      scope = openid+profile+offline_access+eduperson_entitlement
+    }
   }
 }
 """
@@ -77,6 +83,7 @@ idps = IdProviderFactory()
         ("SomeIdP1.2", {"OK": True}, "https://idp.url/", "IdP_client_id2", "IdP_client_secret"),
         ("SomeIdP2", {"OK": True}, "https://another-idp.url/", "IdP_client_id1", "IdP_client_secret"),
         ("SomeIdP3", {"OK": True}, "https://and-another-idp.url/", "IdP_client_id3", "IdP_client_secret"),
+        ("SomeIdP3.partial", {"OK": True}, "https://and-another-idp.url/", None, None),
         # Try to get an unknown DIRAC client
         ("DIRACUnknown", {"OK": False, "Message": "DIRACUnknown does not exist"}, None, None, None),
     ],

{dirac-9.0.11 → dirac-9.0.13}/src/DIRAC/Resources/Storage/FileStorage.py

@@ -1,10 +1,13 @@
 """
- This is the File StorageClass, only meant to be used localy
- """
+This is the File StorageClass, only meant to be used localy
+"""
+
 import os
 import shutil
 import errno
 import stat
+import struct
+import time
 
 from DIRAC import gLogger, S_OK, S_ERROR
 from DIRAC.Resources.Storage.Utilities import checkArgumentFormat
@@ -12,6 +15,116 @@ from DIRAC.Resources.Storage.StorageBase import StorageBase
 from DIRAC.Core.Utilities.Adler import fileAdler
 
 
+def set_xattr_adler32(path, checksum):
+    """
+    Set the adler32 checksum extended attribute on a file.
+
+    This is needed for case where you write the data on a locally mounted
+    file system, but then want to access it from outside via xroot (like the HLT farm)
+
+    Hopefully, this whole function will be part of xroot at some point
+    https://github.com/xrootd/xrootd/pull/2650
+
+
+    This function replicates the exact behavior of the C++ function fSetXattrAdler32
+
+    It writes the checksum in XrdCksData binary format with the following structure:
+        - Name[16]:    Algorithm name ("adler32"), null-padded
+        - fmTime (8):  File modification time (network byte order, int64)
+        - csTime (4):  Time delta from mtime (network byte order, int32)
+        - Rsvd1 (2):   Reserved (int16)
+        - Rsvd2 (1):   Reserved (uint8)
+        - Length (1):  Checksum length in bytes (uint8)
+        - Value[64]:   Binary checksum value (4 bytes for adler32)
+
+    Total structure size: 96 bytes
+
+    Parameters
+    ----------
+    path : str
+        Path to the file (must be a regular file on local filesystem)
+    checksum : str
+        8-character hexadecimal adler32 checksum (e.g., "deadbeef")
+
+
+    Notes
+    -----
+    - The attribute is stored as "user.XrdCks.adler32"
+
+    """
+    # Validate checksum format
+    if not isinstance(checksum, str) or len(checksum) != 8:
+        raise ValueError("Checksum must be exactly 8 characters")
+
+    # Validate it's valid hex
+    try:
+        int(checksum, 16)
+    except ValueError:
+        raise ValueError(f"Checksum must be valid hexadecimal: {checksum}")
+
+    # Check file exists and is regular
+
+    st = os.stat(path)
+
+    # Import xattr module
+    try:
+        import xattr
+    except ImportError:
+        raise ImportError("The 'xattr' module is required. Install it with: pip install xattr")
+
+    # Build XrdCksData structure (96 bytes total)
+    # Reference: src/XrdCks/XrdCksData.hh
+
+    # 1. Name[16] - Algorithm name, null-padded
+    name = b"adler32"
+    name_field = name.ljust(16, b"\x00")
+
+    # 2. fmTime (8 bytes) - File modification time (network byte order = big-endian)
+    fm_time = int(st.st_mtime)
+    fm_time_field = struct.pack(">q", fm_time)  # signed 64-bit big-endian
+
+    # 3. csTime (4 bytes) - Delta from mtime to now (network byte order)
+    cs_time = int(time.time()) - fm_time
+    cs_time_field = struct.pack(">i", cs_time)  # signed 32-bit big-endian
+
+    # 4. Rsvd1 (2 bytes) - Reserved, set to 0
+    rsvd1_field = struct.pack(">h", 0)  # signed 16-bit big-endian
+
+    # 5. Rsvd2 (1 byte) - Reserved, set to 0
+    rsvd2_field = struct.pack("B", 0)  # unsigned 8-bit
+
+    # 6. Length (1 byte) - Checksum length in bytes
+    # Adler32 is 4 bytes (8 hex chars / 2)
+    length_field = struct.pack("B", 4)  # unsigned 8-bit
+
+    # 7. Value[64] - Binary checksum value
+    # Convert hex string to 4 bytes, pad rest with zeros
+    checksum_bytes = bytes.fromhex(checksum)
+    value_field = checksum_bytes + b"\x00" * (64 - len(checksum_bytes))
+
+    # Assemble complete structure
+    xrd_cks_data = (
+        name_field  # 16 bytes
+        + fm_time_field  #  8 bytes
+        + cs_time_field  #  4 bytes
+        + rsvd1_field  #  2 bytes
+        + rsvd2_field  #  1 byte
+        + length_field  #  1 byte
+        + value_field  # 64 bytes
+    )  # Total: 96 bytes
+
+    assert len(xrd_cks_data) == 96, f"Structure size mismatch: {len(xrd_cks_data)}"
+
+    # Set the extended attribute
+    # XRootD uses "XrdCks.adler32" which becomes "user.XrdCks.adler32" on Linux
+    attr_name = "user.XrdCks.adler32"
+
+    try:
+        xattr.setxattr(path, attr_name, xrd_cks_data)
+    except OSError as e:
+        raise OSError(f"Failed to set extended attribute on {path}: {e}") from e
+
+
 class FileStorage(StorageBase):
     """.. class:: FileStorage
 
@@ -165,6 +278,12 @@ class FileStorage(StorageBase):
                     os.makedirs(dirname)
                 shutil.copy2(src_file, dest_url)
                 fileSize = os.path.getsize(dest_url)
+                try:
+                    src_cks = fileAdler(src_file)
+                    set_xattr_adler32(dest_url, src_cks)
+                except Exception as e:
+                    gLogger.warn("Could not set checksum", f"{e!r}")
+
                 if sourceSize and (sourceSize != fileSize):
                     try:
                         os.unlink(dest_url)

{dirac-9.0.11 → dirac-9.0.13/src/DIRAC.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: DIRAC
-Version: 9.0.11
+Version: 9.0.13
 Summary: DIRAC is an interware, meaning a software framework for distributed computing.
 Home-page: https://github.com/DIRACGrid/DIRAC/
 License: GPL-3.0-only
@@ -19,7 +19,7 @@ Requires-Dist: cachetools
 Requires-Dist: certifi
 Requires-Dist: cwltool
 Requires-Dist: diraccfg
-Requires-Dist: DIRACCommon==v9.0.11
+Requires-Dist: DIRACCommon==v9.0.13
 Requires-Dist: diracx-client>=v0.0.1
 Requires-Dist: diracx-core>=v0.0.1
 Requires-Dist: diracx-cli>=v0.0.1
@@ -47,6 +47,7 @@ Requires-Dist: Authlib>=1.0.0.a2
 Requires-Dist: pyjwt
 Requires-Dist: dominate
 Requires-Dist: zstandard
+Requires-Dist: xattr
 Provides-Extra: server
 Requires-Dist: CMRESHandler; extra == "server"
 Requires-Dist: opensearch-py; extra == "server"

{dirac-9.0.11 → dirac-9.0.13}/src/DIRAC.egg-info/requires.txt

@@ -4,7 +4,7 @@ cachetools
 certifi
 cwltool
 diraccfg
-DIRACCommon==v9.0.11
+DIRACCommon==v9.0.13
 diracx-client>=v0.0.1
 diracx-core>=v0.0.1
 diracx-cli>=v0.0.1
@@ -32,6 +32,7 @@ Authlib>=1.0.0.a2
 pyjwt
 dominate
 zstandard
+xattr
 
 [server]
 CMRESHandler