DIRAC 9.0.0a42__py3-none-any.whl → 9.0.7__py3-none-any.whl

DIRAC/Core/scripts/dirac_platform.py

@@ -23,107 +23,16 @@ try:
 except Exception:
     import argparse
     import platform
-    import os
-    import sys
-    import re
-    import subprocess
 
     parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
     parser.parse_known_args()
 
-    # We need to patch python platform module. It does a string comparison for the libc versions.
-    # it fails when going from 2.9 to 2.10,
-    # the fix converts the version to a tuple and attempts a numeric comparison
-
-    _libc_search = re.compile(r"(__libc_init)" "|" "(GLIBC_([0-9.]+))" "|" r"(libc(_\w+)?\.so(?:\.(\d[0-9.]*))?)")
-
-    def libc_ver(executable=sys.executable, lib="", version="", chunksize=2048):
-        """Tries to determine the libc version that the file executable
-        (which defaults to the Python interpreter) is linked against.
-
-        Returns a tuple of strings (lib,version) which default to the
-        given parameters in case the lookup fails.
-
-        Note that the function has intimate knowledge of how different
-        libc versions add symbols to the executable and thus is probably
-        only useable for executables compiled using gcc.
-
-        The file is read and scanned in chunks of chunksize bytes.
-
-        """
-        with open(executable, "rb") as f:
-            binary = f.read(chunksize)
-            pos = 0
-            version = [0, 0, 0]
-            while True:
-                m = _libc_search.search(binary, pos)
-                if not m:
-                    binary = f.read(chunksize)
-                    if not binary:
-                        break
-                    pos = 0
-                    continue
-                libcinit, glibc, glibcversion, so, threads, soversion = m.groups()
-                if libcinit and not lib:
-                    lib = "libc"
-                elif glibc:
-                    glibcversion_parts = glibcversion.split(".")
-                    for i, part in enumerate(glibcversion_parts):
-                        try:
-                            glibcversion_parts[i] = int(part)
-                        except ValueError:
-                            glibcversion_parts[i] = 0
-                    if libcinit and not lib:
-                        lib = "libc"
-                    elif glibc:
-                        if lib != "glibc":
-                            lib = "glibc"
-                            version = glibcversion_parts
-                        elif glibcversion_parts > version:
-                            version = glibcversion_parts
-                elif so:
-                    if lib != "glibc":
-                        lib = "libc"
-                        version = max(version, soversion)
-                        if threads and version[-len(threads) :] != threads:
-                            version = version + threads
-                pos = m.end()
-        return lib, ".".join(map(str, version))
-
     # Command line interface
     def getPlatformString():
         # Modified to return our desired platform string, R. Graciani
         platformTuple = (platform.system(), platform.machine())
         if platformTuple[0] == "Linux":
-            sp = subprocess.Popen(["/sbin/ldconfig", "--print-cache"], stdout=subprocess.PIPE)
-            ldre = re.compile(r".*=> (.*/libc\.so\..*$)")
-            libs = []
-            for line in sp.stdout.readlines():
-                reM = ldre.match(line)
-                if reM:
-                    libs.append(reM.groups()[0])
-            if not libs:
-                # get version of higher libc installed
-                if platform.machine().find("64") != -1:
-                    lib = "/lib64"
-                else:
-                    lib = "/lib"
-                for libFile in os.listdir(lib):
-                    if libFile.find("libc-") == 0 or libFile.find("libc.so") == 0:
-                        libs.append(os.path.join(lib, libFile))
-            newest_lib = [0, 0, 0]
-            for lib in libs:
-                lib_parts = libc_ver(lib)[1].split(".")
-                for i, part in enumerate(lib_parts):
-                    try:
-                        lib_parts[i] = int(part)
-                    except ValueError:
-                        lib_parts[i] = 0
-                        # print "non integer version numbers"
-                if lib_parts > newest_lib:
-                    newest_lib = lib_parts
-
-            platformTuple += ("glibc-" + ".".join(map(str, newest_lib)),)
+            platformTuple += ("-".join(platform.libc_ver()),)
         elif platformTuple[0] == "Darwin":
             platformTuple += (".".join(platform.mac_ver()[0].split(".")[:2]),)
         else:

DIRAC/DataManagementSystem/Agent/FTS3Agent.py

@@ -719,10 +719,11 @@ class FTS3Agent(AgentModule):
         return self.dataOpSender.concludeSending()
 
     def __sendAccounting(self, ftsJob):
-        self.dataOpSender.sendData(
-            ftsJob.accountingDict,
-            commitFlag=True,
-            delayedCommit=True,
-            startTime=fromString(ftsJob.submitTime),
-            endTime=fromString(ftsJob.lastUpdate),
-        )
+        for accountingDict in ftsJob.accountingDicts:
+            self.dataOpSender.sendData(
+                accountingDict,
+                commitFlag=True,
+                delayedCommit=True,
+                startTime=fromString(ftsJob.submitTime),
+                endTime=fromString(ftsJob.lastUpdate),
+            )

DIRAC/DataManagementSystem/Agent/RequestOperations/RemoveFile.py

@@ -4,17 +4,17 @@
 # Date: 2013/03/25 07:44:19
 ########################################################################
 
-""" :mod: RemoveFile
+""":mod: RemoveFile
 
-    ================
+================
 
-    .. module: RemoveFile
+.. module: RemoveFile
 
-    :synopsis: removeFile operation handler
+:synopsis: removeFile operation handler
 
-    .. moduleauthor:: Krzysztof.Ciba@NOSPAMgmail.com
+.. moduleauthor:: Krzysztof.Ciba@NOSPAMgmail.com
 
-    removeFile operation handler
+removeFile operation handler
 """
 # #
 # @file RemoveFile.py
@@ -132,6 +132,7 @@ class RemoveFile(DMSRequestOperationsBase):
                     self.rmsMonitoringReporter.addRecord(
                         self.createRMSRecord("Successful", len(toRemoveDict) - len(bulkRemoval["Value"]))
                     )
+                toRemoveDict = bulkRemoval["Value"]
 
             # # 2nd step - single file removal
             for lfn, opFile in toRemoveDict.items():

DIRAC/DataManagementSystem/Client/FTS3Job.py

@@ -1,9 +1,11 @@
-""" FTS3Job module containing only the FTS3Job class """
+"""FTS3Job module containing only the FTS3Job class"""
 
 import datetime
 import errno
+import os
 from packaging.version import Version
 
+from cachetools import cachedmethod, LRUCache
 
 # Requires at least version 3.3.3
 from fts3 import __version__ as fts3_version
@@ -26,8 +28,9 @@ from DIRAC.Resources.Storage.StorageElement import StorageElement
 
 from DIRAC.FrameworkSystem.Client.Logger import gLogger
 from DIRAC.FrameworkSystem.Client.TokenManagerClient import gTokenManager
+from DIRAC.FrameworkSystem.Utilities.TokenManagementUtilities import getIdProviderClient
 
-from DIRAC.Core.Utilities.ReturnValues import S_OK, S_ERROR
+from DIRAC.Core.Utilities.ReturnValues import S_OK, S_ERROR, returnValueOrRaise
 from DIRAC.Core.Utilities.DErrno import cmpError
 
 from DIRAC.Core.Utilities.JEncode import JSerializable
@@ -36,6 +39,10 @@ from DIRAC.DataManagementSystem.Client.FTS3File import FTS3File
 # 3 days in seconds
 BRING_ONLINE_TIMEOUT = 259200
 
+# Number of IdP to keep in cache. Should correspond roughly
+# to the number of groups performing transfers
+IDP_CACHE_SIZE = 8
+
 
 class FTS3Job(JSerializable):
     """Abstract class to represent a job to be executed by FTS. It belongs
@@ -78,6 +85,8 @@ class FTS3Job(JSerializable):
         "userGroup",
     ]
 
+    _idp_cache = LRUCache(maxsize=IDP_CACHE_SIZE)
+
     def __init__(self):
         self.submitTime = None
         self.lastUpdate = None
@@ -111,7 +120,12 @@ class FTS3Job(JSerializable):
         # temporary used only for accounting
         # it is set by the monitor method
         # when a job is in a final state
-        self.accountingDict = None
+        self.accountingDicts = None
+
+    @classmethod
+    @cachedmethod(lambda cls: cls._idp_cache)
+    def _getIdpClient(cls, group_name: str):
+        return returnValueOrRaise(getIdProviderClient(group_name, None, client_name_prefix="fts"))
 
     def monitor(self, context=None, ftsServer=None, ucert=None):
         """Queries the fts server to monitor the job.
@@ -143,7 +157,6 @@ class FTS3Job(JSerializable):
 
         if not self.ftsGUID:
             return S_ERROR("FTSGUID not set, FTS job not submitted?")
-
         if not context:
             if not ftsServer:
                 ftsServer = self.ftsServer
@@ -170,13 +183,14 @@ class FTS3Job(JSerializable):
             self.error = jobStatusDict["reason"]
 
         if newStatus in self.FINAL_STATES:
-            self._fillAccountingDict(jobStatusDict)
+            self._fillAccountingDicts(jobStatusDict)
 
         filesInfoList = jobStatusDict["files"]
         filesStatus = {}
         statusSummary = {}
 
         # Make a copy, since we are potentially
+
         # deleting objects
         for fileDict in list(filesInfoList):
             file_state = fileDict["file_state"].capitalize()
@@ -231,7 +245,7 @@ class FTS3Job(JSerializable):
         # so we put this back into the monitoring data such that the accounting is done properly
         jobStatusDict["files"] = filesInfoList
         if newStatus in self.FINAL_STATES:
-            self._fillAccountingDict(jobStatusDict)
+            self._fillAccountingDicts(jobStatusDict)
 
         total = len(filesInfoList)
         completed = sum(statusSummary.get(state, 0) for state in FTS3File.FTS_FINAL_STATES)
@@ -509,11 +523,10 @@ class FTS3Job(JSerializable):
                     if not res["OK"]:
                         return res
                     srcTokenPath = res["Value"]
-                    res = gTokenManager.getToken(
-                        userGroup=self.userGroup,
-                        requiredTimeLeft=3600,
+                    res = self._getIdpClient(self.userGroup).fetchToken(
+                        grant_type="client_credentials",
                         scope=[f"storage.read:/{srcTokenPath}", "offline_access"],
-                        useCache=False,
+                        # TODO: add a specific audience
                     )
                     if not res["OK"]:
                         return res
@@ -528,11 +541,17 @@ class FTS3Job(JSerializable):
                     if not res["OK"]:
                         return res
                     dstTokenPath = res["Value"]
-                    res = gTokenManager.getToken(
-                        userGroup=self.userGroup,
-                        requiredTimeLeft=3600,
-                        scope=[f"storage.modify:/{dstTokenPath}", f"storage.read:/{dstTokenPath}", "offline_access"],
-                        useCache=False,
+                    res = self._getIdpClient(self.userGroup).fetchToken(
+                        grant_type="client_credentials",
+                        scope=[
+                            f"storage.modify:/{dstTokenPath}",
+                            f"storage.read:/{dstTokenPath}",
+                            # Needed because CNAF
+                            # https://ggus.eu/index.php?mode=ticket_info&ticket_id=165048
+                            f"storage.read:/{os.path.dirname(dstTokenPath)}",
+                            "offline_access",
+                        ],
+                        # TODO: add a specific audience
                     )
                     if not res["OK"]:
                         return res
@@ -728,6 +747,7 @@ class FTS3Job(JSerializable):
             retry=3,
             metadata=job_metadata,
             priority=self.priority,
+            unmanaged_tokens=True,
             **dest_spacetoken,
         )
 
@@ -882,9 +902,9 @@ class FTS3Job(JSerializable):
             gLogger.exception("Error generating context", repr(e))
             return S_ERROR(repr(e))
 
-    def _fillAccountingDict(self, jobStatusDict):
-        """This methods generates the necessary information to create a DataOperation
-        accounting record, and stores them as a instance attribute.
+    def _fillAccountingDicts(self, jobStatusDict):
+        """This methods generates the necessary information to create DataOperation
+        accounting records, and stores them as a instance attribute.
 
         For it to be relevant, it should be called only when the job is in a final state.
 
@@ -893,6 +913,7 @@ class FTS3Job(JSerializable):
         :returns: None
         """
 
+        accountingDicts = []
         accountingDict = dict()
         sourceSE = None
         targetSE = None
@@ -903,16 +924,24 @@ class FTS3Job(JSerializable):
         accountingDict["Protocol"] = "FTS3"
         accountingDict["ExecutionSite"] = self.ftsServer
 
+        # Registration values must be set anyway
+        accountingDict["RegistrationTime"] = 0.0
+        accountingDict["RegistrationOK"] = 0
+        accountingDict["RegistrationTotal"] = 0
+
         # We cannot rely on all the transient attributes (like self.filesToSubmit)
         # because it is probably not filed by the time we monitor !
 
         filesInfoList = jobStatusDict["files"]
         successfulFiles = []
+        failedFiles = []
 
         for fileDict in filesInfoList:
             file_state = fileDict["file_state"].capitalize()
             if file_state in FTS3File.FTS_SUCCESS_STATES:
                 successfulFiles.append(fileDict)
+            else:
+                failedFiles.append(fileDict)
 
         job_metadata = jobStatusDict["job_metadata"]
         # previous version of the code did not have dictionary as
@@ -921,23 +950,31 @@ class FTS3Job(JSerializable):
             sourceSE = job_metadata.get("sourceSE")
             targetSE = job_metadata.get("targetSE")
 
-        accountingDict["TransferOK"] = len(successfulFiles)
-        accountingDict["TransferTotal"] = len(filesInfoList)
-        # We need this if in the list comprehension because staging only jobs have `None` as filesize
-        accountingDict["TransferSize"] = sum(
-            fileDict["filesize"] for fileDict in successfulFiles if fileDict["filesize"]
-        )
-        accountingDict["FinalStatus"] = self.status
         accountingDict["Source"] = sourceSE
         accountingDict["Destination"] = targetSE
-        # We need this if in the list comprehension because staging only jobs have `None` as tx_duration
-        accountingDict["TransferTime"] = sum(
-            int(fileDict["tx_duration"]) for fileDict in successfulFiles if fileDict["tx_duration"]
-        )
 
-        # Registration values must be set anyway
-        accountingDict["RegistrationTime"] = 0.0
-        accountingDict["RegistrationOK"] = 0
-        accountingDict["RegistrationTotal"] = 0
+        if successfulFiles:
+            successfulDict = accountingDict.copy()
+            successfulDict["TransferOK"] = len(successfulFiles)
+            successfulDict["TransferTotal"] = len(successfulFiles)
+            # We need this if in the list comprehension because staging only jobs have `None` as filesize
+            successfulDict["TransferSize"] = sum(
+                fileDict["filesize"] for fileDict in successfulFiles if fileDict["filesize"]
+            )
+            successfulDict["FinalStatus"] = "Finished"
 
-        self.accountingDict = accountingDict
+            # We need this if in the list comprehension because staging only jobs have `None` as tx_duration
+            successfulDict["TransferTime"] = sum(
+                int(fileDict["tx_duration"]) for fileDict in successfulFiles if fileDict["tx_duration"]
+            )
+            accountingDicts.append(successfulDict)
+        if failedFiles:
+            failedDict = accountingDict.copy()
+            failedDict["TransferOK"] = 0
+            failedDict["TransferTotal"] = len(failedFiles)
+            failedDict["TransferSize"] = 0
+            failedDict["FinalStatus"] = "Failed"
+            failedDict["TransferTime"] = 0
+            accountingDicts.append(failedDict)
+
+        self.accountingDicts = accountingDicts

DIRAC/DataManagementSystem/DB/FTS3DB.py

@@ -1,5 +1,5 @@
-""" Frontend to FTS3 MySQL DB. Written using sqlalchemy
-"""
+"""Frontend to FTS3 MySQL DB. Written using sqlalchemy"""
+
 # We disable the no-member error because
 # they are constructed by SQLAlchemy for all
 # the objects mapped to a table.
@@ -7,6 +7,7 @@
 
 import datetime
 import errno
+from urllib.parse import quote_plus
 
 from sqlalchemy import (
     BigInteger,
@@ -15,6 +16,7 @@ from sqlalchemy import (
     Enum,
     Float,
     ForeignKey,
+    Index,
     Integer,
     MetaData,
     SmallInteger,
@@ -84,6 +86,7 @@ fts3JobTable = Table(
     Column("error", String(2048)),
     Column("status", Enum(*FTS3Job.ALL_STATES), server_default=FTS3Job.INIT_STATE, index=True),
     Column("assignment", String(255), server_default=None),
+    Index("idx_jobs_lastupdate_assignment", "lastUpdate", "assignment"),
     mysql_engine="InnoDB",
 )
 
@@ -109,6 +112,7 @@ fts3OperationTable = Table(
     Column("error", String(1024)),
     Column("type", String(255)),
     Column("assignment", String(255), server_default=None),
+    Index("idx_operations_lastupdate_assignment", "lastUpdate", "assignment"),
     mysql_engine="InnoDB",
 )
 
@@ -184,7 +188,7 @@ class FTS3DB:
         self.dbHost = dbParameters["Host"]
         self.dbPort = dbParameters["Port"]
         self.dbUser = dbParameters["User"]
-        self.dbPass = dbParameters["Password"]
+        self.dbPass = quote_plus(dbParameters["Password"])
         self.dbName = dbParameters["DBName"]
 
     def __init__(self, pool_size=15, url=None, parentLogger=None):

DIRAC/DataManagementSystem/DB/FileCatalogComponents/DatasetManager/DatasetManager.py

@@ -12,7 +12,7 @@ class DatasetManager:
     _tables["FC_MetaDatasets"] = {
         "Fields": {
             "DatasetID": "INT AUTO_INCREMENT",
-            "DatasetName": "VARCHAR(128) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL",
+            "DatasetName": "VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL",
             "MetaQuery": "VARCHAR(512)",
             "DirID": "INT NOT NULL DEFAULT 0",
             "TotalSize": "BIGINT UNSIGNED NOT NULL",

DIRAC/DataManagementSystem/DB/FileCatalogDB.sql

@@ -23,7 +23,7 @@ CREATE TABLE FC_Files(
     UID SMALLINT UNSIGNED NOT NULL,
     GID TINYINT UNSIGNED NOT NULL,
     Status SMALLINT UNSIGNED NOT NULL,
-    FileName VARCHAR(128) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL,
+    FileName VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,
     INDEX (DirID),
     INDEX (UID,GID),
     INDEX (Status),
@@ -119,7 +119,7 @@ CREATE TABLE FC_Users (
 
 CREATE TABLE FC_StorageElements (
     SEID INTEGER AUTO_INCREMENT PRIMARY KEY,
-    SEName VARCHAR(127) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL,
+    SEName VARCHAR(127) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,
     AliasName VARCHAR(127) DEFAULT '',
     UNIQUE KEY (SEName)
 ) ENGINE = INNODB;
@@ -159,7 +159,7 @@ CREATE TABLE FC_DirectoryInfo (
 
 CREATE TABLE FC_DirMeta (
     DirID INTEGER NOT NULL,
-    MetaKey VARCHAR(31) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL DEFAULT 'Noname',
+    MetaKey VARCHAR(31) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT 'Noname',
     MetaValue VARCHAR(31) NOT NULL DEFAULT 'Noname',
     PRIMARY KEY (DirID,MetaKey)
 ) ENGINE = INNODB;
@@ -168,7 +168,7 @@ CREATE TABLE FC_DirMeta (
 
 CREATE TABLE FC_FileMeta (
     FileID INTEGER NOT NULL,
-    MetaKey VARCHAR(31) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL DEFAULT 'Noname',
+    MetaKey VARCHAR(31) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT 'Noname',
     MetaValue VARCHAR(31) NOT NULL DEFAULT 'Noname',
     PRIMARY KEY (FileID,MetaKey)
 ) ENGINE = INNODB;
@@ -177,7 +177,7 @@ CREATE TABLE FC_FileMeta (
 
 CREATE TABLE FC_DirectoryTree (
  DirID INT AUTO_INCREMENT PRIMARY KEY,
- DirName VARCHAR(1024) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL,
+ DirName VARCHAR(1024) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,
  Parent INT NOT NULL DEFAULT 0,
  INDEX (Parent),
  INDEX (DirName)
@@ -187,7 +187,7 @@ CREATE TABLE FC_DirectoryTree (
 
 CREATE TABLE FC_DirectoryTreeM (
  DirID INT AUTO_INCREMENT PRIMARY KEY,
- DirName VARCHAR(255) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL,
+ DirName VARCHAR(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,
  Parent INT NOT NULL DEFAULT 0,
  Level INT NOT NULL,
  INDEX (Level),
@@ -199,7 +199,7 @@ CREATE TABLE FC_DirectoryTreeM (
 
 CREATE TABLE FC_DirectoryLevelTree (
  DirID INT AUTO_INCREMENT PRIMARY KEY,
- DirName VARCHAR(255) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL,
+ DirName VARCHAR(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,
  Parent INT NOT NULL DEFAULT 0,
  Level INT NOT NULL,
  LPATH1 INT NOT NULL DEFAULT 0,
@@ -240,7 +240,7 @@ CREATE TABLE FC_DirectoryUsage(
 
 CREATE TABLE FC_MetaFields (
   MetaID INT AUTO_INCREMENT PRIMARY KEY,
-  MetaName VARCHAR(64) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL,
+  MetaName VARCHAR(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,
   MetaType VARCHAR(128) NOT NULL
 ) ENGINE = INNODB;
 
@@ -248,7 +248,7 @@ CREATE TABLE FC_MetaFields (
 
 CREATE TABLE FC_FileMetaFields (
   MetaID INT AUTO_INCREMENT PRIMARY KEY,
-  MetaName VARCHAR(64) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL,
+  MetaName VARCHAR(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,
   MetaType VARCHAR(128) NOT NULL
 ) ENGINE = INNODB;
 

DIRAC/DataManagementSystem/DB/FileCatalogWithFkAndPsDB.sql

@@ -36,7 +36,7 @@ INSERT INTO FC_Statuses (StatusID, Status) values (1, 'FakeStatus');
 
 CREATE TABLE FC_StorageElements (
     SEID INTEGER AUTO_INCREMENT,
-    SEName VARCHAR(127) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL,
+    SEName VARCHAR(127) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,
     AliasName VARCHAR(127) DEFAULT '',
 
     PRIMARY KEY (SEID),
@@ -79,7 +79,7 @@ INSERT INTO FC_Users (UID, UserName) values (1, 'root');
 --
 -- create table FC_DirectoryList (
 --   DirID INT NOT NULL AUTO_INCREMENT,
---   Name varchar(255)CHARACTER SET latin1 COLLATE latin1_bin NOT NULL,
+--   Name varchar(255)CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,
 --
 --   PRIMARY KEY (DirID),
 --
@@ -116,7 +116,7 @@ create table FC_DirectoryList (
   ModificationDate DATETIME,
   Mode SMALLINT UNSIGNED NOT NULL DEFAULT 775,
   Status INTEGER NOT NULL DEFAULT 0,
-  Name varchar(255)CHARACTER SET latin1 COLLATE latin1_bin NOT NULL,
+  Name varchar(255)CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,
 
   PRIMARY KEY (DirID),
   FOREIGN KEY (UID) REFERENCES FC_Users(UID),
@@ -158,7 +158,7 @@ CREATE TABLE FC_Files(
     Mode SMALLINT UNSIGNED NOT NULL DEFAULT 775,
     ChecksumType ENUM('Adler32','MD5'),
     Checksum VARCHAR(32),
-    FileName VARCHAR(128) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL,
+    FileName VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,
 
     PRIMARY KEY (FileID),
     FOREIGN KEY (DirID) REFERENCES FC_DirectoryList(DirID) ON DELETE CASCADE,
@@ -223,7 +223,7 @@ CREATE TABLE FC_DirectoryUsage(
 
 CREATE TABLE FC_DirMeta (
     DirID INTEGER NOT NULL,
-    MetaKey VARCHAR(31) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL DEFAULT 'Noname',
+    MetaKey VARCHAR(31) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT 'Noname',
     MetaValue VARCHAR(31) NOT NULL DEFAULT 'Noname',
     PRIMARY KEY (DirID,MetaKey)
 ) ENGINE = INNODB;
@@ -232,7 +232,7 @@ CREATE TABLE FC_DirMeta (
 
 CREATE TABLE FC_FileMeta (
     FileID INTEGER NOT NULL,
-    MetaKey VARCHAR(31) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL DEFAULT 'Noname',
+    MetaKey VARCHAR(31) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT 'Noname',
     MetaValue VARCHAR(31) NOT NULL DEFAULT 'Noname',
     PRIMARY KEY (FileID,MetaKey)
 ) ENGINE = INNODB;
@@ -242,7 +242,7 @@ CREATE TABLE FC_FileMeta (
 
 CREATE TABLE FC_MetaFields (
   MetaID INT AUTO_INCREMENT PRIMARY KEY,
-  MetaName VARCHAR(64) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL,
+  MetaName VARCHAR(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,
   MetaType VARCHAR(128) NOT NULL
 ) ENGINE = INNODB;
 
@@ -250,7 +250,7 @@ CREATE TABLE FC_MetaFields (
 
 CREATE TABLE FC_FileMetaFields (
   MetaID INT AUTO_INCREMENT PRIMARY KEY,
-  MetaName VARCHAR(64) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL,
+  MetaName VARCHAR(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,
   MetaType VARCHAR(128) NOT NULL
 ) ENGINE = INNODB;
 
@@ -280,7 +280,7 @@ CREATE TABLE FC_FileAncestors (
 
 CREATE TABLE FC_MetaDatasets (
   DatasetID INT AUTO_INCREMENT,
-  DatasetName VARCHAR(128) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL,
+  DatasetName VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,
   MetaQuery VARCHAR(512),
   DirID INT NOT NULL DEFAULT 0,
   TotalSize BIGINT UNSIGNED NOT NULL,

DIRAC/DataManagementSystem/Utilities/DMSHelpers.py

@@ -1,7 +1,8 @@
 """
-  This module contains helper methods for accessing operational attributes or parameters of DMS objects
+This module contains helper methods for accessing operational attributes or parameters of DMS objects
 
 """
+
 from collections import defaultdict
 from DIRAC import gConfig, gLogger, S_OK, S_ERROR
 from DIRAC.ConfigurationSystem.Client.Helpers.Path import cfgPath
@@ -17,6 +18,9 @@ sLog = gLogger.getSubLogger(__name__)
 def resolveSEGroup(seGroupList, allSEs=None):
     """
     Resolves recursively a (list of) SEs that can be groupSEs
+    For modules, JobWrapper or whatever runs at a given site,
+    prefer using :py:func:`~DIRAC.DataManagementSystem.Utilities.ResolveSE.getDestinationSEList`
+
 
     :param seGroupList: list of SEs to resolve or comma-separated SEs
     :type seGroupList: list or string
@@ -402,7 +406,7 @@ class DMSHelpers:
             return sesAtSite
         foundSEs = set(seList) & set(sesAtSite["Value"])
         if not foundSEs:
-            sLog.warn("No SE found at that site", f"in group {seGroup} at {site}")
+            sLog.verbose("No SE found at that site", f"in group {seGroup} at {site}")
             return S_OK()
         return S_OK(sorted(foundSEs))