DIRAC 7.2.0a36__tar.gz → 7.2.0a39__tar.gz

{DIRAC-7.2.0a36/src/DIRAC.egg-info → DIRAC-7.2.0a39}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: DIRAC
-Version: 7.2.0a36
+Version: 7.2.0a39
 Summary: DIRAC is an interware, meaning a software framework for distributed computing.
 Home-page: https://github.com/DIRACGrid/DIRAC/
 License: GPL-3.0-only

{DIRAC-7.2.0a36 → DIRAC-7.2.0a39}/environment-py3.yml

@@ -63,6 +63,7 @@ dependencies:
   # docs
   - pygments >=1.5
   - sphinx
+  - graphviz
   # unused
   - funcsigs
   - jinja2

{DIRAC-7.2.0a36 → DIRAC-7.2.0a39}/release.notes

@@ -1,4 +1,4 @@
-[v7r2-pre36]
+[v7r2-pre39]
 
 FIX: python 2-3 compatibility fixes
 NEW: (#4209) Extends the MonitoringSystem to support RMS and DMS/Agents/RequestOperations.
@@ -9,6 +9,7 @@ NEW: Moved to a src/ repository layout
 NEW: (#4906) Deploy releases to PyPI
 FIX: (#4921) use six.moves for urllib
 CHANGE: (#4958) use __doc__ for script help message, remove filecfg as argument from usage description
+FIX: (#5015) Add prompt-toolkit to setuptools metadata
 
 *Core
 FIX: (#4283) Added protection against empty certificates directory in dirac-install
@@ -39,11 +40,13 @@ FIX: (#4956) use logging.warning in favor of deprecated logging.warn
 CHANGE: (#4997) Use prompt-toolkit for password input when generating proxies
 CHANGE: (#4994) dirac-install.py should now be taken from management
 CHANGE: (#5010) Use ConfigurationClient in CSCLI and CSShellCLI
+CHANGE: (#5026) Remove DIRAC.Core.Utilities.ExitCallback
 
 *Configuration
 CHANGE: (#4866) split standard and tornado refresher in two files to avoid tornado dependency on the client
 CHANGE: (#4958) setUsageMessage in LocalConfiguration will parse received str value for building help message
 FIX: (#5008) HTTPS: allow to run master CS in parallel of other tornado services
+FIX: (#5016) harmonize https and dips version of ServiceHandler, and make doc consistent with reality
 
 *Framework
 CHANGE: (#4787) expose deleteProxyBundle call to ProxyManagerClient
@@ -78,11 +81,16 @@ CHANGE: (#4925) SiteDirector - removed using of SubmitPools requirement by defau
 CHANGE: (#4925) SiteDirector, JobAgent - no more using ProxyManager tokens while user proxy retrieval
 NEW: (#4950) JobAgent adds GridCE and queue to Job Parameters
 CHANGE: (#4966) added flag to exclude the master CS from the list of servers synchronized by PilotCS2JSON
+FIX: (#5017) Fix uploading jobDescription.xml in upload sandboxes with Python 3
+
+*DataManagement
+CHANGE: (#5019) Remove IRODSStorageElementHandler
 
 *Monitoring
 FIX: (#4768) Adapting RMSMonitoring type to ES7
 FIX: (#4844) Correct the rounding of integers when computing buckets in MonitoringDB
 NEW: (#4898) added possibility to configure the period and so the indexing strategy
+CHANGE: (#4976) removed DynamicMonitoring (replaced by ActivityMonitoring)
 
 *ResourceStatusSystem
 CHANGE: (#4235) Removes locals() way of passing parameters
@@ -107,6 +115,22 @@ NEW: (#4910) --runslow option on unit tests to allow faster local tests
 NEW: (#4938) added a helloworld test for the (yet to be implemented) cloud testing in certification
 CHANGE: (#4968) Change the defaults for tests (to MySQL 8 and ES 7)
 
+[v7r1p33]
+
+*DMS
+FIX: (#4960) FTS3 explicitly use verify_checksum
+NEW: (#4960) add --FTSOnly and --ExcludeSE options to dirac-dms-protocol-matrix
+
+*RMS
+CHANGE: (#4960) ReqClient prints the FTS3 job url directly
+
+[v7r1p32]
+
+*Core
+FIX: (#5021) Glue2: fix error when too many execution environments were looked for
+FIX: (#5021) Subprocess: when the systemCall failed, S_ERROR now contains a string
+             in 'Message' instead of an exception object
+
 [v7r1p31]
 
 *Resources

{DIRAC-7.2.0a36 → DIRAC-7.2.0a39}/releases.cfg

@@ -24,9 +24,19 @@ Releases
     Modules = DIRAC
     DIRACOS = master
   }
-  v7r2-pre36
+  v7r2-pre39
   {
-    Modules = DIRAC, VMDIRAC:v2r4p9, RESTDIRAC:v0r5, COMDIRAC:v0r19, WebAppDIRAC:v4r2-pre5, OAuthDIRAC:v0r1-pre6
+    Modules = DIRAC, VMDIRAC:v2r4p9, RESTDIRAC:v0r5, COMDIRAC:v0r19, WebAppDIRAC:v4r2-pre6, OAuthDIRAC:v0r1-pre6
+    DIRACOS = v1r18
+  }
+  v7r1p33
+  {
+    Modules = DIRAC, VMDIRAC:v2r4p9, RESTDIRAC:v0r5, COMDIRAC:v0r19, WebAppDIRAC:v4r0p34, OAuthDIRAC:v0r1-pre6
+    DIRACOS = v1r18
+  }
+  v7r1p32
+  {
+    Modules = DIRAC, VMDIRAC:v2r4p9, RESTDIRAC:v0r5, COMDIRAC:v0r19, WebAppDIRAC:v4r0p34, OAuthDIRAC:v0r1-pre6
     DIRACOS = v1r18
   }
   v7r1p31

{DIRAC-7.2.0a36 → DIRAC-7.2.0a39}/setup.cfg

@@ -34,6 +34,7 @@ install_requires =
 	gfal2-python
 	M2Crypto >=0.36
 	pexpect
+	prompt-toolkit >=3
 	psutil
 	pyasn1
 	pyasn1-modules
@@ -60,7 +61,6 @@ server =
 	mysqlclient
 	numpy
 	pillow
-	python-irodsclient
 	python-json-logger
 	stomp.py
 	suds-jurko

{DIRAC-7.2.0a36 → DIRAC-7.2.0a39}/src/DIRAC/AccountingSystem/Client/AccountingCLI.py

@@ -14,7 +14,6 @@ import six
 
 from DIRAC import gLogger
 from DIRAC.Core.Base.CLI import CLI, colorize
-from DIRAC.Core.Utilities import ExitCallback
 from DIRAC.AccountingSystem.Client.DataStoreClient import DataStoreClient
 
 
@@ -23,7 +22,6 @@ class AccountingCLI(CLI):
   def __init__(self):
     CLI.__init__(self)
     self.do_connect(None)
-    ExitCallback.registerExitCallback(self.do_quit)
 
   def start(self):
     """

{DIRAC-7.2.0a36 → DIRAC-7.2.0a39}/src/DIRAC/AccountingSystem/Client/ReportCLI.py

@@ -28,7 +28,6 @@ import sys
 import datetime
 
 from DIRAC.Core.Base.CLI import CLI, colorize
-from DIRAC.Core.Utilities import ExitCallback
 from DIRAC.AccountingSystem.Client.ReportsClient import ReportsClient
 from DIRAC import gLogger
 
@@ -38,7 +37,6 @@ class ReportCLI(CLI):
   def __init__(self):
     CLI.__init__(self)
     self.do_connect(None)
-    ExitCallback.registerExitCallback(self.do_quit)
 
   def start(self):
     """

{DIRAC-7.2.0a36 → DIRAC-7.2.0a39}/src/DIRAC/ConfigurationSystem/Service/ConfigurationHandler.py

@@ -83,7 +83,7 @@ class ConfigurationHandler(RequestHandler):
     """
     Attempt to request all the configured services to update their configuration
 
-    :return: S_OK, Value Successful/Failed dict with service URLs
+    :return: S_OK
     """
     return gServiceInterface.forceGlobalUpdate()
 

{DIRAC-7.2.0a36 → DIRAC-7.2.0a39}/src/DIRAC/ConfigurationSystem/private/ConfigurationData.py

@@ -285,7 +285,7 @@ class ConfigurationData(object):
 
   def getCompressedData(self):
     if self.__compressedConfigurationData is None:
-      self.__compressedConfigurationData = zlib.compress(str(self.remoteCFG), 9)
+      self.__compressedConfigurationData = zlib.compress(str(self.remoteCFG).encode(), 9)
     return self.__compressedConfigurationData
 
   def isMaster(self):

{DIRAC-7.2.0a36 → DIRAC-7.2.0a39}/src/DIRAC/ConfigurationSystem/private/Modificator.py

@@ -243,7 +243,7 @@ class Modificator(object):
     return str(self.cfgData)
 
   def commit(self):
-    compressedData = zlib.compress(str(self.cfgData), 9)
+    compressedData = zlib.compress(str(self.cfgData).encode(), 9)
     return self.rpcClient.commitNewData(compressedData)
 
   def getHistory(self, limit=0):

{DIRAC-7.2.0a36 → DIRAC-7.2.0a39}/src/DIRAC/ConfigurationSystem/private/ServiceInterface.py

@@ -45,7 +45,7 @@ class ServiceInterface(ServiceInterfaceBase, threading.Thread):
 
     :param set urlSet: a set of service URLs
     :param fromMaster: flag to force updating from the master CS
-    :return: S_OK/S_ERROR, Value Successful/Failed dict with service URLs
+    :return: Nothing
     """
     pool = ThreadPool(len(urlSet))
     for url in urlSet:
@@ -54,7 +54,6 @@ class ServiceInterface(ServiceInterfaceBase, threading.Thread):
                                  kwargs={},
                                  oCallback=self.__processResults)
     pool.processAllResults()
-    return S_OK()
 
   def __processResults(self, _id, result):
     if not result['OK']:

{DIRAC-7.2.0a36 → DIRAC-7.2.0a39}/src/DIRAC/ConfigurationSystem/private/ServiceInterfaceBase.py

@@ -151,7 +151,7 @@ class ServiceInterfaceBase(object):
 
     :param set urlSet: a set of service URLs
     :param fromMaster: flag to force updating from the master CS
-    :return: S_OK/S_ERROR, Value Successful/Failed dict with service URLs
+    :return: Nothing
     """
     raise NotImplementedError("Should be implemented by the children class")
 
@@ -159,7 +159,7 @@ class ServiceInterfaceBase(object):
     """
     Force updating configuration on all the slave configuration servers
 
-    :return: S_OK/S_ERROR, Value Successful/Failed dict with service URLs
+    :return: Nothing
     """
     gLogger.info("Updating configuration on slave servers")
     iGraceTime = gConfigurationData.getSlavesGraceTime()
@@ -167,13 +167,13 @@ class ServiceInterfaceBase(object):
     for slaveURL in self.dAliveSlaveServers:
       if time.time() - self.dAliveSlaveServers[slaveURL] <= iGraceTime:
         urlSet.add(slaveURL)
-    return self._updateServiceConfiguration(urlSet, fromMaster=True)
+    self._updateServiceConfiguration(urlSet, fromMaster=True)
 
   def forceGlobalUpdate(self):
     """
     Force updating configuration of all the registered services
 
-    :return: S_OK/S_ERROR, Value Successful/Failed dict with service URLs
+    :returns: S_OK (needed for DISET return call)
     """
     gLogger.info("Updating services configuration")
     # Get URLs of all the services except for Configuration services
@@ -184,7 +184,8 @@ class ServiceInterfaceBase(object):
         for url in cfg[system_][instance]['URLs']:
           urlSet = urlSet.union(set([u.strip() for u in cfg[system_][instance]['URLs'][url].split(',')
                                      if 'Configuration/Server' not in u]))
-    return self._updateServiceConfiguration(urlSet)
+    self._updateServiceConfiguration(urlSet)
+    return S_OK()
 
   def updateConfiguration(self, sBuffer, committer="", updateVersionOption=False):
     """
@@ -240,9 +241,7 @@ class ServiceInterfaceBase(object):
 
     # Attempt to update the configuration on currently registered slave services
     if gConfigurationData.getAutoSlaveSync():
-      result = self.forceSlavesUpdate()
-      if not result['OK']:
-        gLogger.warn('Failed to update slave servers')
+      self.forceSlavesUpdate()
 
     return retVal
 
@@ -261,7 +260,7 @@ class ServiceInterfaceBase(object):
     backupDir = gConfigurationData.getBackupDir()
     files = self.__getCfgBackups(backupDir, date)
     for fileName in files:
-      with zipfile.ZipFile("%s/%s" % (backupDir, fileName), "r") as zFile:
+      with zipfile.ZipFile("%s/%s" % (backupDir, fileName), "rb") as zFile:
         cfgName = zFile.namelist()[0]
         retVal = S_OK(zlib.compress(zFile.read(cfgName), 9))
       return retVal

{DIRAC-7.2.0a36 → DIRAC-7.2.0a39}/src/DIRAC/ConfigurationSystem/private/ServiceInterfaceTornado.py

@@ -44,7 +44,7 @@ class ServiceInterfaceTornado(ServiceInterfaceBase):
 
     :param set urlSet: a set of service URLs
     :param fromMaster: flag to force updating from the master CS
-    :return: S_OK/S_ERROR, Value Successful/Failed dict with service URLs
+    :return: Nothing
     """
 
     for url in urlSet:

{DIRAC-7.2.0a36 → DIRAC-7.2.0a39}/src/DIRAC/Core/Base/API.py

@@ -9,8 +9,9 @@ import sys
 
 from DIRAC import gLogger, gConfig, S_OK, S_ERROR
 from DIRAC.Core.Security.ProxyInfo import getProxyInfo, formatProxyInfoAsString
-from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getDNForUsername
 from DIRAC.Core.Utilities.Version import getCurrentVersion
+from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getDNForUsername
+from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getSites
 
 __RCSID__ = '$Id$'
 
@@ -73,6 +74,7 @@ class API(object):
     self.errorDict = {}
     self.setup = gConfig.getValue('/DIRAC/Setup', 'Unknown')
     self.diracInfo = getCurrentVersion()['Value']
+    self._siteSet = set(getSites().get('Value', []))
 
   #############################################################################
 
@@ -132,6 +134,19 @@ class API(object):
 
   #############################################################################
 
+  def _checkSiteIsValid(self, site):
+    """Internal function to check that a site name is valid.
+    """
+    if isinstance(site, (list, set, dict)):
+      site = set(site) - self._siteSet
+      if not site:
+        return S_OK()
+    elif site in self._siteSet:
+      return S_OK()
+    return self._reportError('Specified site %s is not in list of defined sites' % str(site))
+
+  #############################################################################
+
   def _getCurrentUser(self):
     """ Get current user
 

{DIRAC-7.2.0a36 → DIRAC-7.2.0a39}/src/DIRAC/Core/Base/Script.py

@@ -19,7 +19,6 @@ from DIRAC.Core.Utilities.DErrno import includeExtensionErrors
 
 localCfg = LocalConfiguration()
 
-doc = inspect.currentframe().f_back.f_globals['__doc__']
 caller = inspect.currentframe().f_back.f_globals['__name__']
 
 
@@ -51,12 +50,19 @@ i = 0 if i == len(sys.argv) else i
 scriptName = os.path.basename(sys.argv[i].split('::')[0]).replace('.py', '')
 # The first argument DIRAC should parse is the next one
 localCfg.firstOptionIndex = i + 1
-# Read and parse the script __doc__ to create a draft help message
-localCfg.setUsageMessage(doc)
+
+gIsAlreadySetUsageMsg = False
 gIsAlreadyInitialized = False
 
 
 def parseCommandLine(script=False, ignoreErrors=False, initializeMonitor=False):
+  global gIsAlreadySetUsageMsg, gIsAlreadyInitialized
+
+  # Read and parse the script __doc__ to create a draft help message
+  if not gIsAlreadySetUsageMsg:
+    localCfg.setUsageMessage(inspect.currentframe().f_back.f_globals['__doc__'])
+    gIsAlreadySetUsageMsg = True
+
   if gIsAlreadyInitialized:
     return False
   gLogger.showHeaders(False)
@@ -126,6 +132,9 @@ def addDefaultOptionValue(option, value):
 
 
 def setUsageMessage(usageMessage):
+  global gIsAlreadySetUsageMsg
+  gIsAlreadySetUsageMsg = True
+  localCfg.setUsageMessage(inspect.currentframe().f_back.f_globals['__doc__'])
   localCfg.setUsageMessage(usageMessage)
 
 

{DIRAC-7.2.0a36 → DIRAC-7.2.0a39}/src/DIRAC/Core/Utilities/Glue2.py

@@ -23,6 +23,7 @@ from pprint import pformat
 from DIRAC import gLogger, gConfig
 from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getCESiteMapping, getGOCSiteName
 from DIRAC.Core.Utilities.ReturnValues import S_OK, S_ERROR
+from DIRAC.Core.Utilities.List import breakListIntoChunks
 
 __RCSID__ = "$Id$"
 
@@ -91,7 +92,7 @@ def getGlue2CEInfo(vo, host):
 
   siteInfo = __getGlue2ShareInfo(host, shareInfoLists)
   if not siteInfo['OK']:
-    sLog.error("Could not get CE info for", shareID + "  " + siteInfo['Message'])
+    sLog.error("Could not get CE info for", "%s: %s" % (shareID, siteInfo['Message']))
     return siteInfo
   siteDict = siteInfo['Value']
   sLog.debug("Found Sites:\n%s" % pformat(siteDict))
@@ -255,16 +256,23 @@ def __getGlue2ExecutionEnvironmentInfo(host, executionEnvironments):
   :param list executionEnvironments: list of the execution environments to get some information from
   :returns: result of the ldapsearch for all executionEnvironments, Glue2 schema
   """
-  exeFilter = ''
-  for execEnv in executionEnvironments:
-    exeFilter += '(GLUE2ResourceID=%s)' % execEnv
-  filt = "(&(objectClass=GLUE2ExecutionEnvironment)(|%s))" % exeFilter
-  response = __ldapsearchBDII(filt=filt, attr=None, host=host, base="o=glue", selectionString="GLUE2")
-  if not response['OK']:
-    return response
-  if not response['Value']:
-    return S_ERROR("No information found for %s" % executionEnvironments)
-  return response
+  listOfValues = []
+  # break up to avoid argument list too long, it started failing at about 1900 entries
+  for exeEnvs in breakListIntoChunks(executionEnvironments, 1000):
+    exeFilter = ''
+    for execEnv in exeEnvs:
+      exeFilter += '(GLUE2ResourceID=%s)' % execEnv
+    filt = "(&(objectClass=GLUE2ExecutionEnvironment)(|%s))" % exeFilter
+    response = __ldapsearchBDII(filt=filt, attr=None, host=host, base="o=glue", selectionString="GLUE2")
+    if not response['OK']:
+      return response
+    if not response['Value']:
+      sLog.error("No information found for %s" % executionEnvironments)
+      continue
+    listOfValues += response['Value']
+  if not listOfValues:
+    return S_ERROR("No information found for executionEnvironments")
+  return S_OK(listOfValues)
 
 
 def __getGlue2ExecutionEnvironmentInfoForSite(sitename, foreignKeys, exeInfos):

{DIRAC-7.2.0a36 → DIRAC-7.2.0a39}/src/DIRAC/Core/Utilities/MySQL.py

@@ -392,7 +392,7 @@ class MySQL(object):
     self.logger = self.log
 
     # let the derived class decide what to do with if is not 1
-    self._threadsafe = MySQLdb.thread_safe()
+    self._threadsafe = MySQLdb.threadsafety
     # self.log.debug('thread_safe = %s' % self._threadsafe)
 
     self.__hostName = str(hostName)

{DIRAC-7.2.0a36 → DIRAC-7.2.0a39}/src/DIRAC/Core/Utilities/ObjectLoader.py

@@ -98,7 +98,7 @@ class ObjectLoader(object):
       impModule = imp.load_module(modName[0], *impData)
       if impData[0]:
         impData[0].close()
-    except ImportError as excp:
+    except Exception as excp:
       if "No module named" in str(excp) and modName[0] in str(excp):
         return S_OK(None)
       errMsg = "Can't load %s in %s" % (".".join(modName), parentModule.__path__[0])

{DIRAC-7.2.0a36 → DIRAC-7.2.0a39}/src/DIRAC/Core/Utilities/Plotting/FileCoding.py

@@ -21,7 +21,7 @@ def codeRequestInFileId(plotRequest, compressIfPossible=True):
   compress = compressIfPossible and gZCompressionEnabled
   thbStub = False
   if compress:
-    plotStub = "Z:%s" % base64.urlsafe_b64encode(zlib.compress(DEncode.encode(plotRequest), 9))
+    plotStub = "Z:%s" % base64.urlsafe_b64encode(zlib.compress(DEncode.encode(plotRequest), 9)).decode()
   elif not gForceRawEncoding:
     plotStub = "S:%s" % base64.urlsafe_b64encode(DEncode.encode(plotRequest))
   else:
@@ -32,11 +32,11 @@ def codeRequestInFileId(plotRequest, compressIfPossible=True):
     thbStub = plotStub
     extraArgs['thumbnail'] = False
     if compress:
-      plotStub = "Z:%s" % base64.urlsafe_b64encode(zlib.compress(DEncode.encode(plotRequest), 9))
+      plotStub = "Z:%s" % base64.urlsafe_b64encode(zlib.compress(DEncode.encode(plotRequest), 9)).decode()
     elif not gForceRawEncoding:
-      plotStub = "S:%s" % base64.urlsafe_b64encode(DEncode.encode(plotRequest))
+      plotStub = "S:%s" % base64.urlsafe_b64encode(DEncode.encode(plotRequest)).decode()
     else:
-      plotStub = "R:%s" % DEncode.encode(plotRequest)
+      plotStub = "R:%s" % DEncode.encode(plotRequest).decode()
   return S_OK({'plot': plotStub, 'thumbnail': thbStub})
 
 

{DIRAC-7.2.0a36 → DIRAC-7.2.0a39}/src/DIRAC/Core/Utilities/Subprocess.py

@@ -440,7 +440,7 @@ class Subprocess(object):
                                     universal_newlines=True)
       self.childPID = self.child.pid
     except OSError as v:
-      retDict = S_ERROR(v)
+      retDict = S_ERROR(repr(v))
       retDict['Value'] = (-1, '', str(v))
       return retDict
     except Exception as x:
@@ -449,7 +449,7 @@ class Subprocess(object):
         self.child.stderr.close()
       except Exception:
         pass
-      retDict = S_ERROR(x)
+      retDict = S_ERROR(repr(x))
       retDict['Value'] = (-1, '', str(x))
       return retDict
 

{DIRAC-7.2.0a36 → DIRAC-7.2.0a39}/src/DIRAC/Core/scripts/dirac-install.py

@@ -1,8 +1,5 @@
 #!/usr/bin/env python
 """
-The main DIRAC installer script. It can be used to install the main DIRAC software, its
-modules, web, rest etc. and DIRAC extensions.
-
 See https://github.com/DIRACGrid/management/blob/master/dirac-install.py
 """
 import os
@@ -11,10 +8,10 @@ import sys
 import tempfile
 try:
   # For Python 3.0 and later
-  from urllib.request import urlopen, HTTPError, URLError
+  from urllib.request import urlopen
 except ImportError:
   # Fall back to Python 2's urllib2
-  from urllib2 import urlopen, HTTPError, URLError
+  from urllib2 import urlopen
 
 # Download dirac-install.py
 response = urlopen(
@@ -30,9 +27,12 @@ fp = os.fdopen(tmpHandle, "wb")
 fp.write(response.read())
 fp.close()
 
-# Make the dirac-install.py temporay file executable
+# Make the dirac-install.py temporary file executable
 st = os.stat(tmp)
 os.chmod(tmp, st.st_mode | stat.S_IEXEC)
 
+# Suppress Python warnings in dirac-install.py
+os.environ["PYTHONWARNINGS"] = "ignore"
+
 # Replace the current process with the actual dirac-install.py script
 os.execv(tmp, sys.argv)

{DIRAC-7.2.0a36 → DIRAC-7.2.0a39}/src/DIRAC/Core/scripts/install_site.sh

@@ -55,7 +55,7 @@ fi
 
 # Get the version of dirac-install requested - if none is requested, the version will come from integration
 #
-curl -L -o dirac-install "https://github.com/DIRACGrid/DIRAC/raw/${DIRACVERSION}/Core/scripts/dirac-install.py" || exit
+curl -L -o dirac-install https://raw.githubusercontent.com/DIRACGrid/management/master/dirac-install.py || exit
 #
 # define the target Dir
 #

{DIRAC-7.2.0a36 → DIRAC-7.2.0a39}/src/DIRAC/DataManagementSystem/Client/FTS3Job.py

@@ -413,6 +413,7 @@ class FTS3Job(JSerializable):
                        bring_online=bring_online,
                        copy_pin_lifetime=copy_pin_lifetime,
                        retry=3,
+                       verify_checksum='target',  # Only check target vs specified, since we verify the source earlier
                        multihop=bool(allStageURLs),  # if we have stage urls, then we need multihop
                        metadata=job_metadata,
                        priority=self.priority)

{DIRAC-7.2.0a36 → DIRAC-7.2.0a39}/src/DIRAC/DataManagementSystem/ConfigTemplate.cfg

@@ -102,18 +102,6 @@ Services
       }
     }
   }
-  IRODSStorageElement
-  {
-    Port = 9188
-    Authorization
-    {
-      Default = authenticated
-      FileTransfer
-      {
-        Default = authenticated
-      }
-    }
-  }
   ##BEGIN S3Gateway
   S3Gateway
   {

{DIRAC-7.2.0a36 → DIRAC-7.2.0a39}/src/DIRAC/DataManagementSystem/scripts/dirac_dms_directory_sync.py

@@ -28,12 +28,13 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from past.builtins import long
 import os
 import DIRAC
 from DIRAC.Core.Base import Script
 from DIRAC.Core.Utilities.DIRACScript import DIRACScript
 
+__RCSID__ = "$Id$"
+
 
 @DIRACScript()
 def main():
@@ -62,8 +63,6 @@ def main():
   from DIRAC.DataManagementSystem.Client.DataManager import DataManager
   from DIRAC.Resources.Storage.StorageElement import StorageElement
 
-  __RCSID__ = "$Id$"
-
   def getSetOfLocalDirectoriesAndFiles(path):
     """Return a set of all directories and subdirectories and a set of
     files contained therein for a given local path
@@ -91,7 +90,7 @@ def main():
         fullFilename = fullFilename.replace(fullPath, '').lstrip('/')
         fileSize = os.path.getsize(fullPath + "/" + fullFilename)
         if fileSize > 0:
-          files.add((fullFilename, long(fileSize)))
+          files.add((fullFilename, int(fileSize)))
 
     tree = {}
     tree["Directories"] = directories

{DIRAC-7.2.0a36 → DIRAC-7.2.0a39}/src/DIRAC/DataManagementSystem/scripts/dirac_dms_protocol_matrix.py

@@ -45,16 +45,18 @@ from __future__ import print_function
 import csv
 from collections import defaultdict
 
-from DIRAC.Core.Base import Script
 from DIRAC.Core.Utilities.DIRACScript import DIRACScript
 
-
 @DIRACScript()
 def main():
+  from DIRAC.Core.Base import Script
   Script.registerSwitch('', 'FromSE=', 'SE1[,SE2,...]')
   Script.registerSwitch('', 'TargetSE=', 'SE1[,SE2,...]')
   Script.registerSwitch('', 'OutputFile=', 'CSV output file (default /tmp/protocol-matrix.csv)')
   Script.registerSwitch('', 'Bidirection', 'If FromSE or TargetSE are specified, make a square matrix ')
+  Script.registerSwitch('', 'FTSOnly', 'Only display the protocols sent to FTS')
+  Script.registerSwitch('', 'ExcludeSE=', 'SEs to not take into account for the matrix')
+
   from DIRAC.Core.Base.Script import parseCommandLine
   parseCommandLine()
   from DIRAC import gConfig, gLogger
@@ -64,17 +66,23 @@ def main():
 
   fromSE = []
   targetSE = []
+  excludeSE = []
   outputFile = '/tmp/protocol-matrix.csv'
   bidirection = False
+  ftsOnly = False
   for switch in Script.getUnprocessedSwitches():
     if switch[0] == 'FromSE':
       fromSE = switch[1].split(',')
     elif switch[0] == 'TargetSE':
       targetSE = switch[1].split(',')
+    elif switch[0] == 'ExcludeSE':
+      excludeSE = switch[1].split(',')
     elif switch[0] == 'OutputFile':
       outputFile = switch[1]
     elif switch[0] == 'Bidirection':
       bidirection = True
+    elif switch[0] == 'FTSOnly':
+      ftsOnly = True
 
   thirdPartyProtocols = DMSHelpers().getThirdPartyProtocols()
 
@@ -85,6 +93,9 @@ def main():
 
   allSEs = gConfig.getSections('/Resources/StorageElements/')['Value']
 
+  # Remove the SEs that we want to exclude
+  allSEs = set(allSEs) - set(excludeSE)
+
   # We go through all the SEs and fill in the seForSEBases dict.
   # Basically, at the end of the loop, the dict will contain
   # for each baseSE an entry corresponding to one real storage (the first one)
@@ -131,7 +142,7 @@ def main():
   # Now we construct the SE object for each SE that we want to appear
   ses = {}
   for se in set(fromSE + targetSE):
-    ses[se] = StorageElement(seForSeBases[se])
+    ses[se] = StorageElement(seForSeBases.get(se, se))
 
   ret = getVOfromProxyGroup()
   if not ret['OK'] or not ret.get('Value', ''):
@@ -158,7 +169,10 @@ def main():
 
     # Add also the third party protocols
     proto = ','.join(ses[dst].negociateProtocolWithOtherSE(ses[src], thirdPartyProtocols)['Value'])
-    tpMatrix[src][dst] = '%s (%s)' % (surls, proto)
+    if ftsOnly:
+      tpMatrix[src][dst] = '%s' % surls
+    else:
+      tpMatrix[src][dst] = '%s (%s)' % (surls, proto)
     gLogger.verbose("%s -> %s: %s" % (src, dst, surls))
     gLogger.verbose("%s -> %s: %s" % (src, dst, proto))