panda-server 0.3.1__tar.gz → 0.3.3__tar.gz

{panda_server-0.3.1 → panda_server-0.3.3}/Dockerfile

@@ -8,7 +8,7 @@ RUN yum update -y
 RUN yum install -y epel-release
 
 RUN yum install -y httpd httpd-devel gcc gridsite git psmisc less wget logrotate procps which \
-    openssl-devel readline-devel bzip2-devel libffi-devel zlib-devel
+    openssl-devel readline-devel bzip2-devel libffi-devel zlib-devel systemd-udev
 
 # install python
 RUN mkdir /tmp/python && cd /tmp/python && \
@@ -25,6 +25,11 @@ RUN yum install -y https://download.postgresql.org/pub/repos/yum/reporpms/EL-9-x
 RUN yum install --nogpgcheck -y postgresql16
 RUN  yum clean all && rm -rf /var/cache/yum
 
+# update network limitations
+# RUN echo 4096 > /proc/sys/net/core/somaxconn
+# RUN sysctl -w net.core.somaxconn=4096
+RUN echo 'net.core.somaxconn=4096' >> /etc/sysctl.d/999-net.somax.conf
+
 # setup venv with pythonX.Y
 RUN python$(echo ${PYTHON_VERSION} | sed -E 's/\.[0-9]+$//') -m venv /opt/panda
 RUN /opt/panda/bin/pip install --no-cache-dir -U pip

{panda_server-0.3.1 → panda_server-0.3.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: panda-server
-Version: 0.3.1
+Version: 0.3.3
 Summary: PanDA Server Package
 Project-URL: Homepage, https://panda-wms.readthedocs.io/en/latest/
 Author-email: PanDA Team <panda-support@cern.ch>

panda_server-0.3.3/PandaPkgInfo.py

@@ -0,0 +1 @@
+release_version = "0.3.3"

{panda_server-0.3.1 → panda_server-0.3.3}/pandaserver/brokerage/broker.py

@@ -12,7 +12,7 @@ from pandaserver.brokerage import ErrorCode
 from pandaserver.config import panda_config
 from pandaserver.dataservice import DataServiceUtils
 from pandaserver.dataservice.DataServiceUtils import select_scope
-from pandaserver.dataservice.DDM import rucioAPI
+from pandaserver.dataservice.ddm import rucioAPI
 from pandaserver.taskbuffer import ProcessGroups
 
 _log = PandaLogger().getLogger("broker")
@@ -99,7 +99,7 @@ def _getOkFiles(
         # get all replicas
         if rucio_url not in allOkFilesMap:
             allOkFilesMap[rucio_url] = {}
-            tmpStat, tmpAvaFiles = rucioAPI.listFileReplicas(allScopeList, allLFNs, tmpSE)
+            tmpStat, tmpAvaFiles = rucioAPI.list_file_replicas(allScopeList, allLFNs, tmpSE)
             if not tmpStat and tmpLog is not None:
                 tmpLog.debug("getOkFile failed to get file replicas")
                 tmpAvaFiles = {}

{panda_server-0.3.1 → panda_server-0.3.3}/pandaserver/daemons/scripts/datasetManager.py

@@ -14,7 +14,7 @@ from pandaserver.config import panda_config
 from pandaserver.dataservice import DataServiceUtils
 from pandaserver.dataservice.closer import Closer
 from pandaserver.dataservice.DataServiceUtils import select_scope
-from pandaserver.dataservice.DDM import rucioAPI
+from pandaserver.dataservice.ddm import rucioAPI
 from pandaserver.dataservice.finisher import Finisher
 from pandaserver.taskbuffer import EventServiceUtils
 
@@ -198,11 +198,11 @@ def main(tbuf=None, **kwargs):
                         dsExists = False
                     if dsExists:
                         # check if dataset exists
-                        status, out = rucioAPI.getMetaData(name)
+                        status, out = rucioAPI.get_metadata(name)
                         if status is True:
                             if out is not None:
                                 try:
-                                    rucioAPI.closeDataset(name)
+                                    rucioAPI.close_dataset(name)
                                     status = True
                                 except Exception:
                                     errtype, errvalue = sys.exc_info()[:2]
@@ -233,7 +233,7 @@ def main(tbuf=None, **kwargs):
                         if not dsExists:
                             continue
                         # count # of files
-                        status, out = rucioAPI.getNumberOfFiles(name)
+                        status, out = rucioAPI.get_number_of_files(name)
                         if status is not True:
                             if status is False:
                                 _logger.error(out)
@@ -244,7 +244,7 @@ def main(tbuf=None, **kwargs):
                                 if nFile == 0:
                                     # erase dataset
                                     _logger.debug(f"erase {name}")
-                                    status, out = rucioAPI.eraseDataset(name)
+                                    status, out = rucioAPI.erase_dataset(name)
                                     _logger.debug(f"OK with {name}")
                             except Exception:
                                 pass
@@ -401,11 +401,11 @@ def main(tbuf=None, **kwargs):
                                 status, out = True, ""
                             elif dsExists:
                                 # check if dataset exists
-                                status, out = rucioAPI.getMetaData(name)
+                                status, out = rucioAPI.get_metadata(name)
                                 if status is True:
                                     if out is not None:
                                         try:
-                                            rucioAPI.closeDataset(name)
+                                            rucioAPI.close_dataset(name)
                                             status = True
                                         except Exception:
                                             errtype, errvalue = sys.exc_info()[:2]
@@ -434,7 +434,7 @@ def main(tbuf=None, **kwargs):
                                 # set tobedeleted to dis
                                 setTobeDeletedToDis(name)
                                 # count # of files
-                                status, out = rucioAPI.getNumberOfFiles(name)
+                                status, out = rucioAPI.get_number_of_files(name)
                                 if status is not True:
                                     if status is False:
                                         _logger.error(out)
@@ -446,7 +446,7 @@ def main(tbuf=None, **kwargs):
                                         if nFile == 0:
                                             # erase dataset
                                             _logger.debug(f"erase {name}")
-                                            status, out = rucioAPI.eraseDataset(name)
+                                            status, out = rucioAPI.erase_dataset(name)
                                             _logger.debug(f"OK with {name}")
                                     except Exception:
                                         pass
@@ -567,7 +567,7 @@ def main(tbuf=None, **kwargs):
                     _logger.debug(f"Eraser {self.operationType} dis {modDate} {name}")
                     # delete or shorten
                     endStatus = "deleted"
-                    status, out = rucioAPI.eraseDataset(name)
+                    status, out = rucioAPI.erase_dataset(name)
                     if not status:
                         _logger.error(out)
                         continue
@@ -699,7 +699,7 @@ def main(tbuf=None, **kwargs):
                             nTokens += len(file.destinationDBlockToken.split(","))
                     # get files in LRC
                     _logger.debug(f"{job.PandaID} Cloud:{job.cloud}")
-                    tmpStat, okFiles = rucioAPI.listFileReplicas(scopes, lfns, seList)
+                    tmpStat, okFiles = rucioAPI.list_file_replicas(scopes, lfns, seList)
                     if not tmpStat:
                         _logger.error(f"{job.PandaID} failed to get file replicas")
                         okFiles = {}
@@ -852,7 +852,7 @@ def main(tbuf=None, **kwargs):
                             scopes.append(tmpFile.scope)
                     # get file replicas
                     _logger.debug(f"{tmpJob.PandaID} check input files at {tmpJob.computingSite}")
-                    tmpStat, okFiles = rucioAPI.listFileReplicas(scopes, lfns)
+                    tmpStat, okFiles = rucioAPI.list_file_replicas(scopes, lfns)
                     if not tmpStat:
                         pass
                     else:
@@ -955,7 +955,7 @@ def main(tbuf=None, **kwargs):
                         if tmpFile.type == "input" and tmpFile.status != "ready":
                             # get replicas
                             if tmpFile.dispatchDBlock not in replicaMap:
-                                tmpStat, repMap = rucioAPI.listDatasetReplicas(tmpFile.dispatchDBlock)
+                                tmpStat, repMap = rucioAPI.list_dataset_replicas(tmpFile.dispatchDBlock)
                                 if tmpStat != 0:
                                     repMap = {}
                                 replicaMap[tmpFile.dispatchDBlock] = repMap
@@ -1088,7 +1088,7 @@ def main(tbuf=None, **kwargs):
                             if allDone:
                                 _logger.debug(f"deleting sub {name}")
                                 try:
-                                    rucioAPI.eraseDataset(name, grace_period=4)
+                                    rucioAPI.erase_dataset(name, grace_period=4)
                                     status = True
                                 except Exception:
                                     errtype, errvalue = sys.exc_info()[:2]

{panda_server-0.3.1 → panda_server-0.3.3}/pandaserver/daemons/scripts/evpPD2P.py

@@ -9,7 +9,7 @@ from pandacommon.pandalogger.PandaLogger import PandaLogger
 from pandacommon.pandautils.thread_utils import GenericThread
 from pandaserver.brokerage import SiteMapper
 from pandaserver.config import panda_config
-from pandaserver.dataservice.EventPicker import EventPicker
+from pandaserver.dataservice.event_picker import EventPicker
 
 # logger
 _logger = PandaLogger().getLogger("evpPD2P")

{panda_server-0.3.1 → panda_server-0.3.3}/pandaserver/dataservice/AdderAtlasPlugin.py

@@ -15,7 +15,7 @@ from pandaserver.config import panda_config
 from pandaserver.dataservice import DataServiceUtils, ErrorCode
 from pandaserver.dataservice.AdderPluginBase import AdderPluginBase
 from pandaserver.dataservice.DataServiceUtils import select_scope
-from pandaserver.dataservice.DDM import rucioAPI
+from pandaserver.dataservice.ddm import rucioAPI
 from pandaserver.srvcore.MailUtils import MailUtils
 from pandaserver.taskbuffer import EventServiceUtils, JobUtils
 from rucio.common.exception import (
@@ -616,9 +616,9 @@ class AdderAtlasPlugin(AdderPluginBase):
                     regMsgStr = f"File registration with backend={self.ddmBackEnd} for {regNumFiles} files "
                 if len(zipFiles) > 0:
                     self.logger.debug(f"registerZipFiles {str(zipFiles)}")
-                    rucioAPI.registerZipFiles(zipFiles)
+                    rucioAPI.register_zip_files(zipFiles)
                 self.logger.debug(f"registerFilesInDatasets {str(destIdMap)} zip={str(contZipMap)}")
-                out = rucioAPI.registerFilesInDataset(destIdMap, contZipMap)
+                out = rucioAPI.register_files_in_dataset(destIdMap, contZipMap)
             except (
                 DataIdentifierNotFound,
                 FileConsistencyMismatch,
@@ -709,7 +709,7 @@ class AdderAtlasPlugin(AdderPluginBase):
                         for iDDMTry in range(3):
                             isFailed = False
                             try:
-                                status = rucioAPI.registerDatasetSubscription(
+                                status = rucioAPI.register_dataset_subscription(
                                     tmpName,
                                     [dq2ID],
                                     owner="panda",
@@ -763,7 +763,7 @@ class AdderAtlasPlugin(AdderPluginBase):
                                 out = "OK"
                                 isFailed = False
                                 try:
-                                    rucioAPI.registerDatasetLocation(
+                                    rucioAPI.register_dataset_location(
                                         tmpDsNameLoc,
                                         [tmpLocName],
                                         owner="panda",
@@ -854,7 +854,7 @@ class AdderAtlasPlugin(AdderPluginBase):
                                 tmpDN = userInfo["nickname"]
                             tmpMsg = f"registerDatasetLocation for Rucio ds={tmpDsName} site={tmpDQ2ID} id={tmpDN}"
                             self.logger.debug(tmpMsg)
-                            rucioAPI.registerDatasetLocation(
+                            rucioAPI.register_dataset_location(
                                 tmpDsName,
                                 [tmpDQ2ID],
                                 owner=tmpDN,
@@ -997,7 +997,7 @@ class AdderAtlasPlugin(AdderPluginBase):
             if idMap != {}:
                 self.logger.debug(f"adding ES files {str(idMap)}")
                 try:
-                    rucioAPI.registerFilesInDataset(idMap)
+                    rucioAPI.register_files_in_dataset(idMap)
                 except DataIdentifierNotFound:
                     self.logger.debug("ignored DataIdentifierNotFound")
         except Exception:

{panda_server-0.3.1 → panda_server-0.3.3}/pandaserver/dataservice/AdderSimplePlugin.py

@@ -6,7 +6,7 @@ import traceback
 import uuid
 
 from pandaserver.dataservice import DataServiceUtils, ErrorCode
-from pandaserver.dataservice.DDM import rucioAPI
+from pandaserver.dataservice.ddm import rucioAPI
 from rucio.common.exception import (
     DataIdentifierNotFound,
     FileConsistencyMismatch,
@@ -73,7 +73,7 @@ class AdderSimplePlugin(AdderPluginBase):
                     regStart = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None)
                     try:
                         self.logger.debug(f"registerFilesInDatasets {str(destIdMap)}")
-                        out = rucioAPI.registerFilesInDataset(destIdMap, {})
+                        out = rucioAPI.register_files_in_dataset(destIdMap, {})
                     except (
                         DataIdentifierNotFound,
                         FileConsistencyMismatch,

{panda_server-0.3.1 → panda_server-0.3.3}/pandaserver/dataservice/RecoverLostFilesCore.py

@@ -3,7 +3,7 @@ import sys
 
 from pandacommon.pandautils.thread_utils import GenericThread
 from pandaserver.config import panda_config
-from pandaserver.dataservice.DDM import rucioAPI
+from pandaserver.dataservice.ddm import rucioAPI
 from pandaserver.userinterface import Client
 from rucio.client import Client as RucioClient
 from rucio.common.exception import DataIdentifierNotFound

{panda_server-0.3.1 → panda_server-0.3.3}/pandaserver/dataservice/SetupperAtlasPlugin.py

@@ -15,7 +15,7 @@ from pandaserver.brokerage.SiteMapper import SiteMapper
 from pandaserver.config import panda_config
 from pandaserver.dataservice import DataServiceUtils, ErrorCode
 from pandaserver.dataservice.DataServiceUtils import select_scope
-from pandaserver.dataservice.DDM import rucioAPI
+from pandaserver.dataservice.ddm import rucioAPI
 from pandaserver.dataservice.SetupperPluginBase import SetupperPluginBase
 from pandaserver.taskbuffer import EventServiceUtils, JobUtils
 from pandaserver.taskbuffer.DatasetSpec import DatasetSpec
@@ -226,7 +226,7 @@ class SetupperAtlasPlugin(SetupperPluginBase):
                     self.logger.debug("listDatasets " + job.prodDBlock)
                     prodError[job.prodDBlock] = ""
                     for iDDMTry in range(3):
-                        newOut, errMsg = rucioAPI.listDatasets(job.prodDBlock)
+                        newOut, errMsg = rucioAPI.list_datasets(job.prodDBlock)
                         if newOut is None:
                             time.sleep(10)
                         else:
@@ -352,7 +352,7 @@ class SetupperAtlasPlugin(SetupperPluginBase):
                     disFiles = fileList[dispatchDBlock]
                 else:
                     dids = fileList[dispatchDBlock]["lfns"]
-                    tmpZipStat, tmpZipOut = rucioAPI.getZipFiles(dids, None)
+                    tmpZipStat, tmpZipOut = rucioAPI.get_zip_files(dids, None)
                     if not tmpZipStat:
                         self.logger.debug(f"failed to get zip files : {tmpZipOut}")
                         tmpZipOut = {}
@@ -391,7 +391,7 @@ class SetupperAtlasPlugin(SetupperPluginBase):
                 errStr = ""
                 for iDDMTry in range(nDDMTry):
                     try:
-                        out = rucioAPI.registerDataset(
+                        out = rucioAPI.register_dataset(
                             dispatchDBlock,
                             disFiles["lfns"],
                             disFiles["guids"],
@@ -421,7 +421,7 @@ class SetupperAtlasPlugin(SetupperPluginBase):
                 for iDDMTry in range(3):
                     status = False
                     try:
-                        rucioAPI.closeDataset(dispatchDBlock)
+                        rucioAPI.close_dataset(dispatchDBlock)
                         status = True
                         break
                     except Exception:
@@ -632,7 +632,7 @@ class SetupperAtlasPlugin(SetupperPluginBase):
                                 isOK = False
                                 for iDDMTry in range(3):
                                     try:
-                                        out = rucioAPI.registerDataset(
+                                        out = rucioAPI.register_dataset(
                                             name,
                                             metadata=tmpMetadata,
                                             lifetime=tmpLifeTime,
@@ -702,7 +702,7 @@ class SetupperAtlasPlugin(SetupperPluginBase):
                                             break
                                         for iDDMTry in range(3):
                                             try:
-                                                out = rucioAPI.registerDatasetLocation(
+                                                out = rucioAPI.register_dataset_location(
                                                     name,
                                                     [dq2ID],
                                                     lifetime=repLifeTime,
@@ -735,7 +735,7 @@ class SetupperAtlasPlugin(SetupperPluginBase):
                         if newVUID is None:
                             self.logger.debug("listDatasets " + name)
                             for iDDMTry in range(3):
-                                newOut, errMsg = rucioAPI.listDatasets(name)
+                                newOut, errMsg = rucioAPI.list_datasets(name)
                                 if newOut is None:
                                     time.sleep(10)
                                 else:
@@ -1021,12 +1021,12 @@ class SetupperAtlasPlugin(SetupperPluginBase):
                             )
                             for iDDMTry in range(3):
                                 try:
-                                    status = rucioAPI.registerDatasetSubscription(
+                                    status = rucioAPI.register_dataset_subscription(
                                         job.dispatchDBlock,
                                         [dq2ID],
                                         activity=optActivity,
                                         lifetime=7,
-                                        dn=optOwner,
+                                        distinguished_name=optOwner,
                                         comment=optComment,
                                     )
                                     out = "OK"
@@ -1439,7 +1439,7 @@ class SetupperAtlasPlugin(SetupperPluginBase):
         for iDDMTry in range(3):
             try:
                 self.logger.debug("listFilesInDataset " + dataset)
-                items, tmpDummy = rucioAPI.listFilesInDataset(dataset, fileList=fileList)
+                items, tmpDummy = rucioAPI.list_files_in_dataset(dataset, file_list=fileList)
                 status = 0
                 break
             except DataIdentifierNotFound:
@@ -1460,7 +1460,7 @@ class SetupperAtlasPlugin(SetupperPluginBase):
         # get datasets in container
         self.logger.debug("listDatasetsInContainer " + container)
         for iDDMTry in range(3):
-            datasets, out = rucioAPI.listDatasetsInContainer(container)
+            datasets, out = rucioAPI.list_datasets_in_container(container)
             if datasets is None:
                 time.sleep(10)
             else:
@@ -1474,7 +1474,7 @@ class SetupperAtlasPlugin(SetupperPluginBase):
         # get datasets in container
         self.logger.debug("listDatasetsInContainer " + container)
         for iDDMTry in range(3):
-            datasets, out = rucioAPI.listDatasetsInContainer(container)
+            datasets, out = rucioAPI.list_datasets_in_container(container)
             if datasets is None:
                 time.sleep(10)
             else:
@@ -1534,7 +1534,7 @@ class SetupperAtlasPlugin(SetupperPluginBase):
         nTry = 3
         for iDDMTry in range(nTry):
             self.logger.debug(f"{iDDMTry}/{nTry} listDatasetReplicas {dataset}")
-            status, out = rucioAPI.listDatasetReplicas(dataset)
+            status, out = rucioAPI.list_dataset_replicas(dataset)
             if status != 0:
                 time.sleep(10)
             else:
@@ -1699,7 +1699,7 @@ class SetupperAtlasPlugin(SetupperPluginBase):
                         tmpZipOut = {}
                         if tmpJob.useZipToPin():
                             dids = [tmpFileList[tmpSubFileName]["lfn"] for tmpSubFileName in subFileNames]
-                            tmpZipStat, tmpZipOut = rucioAPI.getZipFiles(dids, [tmpLocation])
+                            tmpZipStat, tmpZipOut = rucioAPI.get_zip_files(dids, [tmpLocation])
                             if not tmpZipStat:
                                 self.logger.debug(f"failed to get zip files : {tmpZipOut}")
                                 tmpZipOut = {}
@@ -1741,7 +1741,7 @@ class SetupperAtlasPlugin(SetupperPluginBase):
                         )
                         for iDDMTry in range(nDDMTry):
                             try:
-                                out = rucioAPI.registerDataset(
+                                out = rucioAPI.register_dataset(
                                     disDBlock,
                                     lfns,
                                     guids,
@@ -1785,7 +1785,7 @@ class SetupperAtlasPlugin(SetupperPluginBase):
                         for iDDMTry in range(3):
                             status = False
                             try:
-                                rucioAPI.closeDataset(disDBlock)
+                                rucioAPI.close_dataset(disDBlock)
                                 status = True
                                 break
                             except Exception:
@@ -1801,13 +1801,12 @@ class SetupperAtlasPlugin(SetupperPluginBase):
                         nDDMTry = 3
                         for iDDMTry in range(nDDMTry):
                             try:
-                                out = rucioAPI.registerDatasetLocation(
+                                out = rucioAPI.register_dataset_location(
                                     disDBlock,
                                     [tmpLocation],
                                     7,
                                     activity="Production Input",
                                     scope="panda",
-                                    asynchronous=True,
                                     grouping="NONE",
                                 )
                                 self.logger.debug(out)
@@ -1995,7 +1994,7 @@ class SetupperAtlasPlugin(SetupperPluginBase):
         for iDDMTry in range(nTry):
             try:
                 # register subscription
-                status = rucioAPI.registerDatasetSubscription(dataset, [dq2ID], activity="Production Input")
+                status = rucioAPI.register_dataset_subscription(dataset, [dq2ID], activity="Production Input")
                 out = "OK"
                 break
             except Exception:
@@ -2089,9 +2088,9 @@ class SetupperAtlasPlugin(SetupperPluginBase):
                 # register dis dataset
                 try:
                     self.logger.debug(f"registering jumbo dis dataset {dispatchDBlock} with {len(lfns)} files")
-                    out = rucioAPI.registerDataset(dispatchDBlock, lfns, guids, sizes, checksums, lifetime=14)
+                    out = rucioAPI.register_dataset(dispatchDBlock, lfns, guids, sizes, checksums, lifetime=14)
                     vuid = out["vuid"]
-                    rucioAPI.closeDataset(dispatchDBlock)
+                    rucioAPI.close_dataset(dispatchDBlock)
                 except Exception:
                     errType, errValue = sys.exc_info()[:2]
                     self.logger.debug(f"failed to register jumbo dis dataset {dispatchDBlock} with {errType}:{errValue}")
@@ -2110,7 +2109,7 @@ class SetupperAtlasPlugin(SetupperPluginBase):
                     )
                     endPoint = tmpSiteSpec.ddm_input[scope_input]
                     self.logger.debug(f"subscribing jumbo dis dataset {dispatchDBlock} to {endPoint}")
-                    rucioAPI.registerDatasetSubscription(
+                    rucioAPI.register_dataset_subscription(
                         dispatchDBlock,
                         [endPoint],
                         lifetime=14,

{panda_server-0.3.1 → panda_server-0.3.3}/pandaserver/dataservice/closer.py

@@ -3,18 +3,17 @@ update dataset DB, and then close dataset and start Activator if needed
 
 """
 
-import sys
 import datetime
-
+import sys
 from typing import Dict, List
 
-from pandacommon.pandalogger.PandaLogger import PandaLogger
 from pandacommon.pandalogger.LogWrapper import LogWrapper
+from pandacommon.pandalogger.PandaLogger import PandaLogger
 
 from pandaserver.config import panda_config
+from pandaserver.dataservice import DataServiceUtils
 from pandaserver.dataservice.activator import Activator
 from pandaserver.taskbuffer import EventServiceUtils
-from pandaserver.dataservice import DataServiceUtils
 
 # logger
 _logger = PandaLogger().getLogger("closer")
@@ -30,8 +29,7 @@ class Closer:
     """
 
     # constructor
-    def __init__(self, taskBuffer, destination_data_blocks: List[str], job,
-                 dataset_map: Dict = None) -> None:
+    def __init__(self, taskBuffer, destination_data_blocks: List[str], job, dataset_map: Dict = None) -> None:
         """
         Constructor
 
@@ -56,8 +54,7 @@ class Closer:
         Returns:
             bool: True if all sub datasets are done, False otherwise.
         """
-        tmp_log = LogWrapper(_logger,
-                             f"check_sub_datasets_in_jobset-{datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None).isoformat('/')}")
+        tmp_log = LogWrapper(_logger, f"check_sub_datasets_in_jobset-{datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None).isoformat('/')}")
         # skip already checked
         if self.all_subscription_finished is not None:
             return self.all_subscription_finished
@@ -81,8 +78,7 @@ class Closer:
                 # count the number of unfinished
                 not_finish = self.task_buffer.countFilesWithMap({"destinationDBlock": sub_dataset, "status": "unknown"})
                 if not_finish != 0:
-                    tmp_log.debug(
-                        f"related sub dataset {sub_dataset} from {job_spec.PandaID} has {not_finish} unfinished files")
+                    tmp_log.debug(f"related sub dataset {sub_dataset} from {job_spec.PandaID} has {not_finish} unfinished files")
                     self.all_subscription_finished = False
                     break
         if self.all_subscription_finished is None:
@@ -123,9 +119,7 @@ class Closer:
         closer_plugin_class = panda_config.getPlugin("closer_plugins", self.job.VO)
         if closer_plugin_class is None and self.job.VO == "atlas":
             # use ATLAS plugin for ATLAS
-            from pandaserver.dataservice.closer_atlas_plugin import (
-                CloserAtlasPlugin,
-            )
+            from pandaserver.dataservice.closer_atlas_plugin import CloserAtlasPlugin
 
             closer_plugin_class = CloserAtlasPlugin
         if closer_plugin_class is not None:
@@ -141,9 +135,11 @@ class Closer:
             final_status (str): Final status.
         """
         # start Activator
-        if not DataServiceUtils.is_sub_dataset(
-                dataset.name) and self.job.jobStatus == "finished" and self.job.prodSourceLabel != "panda" and self.job.processingType not in [
-            "merge", "unmerge"]:
+        if (
+            not DataServiceUtils.is_sub_dataset(dataset.name)
+            and self.job.jobStatus == "finished"
+            and (self.job.prodSourceLabel != "panda" or self.job.processingType not in ["merge", "unmerge"])
+        ):
             activator_thread = Activator(self.task_buffer, dataset)
             activator_thread.run()
 
@@ -154,8 +150,7 @@ class Closer:
         updates the dataset status and finalizes pending jobs if necessary.
         """
         try:
-            tmp_log = LogWrapper(_logger,
-                                 f"run-{datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None).isoformat('/')}-{self.panda_id}")
+            tmp_log = LogWrapper(_logger, f"run-{datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None).isoformat('/')}-{self.panda_id}")
             tmp_log.debug(f"Start with job status: {self.job.jobStatus}")
             flag_complete = True
             final_status_dataset = []
@@ -170,10 +165,8 @@ class Closer:
                     continue
 
                 # ignore HC datasets
-                if (DataServiceUtils.is_hammercloud_dataset(destination_data_block) or
-                        DataServiceUtils.is_user_gangarbt_dataset(destination_data_block)):
-                    if (not DataServiceUtils.is_sub_dataset(destination_data_block) and
-                            not DataServiceUtils.is_lib_dataset(destination_data_block)):
+                if DataServiceUtils.is_hammercloud_dataset(destination_data_block) or DataServiceUtils.is_user_gangarbt_dataset(destination_data_block):
+                    if not DataServiceUtils.is_sub_dataset(destination_data_block) and not DataServiceUtils.is_lib_dataset(destination_data_block):
                         tmp_log.debug(f"skip HC {destination_data_block}")
                         continue
 
@@ -197,8 +190,7 @@ class Closer:
                 dataset_list.sort()
 
                 # count number of completed files
-                not_finish = self.task_buffer.countFilesWithMap(
-                    {"destinationDBlock": destination_data_block, "status": "unknown"})
+                not_finish = self.task_buffer.countFilesWithMap({"destinationDBlock": destination_data_block, "status": "unknown"})
                 if not_finish < 0:
                     tmp_log.error(f"Invalid dispatch block file count: {not_finish}")
                     flag_complete = False

{panda_server-0.3.1 → panda_server-0.3.3}/pandaserver/dataservice/closer_atlas_plugin.py

@@ -4,7 +4,7 @@ ATLAS plugin for closer
 import sys
 
 from pandacommon.pandalogger.LogWrapper import LogWrapper
-from pandaserver.dataservice.DDM import rucioAPI
+from pandaserver.dataservice.ddm import rucioAPI
 from pandaserver.dataservice import DataServiceUtils
 
 
@@ -80,7 +80,7 @@ class CloserAtlasPlugin:
                     continue
                 try:
                     self.tmp_log.debug(f"immediate close {datasetSpec.name}")
-                    rucioAPI.closeDataset(datasetSpec.name)
+                    rucioAPI.close_dataset(datasetSpec.name)
                 except Exception:
                     errtype, errvalue = sys.exc_info()[:2]
                     self.tmp_log.warning(f"failed to close : {errtype} {errvalue}")