panda-server-source 0.8.0__py3-none-any.whl

pandaserver/dataservice/adder_plugin_base.py

@@ -0,0 +1,27 @@
+"""
+The Adder module is the core for add_main to post-process jobs’ output data, such as data registration, trigger data aggregation and so on.
+Those post-processing procedures are experiment-dependent so that the Adder also has a plugin structure to load an experiment-specific plugin.
+
+"""
+
+from .adder_result import AdderResult
+
+
+class AdderPluginBase:
+    """
+    Base class for Adder plugins.
+    """
+
+    def __init__(self, job, params):
+        """
+        Initialize the AdderPluginBase.
+
+        :param job: The job object.
+        :param params: Additional parameters.
+        """
+        self.job = job
+        self.logger = None
+        self.result = AdderResult()
+        self.extra_info = {}
+        for key, value in params.items():
+            setattr(self, key, value)

pandaserver/dataservice/adder_result.py

@@ -0,0 +1,75 @@
+"""
+Class to represent the result of the Adder operation
+
+"""
+
+class AdderResult:
+    """
+    Class to represent the result of the Adder operation.
+    """
+    # slot
+    __slot__ = ("statusCode", "transferringFiles")
+
+    # return code
+    rc_succeeded = 0
+    rc_temporary = 1
+    rc_fatal = 2
+
+    # constructor
+    def __init__(self) -> None:
+        """
+        Initialize the AdderResult with default values.
+        """
+        # result of interactions with data management system (DMS)
+        self.status_code = None
+
+        # list of files which are being transferred by DMS
+        self.transferring_files = []
+
+        # list of files which are being merged
+        self.merging_files = []
+
+    # succeeded
+    def set_succeeded(self):
+        """
+        Set the status to succeeded.
+        """
+        self.status_code = self.rc_succeeded
+
+    # temporary error to retry later
+    def set_temporary(self):
+        """
+        Set the status to temporary error.
+        """
+        self.status_code = self.rc_temporary
+
+    # fatal error
+    def set_fatal(self):
+        """
+        Set the status to fatal.
+        """
+        self.status_code = self.rc_fatal
+
+    # check if succeeded
+    def is_succeeded(self):
+        """
+        Check if the status is 'succeeded'.
+        True if status is 'succeeded', False otherwise.
+        """
+        return self.status_code == self.rc_succeeded
+
+    # check if temporary
+    def is_temporary(self):
+        """
+        Check if the status is 'temporary'.
+        True if status is 'temporary', False otherwise.
+        """
+        return self.status_code == self.rc_temporary
+
+    # check if fatal error
+    def is_fatal(self):
+        """
+        Check if the status is 'fatal'.
+        True if status is 'fatal', False otherwise.
+        """
+        return self.status_code == self.rc_fatal

pandaserver/dataservice/adder_simple_plugin.py

@@ -0,0 +1,160 @@
+"""
+Simple plugin of Adder for VOs with Rucio
+
+"""
+
+import datetime
+import time
+import traceback
+import uuid
+
+from pandacommon.pandautils.PandaUtils import naive_utcnow
+from rucio.common.exception import (
+    DataIdentifierNotFound,
+    FileConsistencyMismatch,
+    InvalidObject,
+    InvalidPath,
+    InvalidRSEExpression,
+    RSEFileNameNotSupported,
+    RSENotFound,
+    RSEProtocolNotSupported,
+    UnsupportedOperation,
+)
+
+from pandaserver.dataservice import DataServiceUtils, ErrorCode
+from pandaserver.dataservice.ddm import rucioAPI
+
+from .adder_plugin_base import AdderPluginBase
+
+
+class AdderSimplePlugin(AdderPluginBase):
+    """
+    Simple plugin of Adder for VOs with Rucio.
+    """
+
+    # constructor
+    def __init__(self, job, **params) -> None:
+        """
+        Initialize the AdderSimplePlugin.
+
+        :param job: The job object.
+        :param params: Additional parameters.
+        """
+        AdderPluginBase.__init__(self, job, params)
+
+    # main
+    def execute(self) -> None:
+        """
+        Execute the simple adder plugin.
+
+        :return: None
+        """
+        try:
+            # loop over all files
+            file_map = {}
+            for file_spec in self.job.Files:
+                # ignore inputs
+                if file_spec.type not in ["output", "log"]:
+                    continue
+                # ignore local
+                if file_spec.destinationSE == "local":
+                    continue
+                # collect file attributes
+                try:
+                    file_size = int(file_spec.fsize)
+                except Exception:
+                    file_size = None
+                # set GUID if empty
+                if not file_spec.GUID:
+                    file_spec.GUID = str(uuid.uuid4())
+                file_attrs = {
+                    "guid": file_spec.GUID,
+                    "lfn": file_spec.lfn,
+                    "size": file_size,
+                    "checksum": file_spec.checksum,
+                    "ds": file_spec.destinationDBlock,
+                }
+                if self.extra_info:
+                    if "surl" in self.extra_info and file_spec.lfn in self.extra_info["surl"]:
+                        file_attrs["surl"] = self.extra_info["surl"][file_spec.lfn]
+                    if "nevents" in self.extra_info and file_spec.lfn in self.extra_info["nevents"]:
+                        file_attrs["events"] = self.extra_info["nevents"][file_spec.lfn]
+                file_map.setdefault(file_spec.destinationDBlock, [])
+                file_map[file_spec.destinationDBlock].append(file_attrs)
+            # register files
+            if file_map:
+                destination_rse = self.siteMapper.getSite(self.job.computingSite).ddm_output["default"]
+                destination_id_map = {destination_rse: file_map}
+                max_attempt = 3
+                for attempt_number in range(max_attempt):
+                    is_fatal = False
+                    is_failed = False
+                    registration_start = naive_utcnow()
+                    try:
+                        self.logger.debug(f"registerFilesInDatasets {str(destination_id_map)}")
+                        out = rucioAPI.register_files_in_dataset(destination_id_map, {})
+                    except (
+                        DataIdentifierNotFound,
+                        FileConsistencyMismatch,
+                        UnsupportedOperation,
+                        InvalidPath,
+                        InvalidObject,
+                        RSENotFound,
+                        RSEProtocolNotSupported,
+                        InvalidRSEExpression,
+                        RSEFileNameNotSupported,
+                        KeyError,
+                    ) as e:
+                        # fatal errors
+                        out = f"failed with {str(e)}\n {traceback.format_exc()}"
+                        is_fatal = True
+                        is_failed = True
+                    except Exception as e:
+                        # unknown errors
+                        is_failed = True
+                        out = f"failed with unknown error: {str(e)}\n {traceback.format_exc()}"
+                        if (
+                            "value too large for column" in out
+                            or "unique constraint (ATLAS_RUCIO.DIDS_GUID_IDX) violate" in out
+                            or "unique constraint (ATLAS_RUCIO.DIDS_PK) violated" in out
+                            or "unique constraint (ATLAS_RUCIO.ARCH_CONTENTS_PK) violated" in out
+                        ):
+                            is_fatal = True
+                        else:
+                            is_fatal = False
+                    registration_time = naive_utcnow() - registration_start
+                    self.logger.debug("took %s.%03d sec" % (registration_time.seconds, registration_time.microseconds / 1000))
+
+                    # failed
+                    if is_failed or is_fatal:
+                        self.logger.error(f"{out}")
+                        if (attempt_number + 1) == max_attempt or is_fatal:
+                            self.job.ddmErrorCode = ErrorCode.EC_Adder
+                            # extract important error string
+                            extracted_error = DataServiceUtils.extractImportantError(out)
+                            err_msg = "Could not add files to DDM: "
+                            if extracted_error == "":
+                                self.job.ddmErrorDiag = err_msg + out.split("\n")[-1]
+                            else:
+                                self.job.ddmErrorDiag = err_msg + extracted_error
+                            if is_fatal:
+                                self.result.set_fatal()
+                            else:
+                                self.result.set_temporary()
+                            return
+                        self.logger.error(f"Try:{attempt_number}")
+                        # sleep
+                        time.sleep(10)
+                    else:
+                        self.logger.debug(f"{str(out)}")
+                        break
+            # done
+            self.result.set_succeeded()
+            self.logger.debug("end plugin")
+        except Exception as e:
+            err_str = f"failed to execute with {str(e)}\n"
+            err_str += traceback.format_exc()
+            self.logger.error(err_str)
+            self.result.set_temporary()
+        # return
+        return

pandaserver/dataservice/closer.py

@@ -0,0 +1,247 @@
+"""
+update dataset DB, and then close dataset and start Activator if needed
+
+"""
+
+import datetime
+import sys
+from typing import Dict, List
+
+from pandacommon.pandalogger.LogWrapper import LogWrapper
+from pandacommon.pandalogger.PandaLogger import PandaLogger
+from pandacommon.pandautils.PandaUtils import naive_utcnow
+
+from pandaserver.config import panda_config
+from pandaserver.dataservice import DataServiceUtils
+from pandaserver.dataservice.activator import Activator
+from pandaserver.taskbuffer import EventServiceUtils
+
+# logger
+_logger = PandaLogger().getLogger("closer")
+
+
+class Closer:
+    """
+    Update dataset DB, and then close dataset and start Activator if needed.
+    The class updates output-related dataset records (_sub, output, and log) in
+    the database, closes _sub datasets if necessary, and activates downstream
+    runJobs if the output dataset is for buildJob (libDS, library dataset).
+    Activator changes job status from defined/assigned to activated.
+    """
+
+    # constructor
+    def __init__(self, taskBuffer, destination_data_blocks: List[str], job, dataset_map: Dict = None) -> None:
+        """
+        Constructor
+
+        Args:
+            taskBuffer: Task buffer.
+            destination_data_blocks (List[str]): Destination Dispatch blocks.
+            job: Job.
+            dataset_map (Dict, optional): Dataset map. Defaults to {}.
+        """
+        self.task_buffer = taskBuffer
+        self.destination_data_blocks = destination_data_blocks
+        self.job = job
+        self.panda_id = job.PandaID
+        self.site_mapper = None
+        self.dataset_map = dataset_map if dataset_map is not None else {}
+        self.all_subscription_finished = None
+
+    def check_sub_datasets_in_jobset(self) -> bool:
+        """
+        Check sub datasets with the same jobset
+
+        Returns:
+            bool: True if all sub datasets are done, False otherwise.
+        """
+        tmp_log = LogWrapper(_logger, f"check_sub_datasets_in_jobset-{naive_utcnow().isoformat('/')}")
+        # skip already checked
+        if self.all_subscription_finished is not None:
+            return self.all_subscription_finished
+        # get consumers in the jobset
+        jobs = self.task_buffer.getOriginalConsumers(self.job.jediTaskID, self.job.jobsetID, self.job.panda_id)
+        checked_dataset = set()
+        for job_spec in jobs:
+            # collect all sub datasets
+            sub_datasets = set()
+            for file_spec in job_spec.Files:
+                if file_spec.type == "output":
+                    sub_datasets.add(file_spec.destinationDBlock)
+            sub_datasets = sorted(sub_datasets)
+            if len(sub_datasets) > 0:
+                # use the first sub dataset
+                sub_dataset = sub_datasets[0]
+                # skip if already checked
+                if sub_dataset in checked_dataset:
+                    continue
+                checked_dataset.add(sub_dataset)
+                # count the number of unfinished
+                not_finish = self.task_buffer.countFilesWithMap({"destinationDBlock": sub_dataset, "status": "unknown"})
+                if not_finish != 0:
+                    tmp_log.debug(f"related sub dataset {sub_dataset} from {job_spec.PandaID} has {not_finish} unfinished files")
+                    self.all_subscription_finished = False
+                    break
+        if self.all_subscription_finished is None:
+            tmp_log.debug("all related sub datasets are done")
+            self.all_subscription_finished = True
+        return self.all_subscription_finished
+
+    def determine_final_status(self, destination_data_block: str) -> str:
+        """
+        Determine the final status of a dispatch block.
+
+        Args:
+            destination_data_block (str): The destination dispatch block.
+
+        Returns:
+            str: The final status.
+        """
+        # for queues without rucio storage element attached
+        if self.job.destinationSE == "local" and self.job.prodSourceLabel in ["user", "panda"]:
+            # close non-Rucio destinationDBlock immediately
+            return "closed"
+        if self.job.lockedby == "jedi" and DataServiceUtils.is_top_level_dataset(destination_data_block):
+            # set it closed in order not to trigger DDM cleanup. It will be closed by JEDI
+            return "closed"
+        if self.job.produceUnMerge():
+            return "doing"
+
+        # set status to 'tobeclosed' to trigger Rucio closing
+        return "tobeclosed"
+
+    def perform_vo_actions(self, final_status_dataset: list) -> None:
+        """
+        Perform special actions for vo.
+
+        Args:
+            final_status_dataset (list): The final status dataset.
+        """
+        closer_plugin_class = panda_config.getPlugin("closer_plugins", self.job.VO)
+        if closer_plugin_class is None and self.job.VO == "atlas":
+            # use ATLAS plugin for ATLAS
+            from pandaserver.dataservice.closer_atlas_plugin import CloserAtlasPlugin
+
+            closer_plugin_class = CloserAtlasPlugin
+        if closer_plugin_class is not None:
+            closer_plugin = closer_plugin_class(self.job, final_status_dataset, _logger)
+            closer_plugin.execute()
+
+    def start_activator(self, dataset):
+        """
+        Start the activator
+
+        Args:
+            dataset: Dataset.
+            final_status (str): Final status.
+        """
+        # start Activator
+        if (
+            not DataServiceUtils.is_sub_dataset(dataset.name)
+            and self.job.jobStatus == "finished"
+            and (self.job.prodSourceLabel != "panda" or self.job.processingType not in ["merge", "unmerge"])
+        ):
+            activator_thread = Activator(self.task_buffer, dataset)
+            activator_thread.run()
+
+    # main
+    def run(self):
+        """
+        Main method to run the Closer class. It processes each destination dispatch block,
+        updates the dataset status and finalizes pending jobs if necessary.
+        """
+        try:
+            tmp_log = LogWrapper(_logger, f"run-{naive_utcnow().isoformat('/')}-{self.panda_id}")
+            tmp_log.debug(f"Start with job status: {self.job.jobStatus}")
+            flag_complete = True
+            final_status_dataset = []
+
+            for destination_data_block in self.destination_data_blocks:
+                dataset_list = []
+                tmp_log.debug(f"start with destination dispatch block: {destination_data_block}")
+
+                # ignore task output datasets (tid) datasets
+                if DataServiceUtils.is_tid_dataset(destination_data_block):
+                    tmp_log.debug(f"skip {destination_data_block}")
+                    continue
+
+                # ignore HC datasets
+                if DataServiceUtils.is_hammercloud_dataset(destination_data_block) or DataServiceUtils.is_user_gangarbt_dataset(destination_data_block):
+                    if not DataServiceUtils.is_sub_dataset(destination_data_block) and not DataServiceUtils.is_lib_dataset(destination_data_block):
+                        tmp_log.debug(f"skip HC {destination_data_block}")
+                        continue
+
+                # query dataset
+                if destination_data_block in self.dataset_map:
+                    dataset = self.dataset_map[destination_data_block]
+                else:
+                    dataset = self.task_buffer.queryDatasetWithMap({"name": destination_data_block})
+
+                if dataset is None:
+                    tmp_log.error(f"Not found : {destination_data_block}")
+                    flag_complete = False
+                    continue
+
+                # skip tobedeleted/tobeclosed
+                if dataset.status in ["cleanup", "tobeclosed", "completed", "deleted"]:
+                    tmp_log.debug(f"skip {destination_data_block} due to dataset status: {dataset.status}")
+                    continue
+
+                dataset_list.append(dataset)
+                dataset_list.sort()
+
+                # count number of completed files
+                not_finish = self.task_buffer.countFilesWithMap({"destinationDBlock": destination_data_block, "status": "unknown"})
+                if not_finish < 0:
+                    tmp_log.error(f"Invalid dispatch block file count: {not_finish}")
+                    flag_complete = False
+                    continue
+
+                # check if completed
+                tmp_log.debug(f"Pending file count: {not_finish}")
+                final_status = self.determine_final_status(destination_data_block)
+
+                if not_finish == 0 and EventServiceUtils.isEventServiceMerge(self.job):
+                    all_in_jobset_finished = self.check_sub_datasets_in_jobset()
+                else:
+                    all_in_jobset_finished = True
+
+                if not_finish == 0 and all_in_jobset_finished:
+                    tmp_log.debug(f"Set final status: {final_status} to dataset: {destination_data_block}")
+                    # set status
+                    dataset.status = final_status
+                    # update dataset in DB
+                    ret_t = self.task_buffer.updateDatasets(
+                        dataset_list,
+                        withLock=True,
+                        withCriteria="status<>:crStatus AND status<>:lockStatus ",
+                        criteriaMap={":crStatus": final_status, ":lockStatus": "locked"},
+                    )
+                    if len(ret_t) > 0 and ret_t[0] == 1:
+                        final_status_dataset += dataset_list
+                        self.start_activator(dataset)
+                else:
+                    # update dataset in DB
+                    self.task_buffer.updateDatasets(
+                        dataset_list,
+                        withLock=True,
+                        withCriteria="status<>:crStatus AND status<>:lockStatus ",
+                        criteriaMap={":crStatus": final_status, ":lockStatus": "locked"},
+                    )
+                    # unset flag
+                    flag_complete = False
+                # end
+                tmp_log.debug(f"end {destination_data_block}")
+            # special actions for vo
+            if flag_complete:
+                self.perform_vo_actions(final_status_dataset)
+
+            # update unmerged datasets in JEDI to trigger merging
+            if flag_complete and self.job.produceUnMerge() and final_status_dataset:
+                tmp_stat = self.task_buffer.updateUnmergedDatasets(self.job, final_status_dataset)
+                tmp_log.debug(f"updated unmerged datasets with {tmp_stat}")
+
+            tmp_log.debug("End")
+        except Exception:
+            err_type, err_value = sys.exc_info()[:2]
+            tmp_log.error(f"{err_type} {err_value}")

pandaserver/dataservice/closer_atlas_plugin.py

@@ -0,0 +1,90 @@
+"""
+ATLAS plugin for closer
+"""
+import sys
+
+from pandacommon.pandalogger.LogWrapper import LogWrapper
+from pandaserver.dataservice.ddm import rucioAPI
+from pandaserver.dataservice import DataServiceUtils
+
+
+# plugin for ATLAS closer
+class CloserAtlasPlugin:
+    """
+    A class used to represent the ATLAS closer plugin.
+
+    ...
+
+    Attributes
+    ----------
+    jobSpec : object
+        an object representing the job specification
+    datasets : list
+        a list of datasets to be processed
+    tmp_log : LogWrapper
+        a LogWrapper object for logging
+
+    Methods
+    -------
+    __init__(self, job, datasets, log)
+        Constructs all the necessary attributes for the CloserAtlasPlugin object.
+    execute(self)
+        Executes the plugin's main functionality.
+    """
+
+    # constructor
+    def __init__(self, job, datasets, log):
+        """
+        Constructs all the necessary attributes for the CloserAtlasPlugin object.
+
+        Parameters
+        ----------
+            job : object
+                an object representing the job specification
+            datasets : list
+                a list of datasets to be processed
+            log : LogWrapper
+                a LogWrapper object for logging
+        """
+        self.jobSpec = job
+        self.datasets = datasets
+        self.tmp_log = LogWrapper(log, f"{self.jobSpec.PandaID} CloserAtlasPlugin")
+
+    # execute
+    def execute(self):
+        """
+        Executes the main functionality of the CloserAtlasPlugin.
+
+        This method is responsible for closing datasets that meet certain criteria.
+        It only operates on production jobs that are either 'urgent' or have a high priority.
+        For each dataset in the job, if the dataset's name ends with '_sub' followed by a number and its status is 'tobeclosed',
+        it attempts to close the dataset using the rucioAPI. If an error occurs during this process, it logs a warning message.
+
+        Returns
+        -------
+        bool
+            Always returns True regardless of whether the datasets were successfully closed or not.
+        """
+        try:
+            # only for production
+            if self.jobSpec.prodSourceLabel not in ["managed", "test"]:
+                return True
+            # only for urgent or high prio
+            if self.jobSpec.processingType not in ["urgent"] and self.jobSpec.currentPriority <= 1000:
+                return True
+            # close datasets
+            for datasetSpec in self.datasets:
+                if not DataServiceUtils.is_sub_dataset(datasetSpec.name):
+                    continue
+                if datasetSpec.status != "tobeclosed":
+                    continue
+                try:
+                    self.tmp_log.debug(f"immediate close {datasetSpec.name}")
+                    rucioAPI.close_dataset(datasetSpec.name)
+                except Exception:
+                    errtype, errvalue = sys.exc_info()[:2]
+                    self.tmp_log.warning(f"failed to close : {errtype} {errvalue}")
+        except Exception:
+            errtype, errvalue = sys.exc_info()[:2]
+            self.tmp_log.warning(f"failed to execute : {errtype} {errvalue}")
+        return True