panda-server-source 0.8.0__py3-none-any.whl

pandaserver/dataservice/event_lookup_client_ei.py

@@ -0,0 +1,77 @@
+"""
+EventLookupClientEI is a class for looking up events in the EventIndex.
+"""
+import os
+import subprocess
+import tempfile
+
+from typing import List, Tuple
+
+
+class EventLookupClientEI:
+    """
+    EventLookupClientEI is a class for looking up events in the EventIndex.
+    """
+    def do_lookup(self, event_run_list: List[Tuple[int, int]], stream: str = None, tokens: str = None, ami_tag: str = None) -> Tuple[List[str], str, str, str]:
+        """
+        Performs a lookup in the EventIndex for the given parameters.
+
+        Parameters:
+            event_run_list (List[Tuple[int, int]]): The list of run events.
+            stream (str): The name of the stream.
+            tokens (str): The tokens.
+            ami_tag (str): The AMI tag.
+
+        Returns:
+            Tuple[List[str], str, str, str]: A tuple containing the list of GUIDs, the command, the output, and the error.
+        """
+        command = os.path.join(
+            os.getenv(
+                "EIDIR",
+                "/cvmfs/atlas.cern.ch/repo/ATLASLocalRootBase/x86_64/EIClient/current",
+            ),
+            "bin",
+            "event-lookup",
+        )
+        with tempfile.NamedTemporaryFile(mode="w+t") as tmp_event_file:
+            command += f" -F {tmp_event_file.name} "
+            for run_event in event_run_list:
+                # creating a preformatted string. The :08d and :09d parts are formatting these integers to be eight and nine digits long
+                tmp_string = f"{int(run_event[0]):08d} {int(run_event[1]):09d}\n"
+                tmp_event_file.write(tmp_string)
+            tmp_event_file.flush()
+            if stream not in [None, ""]:
+                command += f"-s {stream} "
+            if ami_tag not in [None, ""]:
+                command += f"-a {ami_tag} "
+            command += "-c plain "
+            with subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True,
+                                  universal_newlines=True) as execute_process:
+                tmp_output, tmp_error = execute_process.communicate()
+            guids = {}
+            if tokens == "":
+                tokens = None
+            try:
+                # The output is expected to be a string with multiple lines, each line containing information about a single event.
+                # Each line is split into items, the first two items are interpreted as integers representing a run number and an event number.
+                # These two numbers are combined into a tuple, run_event, which is used as a key in the guids dictionary.
+                # The fourth item in the line is appended to the string "Stream" to form tmp_token, and the third item is assigned to tmp_guid.
+                # An example input (tmp_output) is:
+                # 123 456 789abc StreamXYZ
+                # An example output (guids dictionary) is:
+                # (123, 456): {"789abc"}
+                for tmp_line in tmp_output.split("\n"):
+                    tmp_items = tmp_line.split()
+                    run_event = (int(tmp_items[0]), int(tmp_items[1]))
+                    guids.setdefault(run_event, set())
+                    # check type
+                    tmp_token = "Stream" + tmp_items[3]
+                    tmp_guid = tmp_items[2]
+                    if not tokens or tokens == tmp_token:
+                        guids[run_event].add(tmp_guid)
+                if not guids:
+                    # add dummy
+                    guids[None] = None
+            except Exception:
+                pass
+            return guids, command, tmp_output, tmp_error

pandaserver/dataservice/event_picker.py

@@ -0,0 +1,268 @@
+"""
+add data to dataset
+
+"""
+
+import fcntl
+import os
+import re
+import sys
+import traceback
+
+from pandacommon.pandalogger.LogWrapper import LogWrapper
+from pandacommon.pandalogger.PandaLogger import PandaLogger
+
+from pandaserver.dataservice import dyn_data_distributer
+from pandaserver.srvcore import CoreUtils
+from pandaserver.userinterface import Client
+
+# logger
+_logger = PandaLogger().getLogger("event_picker")
+
+
+class EventPicker:
+    """
+    A class used to add data to a dataset.
+    """
+
+    # constructor
+    def __init__(self, taskBuffer, siteMapper, evpFileName: str, ignoreError: bool):
+        """
+        Constructs all the necessary attributes for the EventPicker object.
+
+        Parameters:
+            taskBuffer : TaskBuffer
+                The task buffer that contains the jobs.
+            siteMapper : SiteMapper
+                The site mapper.
+            evpFileName : str
+                The name of the event picking file.
+            ignoreError : bool
+                Whether to ignore errors.
+        """
+        self.task_buffer = taskBuffer
+        self.site_mapper = siteMapper
+        self.ignore_error = ignoreError
+        self.event_picking_file_name = evpFileName
+        # logger
+        self.logger = LogWrapper(_logger)
+        dataset_lifetime = self.task_buffer.getConfigValue("event_picker", "EVP_DATASET_LIFETIME")
+        if not dataset_lifetime:
+            dataset_lifetime = 28
+        self.pd2p = dyn_data_distributer.DynDataDistributer([], self.site_mapper, token=" ", dataset_lifetime=dataset_lifetime)
+        self.user_dataset_name = ""
+        self.creation_time = ""
+        self.params = ""
+        self.locked_by = ""
+        self.event_picking_file = None
+        self.user_task_name = ""
+        self.user_dn = ""
+        # JEDI
+        self.jedi_task_id = None
+
+    # end with error
+    def end_with_error(self, message: str):
+        """
+        Ends the event picker with an error.
+
+        This method is called when an error occurs during the event picking process. It logs the error message,
+        unlocks and closes the event picking file, and removes it if the error is not to be ignored. It then uploads
+        the log and updates the task status in the task buffer.
+
+        Parameters:
+            message (str): The error message to be logged.
+        """
+        self.logger.error(message)
+        # unlock evp file
+        try:
+            fcntl.flock(self.event_picking_file.fileno(), fcntl.LOCK_UN)
+            self.event_picking_file.close()
+            if not self.ignore_error:
+                # remove evp file
+                os.remove(self.event_picking_file_name)
+        except Exception:
+            pass
+        # upload log
+        if self.jedi_task_id is not None:
+            out_log = self.upload_log()
+            self.task_buffer.updateTaskErrorDialogJEDI(self.jedi_task_id, f"event picking failed. {out_log}")
+            # update task
+            if not self.ignore_error:
+                self.task_buffer.updateTaskModTimeJEDI(self.jedi_task_id, "tobroken")
+            self.logger.debug(out_log)
+        self.logger.debug(f"end {self.event_picking_file_name}")
+
+    # upload log
+    def upload_log(self) -> str:
+        """
+        Uploads the log.
+
+        This method uploads the log of the EventPicker. It first checks if the jediTaskID is not None.
+        If it is None, it returns a message indicating that the jediTaskID could not be found.
+        Otherwise, it dumps the logger content to a string and attempts to upload it using the Client's upload_log method.
+        If the upload is not successful, it returns a message indicating the failure.
+        If the upload is successful and the output starts with "http", it returns a hyperlink to the log.
+        Otherwise, it returns the output of the upload_log method.
+
+        Returns:
+            str: The result of the log upload. This can be a message indicating an error, a hyperlink to the log, or the output of the upload_log method.
+        """
+        if self.jedi_task_id is None:
+            return "cannot find jediTaskID"
+        str_msg = self.logger.dumpToString()
+        status, output = Client.uploadLog(str_msg, self.jedi_task_id)
+        if status != 0:
+            return f"failed to upload log with {status}."
+        if output.startswith("http"):
+            return f'<a href="{output}">log</a>'
+        return output
+
+    def get_options_from_file(self) -> dict:
+        """
+        Gets options from the event picking file.
+
+        This method reads the event picking file and extracts options from it. The options are stored in a dictionary and returned.
+
+        Returns:
+            dict: A dictionary containing the options extracted from the event picking file.
+        """
+        options = {
+            "runEvent": [],
+            "eventPickDataType": "",
+            "eventPickStreamName": "",
+            "eventPickDS": [],
+            "eventPickAmiTag": "",
+            "eventPickNumSites": 1,
+            "inputFileList": [],
+            "tagDS": [],
+            "tagQuery": "",
+            "tagStreamRef": "",
+            "runEvtGuidMap": {},
+            "ei_api": "",
+            "userName": "",
+            "userTaskName": "",
+            "userDatasetName": "",
+            "lockedBy": "",
+            "creationTime": "",
+            "params": "",
+        }
+
+        for tmp_line in self.event_picking_file:
+            # regular expression to parse lines where a key and value are separated by an equal sign
+            tmp_match = re.search("^([^=]+)=(.+)$", tmp_line)
+            if tmp_match is not None:
+                key, value = tmp_match.groups()
+                if key in options:
+                    if key == "runEvent":
+                        tmp_run_event = value.split(",")
+                        if len(tmp_run_event) == 2:
+                            options[key].append(tmp_run_event)
+                    elif key in ["eventPickDS", "inputFileList", "tagDS"]:
+                        options[key] = value.split(",")
+                        if key == "inputFileList":
+                            options[key] = [item for item in options[key] if item != ""]
+                    elif key == "eventPickNumSites":
+                        options[key] = int(value)
+                    elif key == "runEvtGuidMap":
+                        options[key] = eval(value)
+                    else:
+                        options[key] = value
+                        if key == "tagStreamRef" and not options[key].endswith("_ref"):
+                            options[key] += "_ref"
+        return options
+
+    def get_jedi_task_id(self, options: dict) -> int:
+        """
+        Gets the jediTaskID.
+
+        This method gets the jediTaskID from the task buffer using the user's DN and task name.
+
+        Parameters:
+            options (dict): A dictionary containing the options extracted from the event picking file.
+
+        Returns:
+            int: The jediTaskID.
+        """
+        self.user_dn = options["userName"]
+        self.user_task_name = options["userTaskName"]
+        self.user_dataset_name = options["userDatasetName"]
+        self.locked_by = options["lockedBy"]
+        self.creation_time = options["creationTime"]
+        self.params = options["params"]
+
+        # extract task name
+        if self.user_task_name == "" and self.params != "":
+            # regular expression to parse the dataset name from a string where it is prefixed with --outDS and followed by either an equal sign or a space
+            tmp_match = re.search("--outDS(=| ) *([^ ]+)", self.params)
+            if tmp_match is not None:
+                self.user_task_name = tmp_match.group(2)
+                if not self.user_task_name.endswith("/"):
+                    self.user_task_name += "/"
+
+        compact_dn = CoreUtils.clean_user_id(self.user_dn)
+        return self.task_buffer.getTaskIDwithTaskNameJEDI(compact_dn, self.user_task_name)
+
+    # main
+    def run(self) -> bool:
+        """
+        Starts the event picker.
+
+        Returns:
+            bool: True if the event picker ran successfully, False otherwise.
+        """
+        try:
+            self.logger.debug(f"start {self.event_picking_file_name}")
+            # lock event picking file
+            with open(self.event_picking_file_name) as self.event_picking_file:
+                try:
+                    fcntl.flock(self.event_picking_file.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
+                except Exception:
+                    # release
+                    self.logger.debug(f"cannot lock {self.event_picking_file_name}")
+                    return True
+
+                options = self.get_options_from_file()
+
+                self.jedi_task_id = self.get_jedi_task_id(options)
+
+                tmp_ret, location_map, all_files = self.pd2p.convert_evt_run_to_datasets(
+                    options["runEvent"],
+                    options["eventPickDataType"],
+                    options["eventPickStreamName"],
+                    options["eventPickDS"],
+                    options["eventPickAmiTag"],
+                    options["runEvtGuidMap"],
+                )
+
+                if not tmp_ret:
+                    if "isFatal" in location_map and location_map["isFatal"] is True:
+                        self.ignore_error = False
+                    self.end_with_error("Failed to convert the run/event list to a dataset/file list")
+                    return False
+
+                # use only files in the list
+                if options["inputFileList"]:
+                    all_files = [tmp_file for tmp_file in all_files if tmp_file["lfn"] in options["inputFileList"]]
+
+                # remove redundant CN from DN
+                tmp_dn = CoreUtils.get_id_from_dn(self.user_dn)
+                tmp_ret = self.pd2p.register_dataset_container_with_datasets(
+                    self.user_dataset_name,
+                    all_files,
+                    location_map,
+                    n_sites=options["eventPickNumSites"],
+                    owner=tmp_dn,
+                )
+
+                if not tmp_ret:
+                    return False
+
+                fcntl.flock(self.event_picking_file.fileno(), fcntl.LOCK_UN)
+                os.remove(self.event_picking_file_name)
+
+                self.logger.debug(f"end {self.event_picking_file_name}")
+                return True
+        except Exception:
+            error_type, error_value = sys.exc_info()[:2]
+            self.end_with_error(f"Got exception {error_type}:{error_value} {traceback.format_exc()}")
+            return False

pandaserver/dataservice/finisher.py

@@ -0,0 +1,194 @@
+"""
+finish transferring jobs
+
+"""
+
+import datetime
+import json
+import re
+import sys
+import threading
+from typing import List
+
+from pandacommon.pandalogger.LogWrapper import LogWrapper
+from pandacommon.pandalogger.PandaLogger import PandaLogger
+from pandacommon.pandautils.PandaUtils import naive_utcnow
+
+# logger
+_logger = PandaLogger().getLogger("finisher")
+
+
+class Finisher(threading.Thread):
+    """
+    A class used to finish transferring jobs
+
+    Attributes
+    ----------
+    dataset : DatasetSpec
+        The dataset to be transferred
+    taskBuffer : TaskBuffer
+        The task buffer that contains the jobs
+    job : JobSpec, optional
+        The job to be transferred (default is None)
+    site : str, optional
+        The site where the job is to be transferred (default is None)
+
+    Methods
+    -------
+    run():
+        Starts the thread to finish transferring jobs
+    """
+
+    # constructor
+    def __init__(self, taskBuffer, dataset, job: str = None, site: str = None):
+        """
+        Constructs all the necessary attributes for the Finisher object.
+
+        Parameters
+        ----------
+            taskBuffer : TaskBuffer
+                The task buffer that contains the jobs
+            dataset : DatasetSpec
+                The dataset to be transferred
+            job : JobSpec, optional
+                The job to be transferred (default is None)
+            site : str, optional
+                The site where the job is to be transferred (default is None)
+        """
+        threading.Thread.__init__(self)
+        self.dataset = dataset
+        self.task_buffer = taskBuffer
+        self.job = job
+        self.site = site
+
+    def create_json_doc(self, job, failed_files: List[str], no_out_files: List[str]):
+        """
+        This function creates a JSON document for the jobs.
+
+        Parameters:
+        job (JobSpec): The job specification.
+        failed_files (list): List of failed files.
+        no_out_files (list): List of files with no output.
+
+        Returns:
+        str: The created JSON document as a string.
+        """
+        json_dict = {}
+        for file in job.Files:
+            if file.type in ["output", "log"]:
+                # skip failed or no-output files
+                if file.lfn in failed_files + no_out_files:
+                    continue
+                file_dict = {
+                    "guid": file.GUID,
+                    "fsize": file.fsize,
+                    "full_lfn": file.lfn,
+                }
+                if file.checksum.startswith("ad:"):
+                    file_dict["adler32"] = re.sub("^ad:", "", file.checksum)
+                else:
+                    file_dict["md5sum"] = re.sub("^md5:", "", file.checksum)
+                json_dict[file.lfn] = file_dict
+        return json.dumps(json_dict)
+
+    def update_job_output_report(self, job, failed_files: List[str], no_out_files: List[str]):
+        """
+        This function updates the job output report.
+
+        Parameters:
+        job (JobSpec): The job specification.
+        failed_files (list): List of failed files.
+        no_out_files (list): List of files with no output.
+        """
+        json_data = self.create_json_doc(job, failed_files, no_out_files)
+        record_status = "finished" if not failed_files else "failed"
+        tmp_ret = self.task_buffer.updateJobOutputReport(
+            panda_id=job.PandaID,
+            attempt_nr=job.attemptNr,
+            data=json_data,
+        )
+        if not tmp_ret:
+            self.task_buffer.insertJobOutputReport(
+                panda_id=job.PandaID,
+                prod_source_label=job.prodSourceLabel,
+                job_status=record_status,
+                attempt_nr=job.attemptNr,
+                data=json_data,
+            )
+
+    def check_file_status(self, job):
+        """
+        This function checks the status of the files for the job.
+
+        Parameters:
+        job (JobSpec): The job specification.
+
+        Returns:
+        tuple: A tuple containing three elements:
+            - bool: True if all files are ready, False otherwise.
+            - list: A list of failed files.
+            - list: A list of files with no output.
+        """
+        tmp_log = LogWrapper(_logger, f"check_file_status-{naive_utcnow().isoformat('/')}")
+        failed_files = []
+        no_out_files = []
+        for file in job.Files:
+            if file.type in ("output", "log"):
+                if file.status == "failed":
+                    failed_files.append(file.lfn)
+                elif file.status == "nooutput":
+                    no_out_files.append(file.lfn)
+                elif file.status != "ready":
+                    tmp_log.debug(f"Job: {job.PandaID} file:{file.lfn} {file.status} != ready")
+                    return False, failed_files, no_out_files
+        return True, failed_files, no_out_files
+
+    # main
+    def run(self):
+        """
+        Starts the thread to finish transferring jobs
+        """
+        tmp_log = LogWrapper(_logger, f"run-{naive_utcnow().isoformat('/')}")
+        # start
+        try:
+            by_call_back = False
+            if self.job is not None:
+                tmp_log.debug(f"start: {self.job.PandaID}")
+                panda_ids = [self.job.PandaID]
+                jobs = [self.job]
+            else:
+                by_call_back = True
+                tmp_log.debug(f"start: {self.dataset.name}")
+                panda_ids = self.task_buffer.updateOutFilesReturnPandaIDs(self.dataset.name)
+                # set flag for T2 cleanup
+                self.dataset.status = "cleanup"
+                self.task_buffer.updateDatasets([self.dataset])
+                jobs = self.task_buffer.peekJobs(panda_ids, fromDefined=False, fromArchived=False, fromWaiting=False)
+
+            tmp_log.debug(f"IDs: {panda_ids}")
+            if len(panda_ids) != 0:
+                # loop over all jobs
+                for job in jobs:
+                    if job is None or job.jobStatus != "transferring":
+                        continue
+                    job_ready, failed_files, no_out_files = self.check_file_status(job)
+                    # finish job
+                    if job_ready:
+                        if by_call_back:
+                            tmp_log.debug(f"Job: {job.PandaID} all files ready")
+                        else:
+                            tmp_log.debug(f"Job: {job.PandaID} all files checked with catalog")
+                        # create JSON
+                        try:
+                            self.update_job_output_report(job, failed_files, no_out_files)
+                        except Exception:
+                            exc_type, value, _ = sys.exc_info()
+                            tmp_log.error(f"Job: {job.PandaID} {exc_type} {value}")
+                    tmp_log.debug(f"Job: {job.PandaID} status: {job.jobStatus}")
+            if self.job is None:
+                tmp_log.debug(f"end: {self.dataset.name}")
+            else:
+                tmp_log.debug(f"end: {self.job.PandaID}")
+        except Exception:
+            exc_type, value, _ = sys.exc_info()
+            tmp_log.error(f"run() : {exc_type} {value}")