panda-server-source 0.8.0__py3-none-any.whl

pandaserver/dataservice/DataServiceUtils.py

@@ -0,0 +1,323 @@
+import re
+
+from pandaserver.taskbuffer import JobUtils
+
+
+def isCachedFile(dataset_name, site_spec):
+    """
+    Check if the file is cached on a site using CVMFS, e.g. for DB releases.
+
+    Args:
+        dataset_name (str): The name of the dataset.
+        site_spec (SiteSpec): The site specification object.
+    Returns:
+        bool: True if the file is cached, False otherwise.
+    """
+    # Site does not use CVMFS
+    if site_spec.iscvmfs is not True:
+        return False
+    # look for DBR
+    if not dataset_name.startswith("ddo"):
+        return False
+    # look for the letter 'v' followed by 6 digits
+    if re.search(r"v\d{6}$", dataset_name) is None:
+        return False
+    return True
+
+
+# check invalid characters in dataset name
+def checkInvalidCharacters(dataset_name):
+    """
+    Checks the validity of a dataset name.
+    - The dataset name starts with an alphanumeric character ([A-Za-z0-9]).
+    - The rest of the dataset name can contain alphanumeric characters, dots(.), hyphens(-),
+      underscores(_), or slashes( /), and its length is between 1 and 255 characters({1, 255}).
+
+    Args:
+         dataset_name: The name of the dataset.
+    Returns:
+        True if the dataset name is valid, False otherwise.
+    """
+    if re.match("^[A-Za-z0-9][A-Za-z0-9\.\-\_/]{1,255}$", dataset_name) is not None:
+        return True
+    return False
+
+
+# get dataset type
+def getDatasetType(dataset):
+    """
+    Get the dataset type by extracting the fifth element from the dataset name.
+
+    The dataset name is expected to be a string with elements separated by dots.
+    For example:
+    - 'mc23_13p6TeV:mc23_13p6TeV.801169.Py8EG_A14NNPDF23LO_jj_JZ4.merge.EVNT.e8514_e8528_tid38750682_00' would return 'EVNT'.
+    - 'mc23_13p6TeV.801169.Py8EG_A14NNPDF23LO_jj_JZ4.simul.HITS.e8514_e8528_a934_tid41381346_00' would return 'HITS'.
+
+    Args:
+        dataset (str): The name of the dataset.
+    Returns:
+        str: The dataset type if it can be extracted, otherwise None.
+    """
+    try:
+        if dataset.startswith("user") or dataset.startswith("group"):
+            return None
+        dataset_type = dataset.split(".")[4]
+    except Exception:
+        dataset_type = None
+    return dataset_type
+
+
+def getSitesShareDDM(site_mapper, site_name, prod_source_label, job_label, output_share=False):
+    """
+    Get sites which share the DDM endpoint.
+
+    Args:
+        site_mapper (SiteMapper): The site mapper object.
+        site_name (str): The name of the site.
+        prod_source_label (str): The production source label.
+        job_label (str): The job label.
+        output_share (bool): False to get sites which use the output RSE as input,
+                             True to get sites which use the input RSEs as output.
+    Returns:
+        list: A list of site names that share the DDM endpoint.
+    """
+    if not site_mapper.checkSite(site_name):
+        return []
+
+    site_spec = site_mapper.getSite(site_name)
+    scope_site_input, scope_site_output = select_scope(site_spec, prod_source_label, job_label)
+    runs_production = site_spec.runs_production()
+    runs_analysis = site_spec.runs_analysis()
+
+    ret_sites = []
+    for tmp_site_name, tmp_site_spec in site_mapper.siteSpecList.items():
+        scope_tmp_site_input, scope_tmp_site_output = select_scope(tmp_site_spec, prod_source_label, job_label)
+
+        if (runs_production and not tmp_site_spec.runs_production()) or (runs_analysis and not tmp_site_spec.runs_analysis()):
+            continue
+        if tmp_site_spec.status != "online":
+            continue
+
+        try:
+            if not output_share and site_spec.ddm_output[scope_site_output] not in tmp_site_spec.ddm_endpoints_input[scope_tmp_site_input].all:
+                continue
+            if output_share and tmp_site_spec.ddm_output[scope_site_output] not in site_spec.ddm_endpoints_input[scope_tmp_site_input].all:
+                continue
+        except Exception:
+            continue
+
+        if site_name != tmp_site_spec.sitename and tmp_site_spec.sitename not in ret_sites:
+            ret_sites.append(tmp_site_spec.sitename)
+
+    return ret_sites
+
+
+def checkJobDestinationSE(tmp_job):
+    """
+    Check if the job has a destination storage element in a file specification.
+
+    Args:
+        tmp_job (JobSpec): The job object containing file specifications.
+    Returns:
+        str: The destination SE if specified, otherwise None.
+    """
+    for tmp_file in tmp_job.Files:
+        destination_se = getDestinationSE(tmp_file.destinationDBlockToken)
+        if destination_se is not None:
+            return tmp_file.destinationSE
+    return None
+
+
+def getDestinationSE(destination_dblock_token):
+    """
+    Check if the destination is specified (e.g. dst:CERN-PROD_DATADISK) and extract it.
+
+    Args:
+        destination_dblock_token (str): The destination data block token.
+    Returns:
+        str: The extracted destination if specified, otherwise None.
+    """
+    if destination_dblock_token is not None:
+        for tmp_token in destination_dblock_token.split(","):
+            tmp_match = re.search(r"^dst:([^/]*)(/.*)*$", tmp_token)
+            if tmp_match is not None:
+                return tmp_match.group(1)
+    return None
+
+
+def getDistributedDestination(destination_dblock_token, ignore_empty=True):
+    """
+    Check if the destination is distributed (e.g. ddd:CERN-PROD_DATADISK) and extract it.
+
+    Args:
+        destination_dblock_token (str): The destination data block token.
+        ignore_empty (bool): Flag to ignore empty locations.
+
+    Returns:
+        str: The extracted location if specified, otherwise None.
+    """
+    if destination_dblock_token is not None:
+        for tmp_token in destination_dblock_token.split(","):
+            tmp_match = re.search(r"^ddd:([^/]*)(/.*)*$", tmp_token)
+            if tmp_match is not None:
+                location = tmp_match.group(1)
+                if ignore_empty and not location:
+                    return None
+                return location
+    return None
+
+
+def extractImportantError(message):
+    """
+    Extract important error strings from a message. This function searches for specific substrings within a given message and returns
+    a concatenated string of lines containing those substrings.
+
+    Args:
+        message (str): The message string to search within.
+    Returns:
+        str: A concatenated string of lines containing the specified substrings.
+              Returns an empty string if no matches are found or an exception occurs.
+    """
+    try:
+        # list of strings to search for
+        str_list = ["InvalidRSEExpression", "Details:"]
+        return_string = " ".join(line for line in message.split("\n") if any(tmp_string in line for tmp_string in str_list))
+    except Exception:
+        return_string = ""
+    return return_string
+
+
+def getActivityForOut(prod_source_label):
+    """
+    Get the DDM activity type for the job output based on the production source label.
+
+    Args:
+        prod_source_label (str): The production source label.
+    Returns:
+        str: The activity type for output.
+    """
+    if prod_source_label in ["managed"]:
+        activity = "Production Output"
+    elif prod_source_label in ["user", "panda"]:
+        activity = "Analysis Output"
+    else:
+        activity = "Functional Test"
+    return activity
+
+
+def select_scope(site_spec, prod_source_label, job_label):
+    """
+    Select the scopes of the activity for input and output. The scope was introduced for prod-analy queues where you might want to associate
+    different RSEs depending on production or analysis.
+
+    Args:
+        site_spec (SiteSpec): The site specification object.
+        prod_source_label (str): The production source label.
+        job_label (str): The job label.
+    Returns:
+        tuple: A tuple containing the input scope and output scope.
+    """
+    scope_input = "default"
+    aux_scopes_input = site_spec.ddm_endpoints_input.keys()
+    if (job_label == JobUtils.ANALY_PS or prod_source_label in JobUtils.analy_sources) and "analysis" in aux_scopes_input:
+        scope_input = "analysis"
+
+    scope_output = "default"
+    aux_scopes_output = site_spec.ddm_endpoints_output.keys()
+    if (job_label == JobUtils.ANALY_PS or prod_source_label in JobUtils.analy_sources) and "analysis" in aux_scopes_output:
+        scope_output = "analysis"
+
+    return scope_input, scope_output
+
+
+def isDBR(dataset_name):
+    """
+    Check if the dataset is a DB release. A DB release dataset name starts with 'ddo'.
+
+    Args:
+        dataset_name (str): The name of the dataset.
+
+    Returns:
+        bool: True if the dataset name starts with 'ddo', False otherwise.
+    """
+    if dataset_name.startswith("ddo"):
+        return True
+    return False
+
+
+def is_top_level_dataset(dataset_name: str) -> bool:
+    """
+    Check if top dataset. Top datasets do not finish with '_sub' followed by one or more digits.
+
+    Args:
+        dataset_name (str): Dataset name.
+
+    Returns:
+        bool: True if top dataset, False otherwise.
+    """
+    return re.sub("_sub\d+$", "", dataset_name) == dataset_name
+
+
+def is_sub_dataset(dataset_name: str) -> bool:
+    """
+    Check if the dataset name ends with '_sub' followed by one or more digits.
+
+    Args:
+        dataset_name (str): The name of the dataset.
+
+    Returns:
+        bool: True if the dataset name ends with '_sub' followed by one or more digits, False otherwise.
+    """
+    return re.search("_sub\d+$", dataset_name) is not None
+
+
+def is_tid_dataset(destination_data_block: str) -> bool:
+    """
+    Check if the destination data block ends with '_tid' followed by one or more digits.
+
+    Args:
+        destination_data_block (str): The destination data block.
+
+    Returns:
+        bool: True if the destination data block ends with '_tid' followed by one or more digits, False otherwise.
+    """
+    return re.search("_tid[\d_]+$", destination_data_block) is not None
+
+
+def is_hammercloud_dataset(destination_data_block: str) -> bool:
+    """
+    Check if the destination data block starts with 'hc_test.'.
+
+    Args:
+        destination_data_block (str): The destination data block.
+
+    Returns:
+        bool: True if the destination data block starts with 'hc_test.', False otherwise.
+    """
+    return re.search("^hc_test\.", destination_data_block) is not None
+
+
+def is_user_gangarbt_dataset(destination_data_block: str) -> bool:
+    """
+    Check if the destination data block starts with 'user.gangarbt.'.
+
+    Args:
+        destination_data_block (str): The destination data block.
+
+    Returns:
+        bool: True if the destination data block starts with 'user.gangarbt.', False otherwise.
+    """
+    return re.search("^user\.gangarbt\.", destination_data_block) is not None
+
+
+def is_lib_dataset(destination_data_block: str) -> bool:
+    """
+    Check if the destination data block ends with '.lib'.
+
+    Args:
+        destination_data_block (str): The destination data block.
+
+    Returns:
+        bool: True if the destination data block ends with '.lib', False otherwise.
+    """
+    return re.search("\.lib$", destination_data_block) is not None

pandaserver/dataservice/ErrorCode.py

@@ -0,0 +1,25 @@
+# error code
+
+# Setupper
+EC_Setupper = 100
+
+# Setupper
+EC_GUID = 101
+
+# Adder
+EC_Adder = 200
+
+# Subscription failures
+EC_Subscription = 201
+
+# location registration failures
+EC_Location = 203
+
+# missing nEvents
+EC_MissingNumEvents = 204
+
+# inconsistent GUID
+EC_InconsistentGUID = 205
+
+# wrong RSE configuration
+EC_RSE = 206

pandaserver/dataservice/RecoverLostFilesCore.py

@@ -0,0 +1,322 @@
+import argparse
+import os
+import sys
+
+from pandacommon.pandalogger.LogWrapper import LogWrapper
+from pandacommon.pandautils.thread_utils import GenericThread
+from rucio.client import Client as RucioClient
+from rucio.common.exception import DataIdentifierNotFound
+
+from pandaserver.config import panda_config
+from pandaserver.dataservice.ddm import rucioAPI
+from pandaserver.userinterface import Client
+
+
+# get files form rucio
+def get_files_from_rucio(ds_name):
+    # get files from rucio
+    try:
+        rc = RucioClient()
+        scope, name = rucioAPI.extract_scope(ds_name)
+        if name.endswith("/"):
+            name = name[:-1]
+        files_rucio = set()
+        for i in rc.list_files(scope, name):
+            files_rucio.add(i["name"])
+        return True, files_rucio
+    except DataIdentifierNotFound:
+        return False, f"unknown dataset {ds_name}"
+    except Exception as e:
+        msg_str = f"failed to get files from rucio: {str(e)}"
+        return None, msg_str
+
+
+# print a message
+def print_msg(message: str, log_stream: LogWrapper | None, is_error: bool = False, put_log: str | None = None):
+    """
+    Print a message to log stream or stdout.
+
+    :param message: Message string.
+    :param log_stream: LogWrapper object or None.
+    :param is_error: If True, log as error.
+    :param put_log: Log filename to dump log to this file in cache_dir.
+    """
+    if log_stream:
+        if is_error:
+            log_stream.error("Error: " + message)
+        else:
+            log_stream.info(message)
+        if put_log:
+            with open(os.path.join(panda_config.cache_dir, put_log), "w") as f:
+                f.write(log_stream.dumpToString())
+                f.write("done\n")
+    else:
+        if is_error:
+            print("ERROR: ", message)
+        else:
+            print(message)
+
+
+# main
+def main(taskBuffer=None, exec_options=None, log_stream=None, args_list=None):
+    # options
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--ds",
+        action="store",
+        dest="ds",
+        default=None,
+        help="dataset name",
+    )
+    parser.add_argument(
+        "--files",
+        action="store",
+        dest="files",
+        default=None,
+        help="comma-separated list of lost file names. The list is deduced if this option is omitted",
+    )
+    parser.add_argument(
+        "--jediTaskID",
+        action="store",
+        dest="jediTaskID",
+        type=int,
+        default=None,
+        help="JEDI task ID that produced the dataset",
+    )
+    parser.add_argument(
+        "--noChildRetry",
+        action="store_const",
+        const=True,
+        dest="noChildRetry",
+        default=False,
+        help="not retry child tasks",
+    )
+    parser.add_argument(
+        "--resurrectDS",
+        action="store_const",
+        const=True,
+        dest="resurrectDS",
+        default=False,
+        help="resurrect output and log datasets if they were already deleted",
+    )
+    parser.add_argument(
+        "--dryRun",
+        action="store_const",
+        const=True,
+        dest="dryRun",
+        default=False,
+        help="dry run",
+    )
+    parser.add_argument(
+        "--force",
+        action="store_const",
+        const=True,
+        dest="force",
+        default=False,
+        help="force retry even if no lost files",
+    )
+    parser.add_argument(
+        "--reproduceParent",
+        action="store_const",
+        const=True,
+        dest="reproduceParent",
+        default=False,
+        help="reproduce the input files from which the lost files were produced. "
+        "Typically useful to recover merged files when unmerged files were already deleted",
+    )
+    parser.add_argument(
+        "--reproduceUptoNthGen",
+        action="store",
+        dest="reproduceUptoNthGen",
+        type=int,
+        default=0,
+        help="reproduce files up to N-th upper generation. --reproduceUptoNthGen=1 corresponds "
+        "to --reproduceParent. 0 by default so that any provenance files are not reproduced",
+    )
+    # parse options
+    if taskBuffer:
+        if args_list:
+            options = parser.parse_args(args_list)
+        else:
+            options, unknown = parser.parse_known_args()
+    else:
+        if args_list:
+            options = parser.parse_args(args_list)
+        else:
+            options = parser.parse_args()
+
+    requester_id = GenericThread().get_full_id(__name__, sys.modules[__name__].__file__)
+
+    if taskBuffer is None:
+        # instantiate TB
+        from pandaserver.taskbuffer.TaskBuffer import taskBuffer
+
+        taskBuffer.init(
+            panda_config.dbhost,
+            panda_config.dbpasswd,
+            nDBConnection=1,
+            requester=requester_id,
+        )
+
+    # set options from dict
+    if exec_options is None:
+        exec_options = {}
+    keys = set(vars(options).keys())
+    for k in exec_options:
+        if k in keys:
+            setattr(options, k, exec_options[k])
+
+    dn = exec_options.get("userName")
+    is_production_manager = exec_options.get("isProductionManager", False)
+    log_filename = exec_options.get("logFilename")
+    print_msg(f"User='{dn}' is_prod={is_production_manager} log_name={log_filename}", log_stream)
+
+    if options.reproduceUptoNthGen > 0:
+        options.reproduceUptoNthGen -= 1
+        options.reproduceParent = True
+
+    # check if dataset name or taskID is given
+    if options.jediTaskID is None and options.ds is None:
+        msg_str = "Dataset name or jediTaskID is required"
+        print_msg(msg_str, log_stream, is_error=True, put_log=log_filename)
+        return False, msg_str
+
+    ds_files = {}
+    if options.files is not None:
+        files = options.files.split(",")
+        ds_files[options.ds] = files
+    else:
+        # look for lost files
+        if not options.jediTaskID:
+            # get files from rucio
+            st, files_rucio = get_files_from_rucio(options.ds)
+            if st is not True:
+                print_msg(files_rucio, log_stream, is_error=True, put_log=log_filename)
+                return st, files_rucio
+            # get files from panda
+            dsName = options.ds.split(":")[-1]
+            fd, fo = taskBuffer.querySQLS(
+                "SELECT c.lfn FROM ATLAS_PANDA.JEDI_Datasets d,ATLAS_PANDA.JEDI_Dataset_Contents c "
+                "WHERE c.jediTaskID=d.jediTaskID AND c.datasetID=d.datasetID AND "
+                "d.type=:t1 AND c.status=:s AND d.datasetName=:name ",
+                {":s": "finished", ":t1": "output", ":name": dsName},
+            )
+            for (tmpLFN,) in fo:
+                if tmpLFN not in files_rucio:
+                    ds_files.setdefault(options.ds, [])
+                    ds_files[options.ds].append(tmpLFN)
+            # get taskID
+            td, to = taskBuffer.querySQLS(
+                "SELECT jediTaskID FROM ATLAS_PANDA.JEDI_Datasets " "WHERE datasetName=:datasetName AND type=:t1 ",
+                {":t1": "output", ":datasetName": dsName},
+            )
+            (jediTaskID,) = to[0]
+        else:
+            # get dataset names
+            dd, do = taskBuffer.querySQLS(
+                "SELECT datasetName FROM ATLAS_PANDA.JEDI_Datasets " "WHERE jediTaskID=:jediTaskID AND type=:t1 ",
+                {":t1": "output", ":jediTaskID": options.jediTaskID},
+            )
+            # get files from rucio
+            files_rucio = set()
+            for (tmpDS,) in do:
+                st, tmp_files_rucio = get_files_from_rucio(tmpDS)
+                if st is None:
+                    print_msg(tmp_files_rucio, log_stream, is_error=True, put_log=log_filename)
+                    return st, tmp_files_rucio
+                # ignore unknown dataset
+                if st:
+                    files_rucio = files_rucio.union(tmp_files_rucio)
+            # get files from PanDA
+            fd, fo = taskBuffer.querySQLS(
+                "SELECT d.datasetName,c.lfn FROM ATLAS_PANDA.JEDI_Datasets d,ATLAS_PANDA.JEDI_Dataset_Contents c "
+                "WHERE d.jediTaskID=:jediTaskID AND c.jediTaskID=d.jediTaskID AND c.datasetID=d.datasetID AND "
+                "d.type=:t1 AND c.status=:s ",
+                {
+                    ":s": "finished",
+                    ":t1": "output",
+                    ":jediTaskID": options.jediTaskID,
+                },
+            )
+            for tmpDS, tmpLFN in fo:
+                if tmpLFN not in files_rucio:
+                    ds_files.setdefault(tmpDS, [])
+                    ds_files[tmpDS].append(tmpLFN)
+        for tmpDS in ds_files:
+            files = ds_files[tmpDS]
+            msg_str = f"{tmpDS} has {len(files)} lost files -> {','.join(files)}"
+            print_msg(msg_str, log_stream)
+
+    # no lost files
+    if not ds_files and not options.force:
+        msg_str = "No lost files. Use --force to ignore this check"
+        print_msg(msg_str, log_stream, put_log=log_filename)
+        return True, msg_str
+
+    # reset file status
+    s = False
+    for tmpDS in ds_files:
+        files = ds_files[tmpDS]
+        if not is_production_manager:
+            ts, jediTaskID, lostInputFiles, error_message = taskBuffer.resetFileStatusInJEDI(dn, False, tmpDS, files, options.reproduceParent, options.dryRun)
+        else:
+            ts, jediTaskID, lostInputFiles, error_message = taskBuffer.resetFileStatusInJEDI("", True, tmpDS, files, options.reproduceParent, options.dryRun)
+        if error_message:
+            msg_str = f"Failed to reset file status in the DB since {error_message}"
+            print_msg(msg_str, log_stream, is_error=True)
+        else:
+            msg_str = f"Reset file status for {tmpDS} in the DB: done with {ts} for jediTaskID={jediTaskID}"
+            print_msg(msg_str, log_stream)
+        s |= ts
+        # recover provenance
+        if options.reproduceParent:
+            # reproduce input
+            for lostDS in lostInputFiles:
+                com_args = ["--ds", lostDS, "--noChildRetry", "--resurrectDS"]
+                if options.reproduceUptoNthGen > 0:
+                    com_args += [
+                        "--reproduceUptoNthGen",
+                        str(options.reproduceUptoNthGen),
+                    ]
+                if options.dryRun:
+                    com_args.append("--dryRun")
+                com_args += ["--files", ",".join(lostInputFiles[lostDS])]
+            main(taskBuffer=taskBuffer, log_stream=log_stream, args_list=com_args)
+
+    # go ahead
+    if options.dryRun:
+        msg_str = f"Done in the dry-run mode with {s}"
+        print_msg(msg_str, log_stream, put_log=log_filename)
+        return True, msg_str
+    if s or options.force:
+        if options.resurrectDS:
+            sd, so = taskBuffer.querySQLS(
+                "SELECT datasetName FROM ATLAS_PANDA.JEDI_Datasets WHERE jediTaskID=:id AND type IN (:t1,:t2)",
+                {":id": jediTaskID, ":t1": "output", ":t2": "log"},
+            )
+            rc = RucioClient()
+            for (datasetName,) in so:
+                for i in range(3):
+                    try:
+                        scope, name = rucioAPI.extract_scope(datasetName)
+                        rc.get_did(scope, name)
+                        break
+                    except DataIdentifierNotFound:
+                        msg_str = f"resurrect {datasetName}"
+                        print_msg(msg_str, log_stream)
+                        rc.resurrect([{"scope": scope, "name": name}])
+                        try:
+                            rc.set_metadata(scope, name, "lifetime", None)
+                        except Exception:
+                            pass
+        if not options.reproduceParent:
+            tmp_ret = Client.retry_task(jediTaskID, no_child_retry=options.noChildRetry)
+        else:
+            tmp_ret = Client.reload_input(jediTaskID)
+        msg_str = f"Retried task {jediTaskID}: done with {tmp_ret}"
+        print_msg(msg_str, log_stream, put_log=log_filename)
+        return True, msg_str
+    else:
+        msg_str = "Failed"
+        print_msg(msg_str, log_stream, is_error=True, put_log=log_filename)
+        return False, msg_str