panda-server-source 0.8.0__py3-none-any.whl

pandaserver/workflow/data_handler_plugins/base_data_handler.py

@@ -0,0 +1,48 @@
+from pandaserver.workflow.workflow_base import (
+    WFDataSpec,
+    WFDataStatus,
+    WFDataTargetCheckResult,
+    WFDataType,
+    WFStepSpec,
+    WFStepStatus,
+    WFStepTargetCheckResult,
+    WFStepTargetSubmitResult,
+    WFStepType,
+    WorkflowSpec,
+    WorkflowStatus,
+)
+
+
+class BaseDataHandler:
+    """
+    Base class for data handlers in the workflow system.
+    This class provides a common interface and some utility methods for data handlers.
+    """
+
+    def __init__(self, task_buffer, ddm_if, *args, **kwargs):
+        """
+        Initialize the step handler with necessary parameters.
+
+        Args:
+            task_buffer: The task buffer interface to interact with the task database.
+            ddm_if: The DDM interface to interact with the DDM system.
+            *args: Additional positional arguments.
+            **kwargs: Additional keyword arguments.
+        """
+        self.tbif = task_buffer
+        self.ddm_if = ddm_if
+
+    def check_target(self, data_spec: WFDataSpec, **kwargs) -> WFDataTargetCheckResult:
+        """
+        Check the status of the data target.
+        This method should be implemented by subclasses to handle the specifics of data target status checking.
+        This method should NOT modify data_spec. Any update information should be stored in the WFStepTargetCheckResult returned instead.
+
+        Args:
+            data_spec (WFDataSpec): The data specification to check.
+            **kwargs: Additional keyword arguments.
+
+        Returns:
+            WFDataTargetCheckResult: The result of the target check.
+        """
+        raise NotImplementedError("Subclasses must implement this method.")

pandaserver/workflow/data_handler_plugins/ddm_collection_data_handler.py

@@ -0,0 +1,100 @@
+import json
+import traceback
+import uuid
+
+from pandacommon.pandalogger.LogWrapper import LogWrapper
+from pandacommon.pandalogger.PandaLogger import PandaLogger
+
+from pandaserver.workflow.data_handler_plugins.base_data_handler import BaseDataHandler
+from pandaserver.workflow.workflow_base import (
+    WFDataSpec,
+    WFDataStatus,
+    WFDataTargetCheckResult,
+    WFDataTargetCheckStatus,
+    WFDataType,
+    WFStepSpec,
+    WFStepStatus,
+    WFStepType,
+    WorkflowSpec,
+    WorkflowStatus,
+)
+
+# main logger
+logger = PandaLogger().getLogger(__name__.split(".")[-1])
+
+
+class DDMCollectionDIDType:
+    """
+    Data Identifier Types for DDM Collections
+    """
+
+    DATASET = "DATASET"
+    CONTAINER = "CONTAINER"
+
+
+class DDMCollectionState:
+    """
+    States for DDM Collections
+    """
+
+    open = "open"
+    closed = "closed"
+    missing = "missing"
+
+
+class DDMCollectionDataHandler(BaseDataHandler):
+    """
+    Handler for DDM collection data in the workflow.
+    This class is responsible for managing the DDM collection data within a workflow.
+    """
+
+    def __init__(self, *args, **kwargs):
+        """
+        Initialize the data handler with necessary parameters.
+        """
+        # Initialize base class or any required modules here
+        super().__init__(*args, **kwargs)
+        self.plugin_flavor = "ddm_collection"
+
+    def check_target(self, data_spec: WFDataSpec, **kwargs) -> WFDataTargetCheckResult:
+        """
+        Check the status of the DDM collection data target.
+        This method should be implemented to handle the specifics of DDM collection data status checking.
+
+        Args:
+            data_spec (WFDataSpec): The data specification containing details about the data to be checked.
+            **kwargs: Additional keyword arguments that may be required for checking.
+
+        Returns:
+            WFDataTargetCheckResult: An object containing the result of the check, including success status, current data status, and message.
+        """
+        tmp_log = LogWrapper(logger, f"check_target workflow_id={data_spec.workflow_id} data_id={data_spec.data_id}")
+        # Initialize
+        check_result = WFDataTargetCheckResult()
+        # Check data flavor
+        if data_spec.flavor != self.plugin_flavor:
+            tmp_log.warning(f"flavor={data_spec.flavor} not {self.plugin_flavor}; skipped")
+            check_result.message = f"flavor not {self.plugin_flavor}; skipped"
+            return check_result
+        # Check DDM collection status
+        collection = data_spec.target_id
+        collection_meta = self.ddm_if.get_dataset_metadata(collection, ignore_missing=True)
+        if collection_meta is None:
+            check_result.success = False
+            check_result.message = f"Failed to get metadata for collection {collection}"
+            tmp_log.error(f"{check_result.message}")
+            return check_result
+        match collection_meta.get("state"):
+            case DDMCollectionState.missing:
+                check_result.check_status = WFDataTargetCheckStatus.nonexist
+            case DDMCollectionState.open:
+                if collection_meta.get("length", 0) == 0:
+                    check_result.check_status = WFDataTargetCheckStatus.insuffi
+                else:
+                    check_result.check_status = WFDataTargetCheckStatus.suffice
+            case DDMCollectionState.closed:
+                check_result.check_status = WFDataTargetCheckStatus.complete
+        check_result.metadata = collection_meta
+        check_result.success = True
+        tmp_log.info(f"Got collection {collection} check_status={check_result.check_status}")
+        return check_result

pandaserver/workflow/data_handler_plugins/panda_task_data_handler.py

@@ -0,0 +1,140 @@
+import json
+import traceback
+import uuid
+
+from pandacommon.pandalogger.LogWrapper import LogWrapper
+from pandacommon.pandalogger.PandaLogger import PandaLogger
+
+from pandaserver.workflow.data_handler_plugins.base_data_handler import BaseDataHandler
+from pandaserver.workflow.workflow_base import (
+    WFDataSpec,
+    WFDataStatus,
+    WFDataTargetCheckResult,
+    WFDataTargetCheckStatus,
+    WFDataType,
+    WFStepSpec,
+    WFStepStatus,
+    WFStepType,
+    WorkflowSpec,
+    WorkflowStatus,
+)
+
+# Whether to consider partial data (e.g. some files in DDM collection) as sufficient for step input
+PARTIAL_DATA_SUFFICE = False
+
+# main logger
+logger = PandaLogger().getLogger(__name__.split(".")[-1])
+
+
+class DDMCollectionDIDType:
+    """
+    Data Identifier Types for DDM Collections
+    """
+
+    DATASET = "DATASET"
+    CONTAINER = "CONTAINER"
+
+
+class DDMCollectionState:
+    """
+    States for DDM Collections
+    """
+
+    open = "open"
+    closed = "closed"
+    missing = "missing"
+
+
+class PandaTaskDataHandler(BaseDataHandler):
+    """
+    Handler for PanDA task intermediate/output data in the workflow.
+    This class is responsible for managing the data generated by PanDA task within a workflow.
+    The output data from a PanDA task is usually a DDM container, which remains open even after the task completion.
+    Thus, the handler not only checks the status of the DDM collection to determine if there are files available, but also verifies the step status of source workflow steps to ensure that the data generation process has been completed successfully.
+    """
+
+    def __init__(self, *args, **kwargs):
+        """
+        Initialize the data handler with necessary parameters.
+        """
+        # Initialize base class or any required modules here
+        super().__init__(*args, **kwargs)
+        self.plugin_flavor = "panda_task"
+
+    def check_target(self, data_spec: WFDataSpec, **kwargs) -> WFDataTargetCheckResult:
+        """
+        Check the status of the PanDA task data target.
+        This method should be implemented to handle the specifics of PanDA task data status checking.
+
+        Args:
+            data_spec (WFDataSpec): The data specification containing details about the data to be checked.
+            **kwargs: Additional keyword arguments that may be required for checking.
+
+        Returns:
+            WFDataTargetCheckResult: An object containing the result of the check, including success status, current data status, and message.
+        """
+        tmp_log = LogWrapper(logger, f"check_target workflow_id={data_spec.workflow_id} data_id={data_spec.data_id}")
+        # Initialize
+        check_result = WFDataTargetCheckResult()
+        # Check data flavor
+        if data_spec.flavor != self.plugin_flavor:
+            tmp_log.warning(f"flavor={data_spec.flavor} not {self.plugin_flavor}; skipped")
+            check_result.message = f"flavor not {self.plugin_flavor}; skipped"
+            return check_result
+        # Check source step status
+        if data_spec.source_step_id is not None:
+            source_step_spec = self.tbif.get_workflow_step(data_spec.source_step_id)
+            if source_step_spec is None:
+                check_result.success = False
+                check_result.message = f"Failed to get source step step_id={data_spec.source_step_id}; skipped"
+                tmp_log.error(f"{check_result.message}")
+                return check_result
+            if source_step_spec.status == WFStepStatus.done:
+                # Source step done; consider data fully available
+                check_result.success = True
+                check_result.check_status = WFDataTargetCheckStatus.complete
+                tmp_log.info(f"Source step step_id={source_step_spec.step_id} done, data considered fully available; check_status={check_result.check_status}")
+                return check_result
+            elif source_step_spec.status in WFStepStatus.final_statuses:
+                # Source step in final status but not done; skip data availability
+                check_result.success = True
+                check_result.message = f"Source step step_id={source_step_spec.step_id} {source_step_spec.status}; skip data availability check"
+                tmp_log.warning(f"{check_result.message}")
+                return check_result
+            else:
+                tmp_log.info(f"Source step step_id={source_step_spec.step_id} status={source_step_spec.status}; checking data availability")
+        else:
+            tmp_log.info("No source step yet; checking data availability")
+        # Without source step or source step not terminated; check number of files in DDM collections
+        total_n_files = 0
+        none_exist = True
+        output_types = data_spec.get_parameter("output_types")
+        if output_types is None:
+            output_types = []
+        for output_type in output_types:
+            collection = f"{data_spec.target_id}_{output_type}"
+            tmp_stat, tmp_res = self.ddm_if.get_number_of_files(collection)
+            if tmp_stat is None:
+                tmp_log.debug(f"Collection {collection} does not exist")
+            elif not tmp_stat:
+                # Error in getting number of files
+                check_result.success = False
+                check_result.message = f"Failed to get number of files for collection {collection}: {tmp_res}"
+                tmp_log.error(f"{check_result.message}")
+                return check_result
+            else:
+                none_exist = False
+                n_files = tmp_res
+                total_n_files += n_files
+                tmp_log.debug(f"Got collection {collection} n_files={n_files}")
+        # Check number of files
+        if none_exist:
+            check_result.check_status = WFDataTargetCheckStatus.nonexist
+        elif PARTIAL_DATA_SUFFICE and total_n_files > 0:
+            # At least 1 file for step input
+            check_result.check_status = WFDataTargetCheckStatus.suffice
+        else:
+            check_result.check_status = WFDataTargetCheckStatus.insuffi
+        check_result.success = True
+        tmp_log.info(f"Got total_n_files={total_n_files}; check_status={check_result.check_status}")
+        return check_result

pandaserver/workflow/pcwl_test.py

@@ -0,0 +1,122 @@
+import logging
+import sys
+
+from pandaserver.workflow.pcwl_utils import parse_workflow_file, resolve_nodes
+from pandaserver.workflow.workflow_utils import (
+    convert_nodes_to_workflow,
+    dump_nodes,
+    get_node_id_map,
+    set_workflow_outputs,
+)
+from ruamel import yaml
+
+logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s", level=logging.DEBUG)
+with open(sys.argv[2]) as f:
+    data = yaml.safe_load(f) or dict()
+nodes, root_in = parse_workflow_file(sys.argv[1], logging)
+s_id, t_nodes, nodes = resolve_nodes(nodes, root_in, data, 0, set(), sys.argv[3], logging)
+set_workflow_outputs(nodes)
+id_map = get_node_id_map(nodes)
+# task template
+template = {
+    "buildSpec": {
+        "jobParameters": "-i ${IN} -o ${OUT} --sourceURL ${SURL} " "-r ./ --useAthenaPackages --useCMake --cmtConfig x86_64-slc6-gcc49-opt ",
+        "archiveName": "sources.5bca61db-3b0d-4e65-b102-7d56d82f4567.tar.gz",
+        "prodSourceLabel": "panda",
+    },
+    "sourceURL": "https://aipanda047.cern.ch:25443",
+    "cliParams": "prun --exec ls --useAthenaPackages --outDS user.tmaeno.4586bbc2-e891-441b-a9f4-4a9ceb9a9f8d",
+    "site": None,
+    "vo": "atlas",
+    "respectSplitRule": True,
+    "osInfo": "Linux-3.10.0-1160.36.2.el7.x86_64-x86_64-with-centos-7.9.2009-Core",
+    "log": {
+        "type": "template",
+        "param_type": "log",
+        "container": "user.tmaeno.4586bbc2-e891-441b-a9f4-4a9ceb9a9f8d.log/",
+        "value": "user.tmaeno.4586bbc2-e891-441b-a9f4-4a9ceb9a9f8d.log.$JEDITASKID.${SN}.log.tgz",
+        "dataset": "user.tmaeno.4586bbc2-e891-441b-a9f4-4a9ceb9a9f8d.log/",
+    },
+    "transUses": "Atlas-21.0.6",
+    "excludedSite": [],
+    "nMaxFilesPerJob": 200,
+    "uniqueTaskName": True,
+    "taskName": "user.tmaeno.4586bbc2-e891-441b-a9f4-4a9ceb9a9f8d/",
+    "transHome": "AnalysisTransforms",
+    "includedSite": None,
+    "jobParameters": [
+        {"type": "constant", "value": '-j "" --sourceURL ${SURL}'},
+        {"type": "constant", "value": "-r ./"},
+        {"padding": False, "type": "constant", "value": '-p "'},
+        {"padding": False, "type": "constant", "value": "ls"},
+        {"type": "constant", "value": '"'},
+        {"type": "constant", "value": "-l ${LIB}"},
+        {
+            "type": "constant",
+            "value": "--useAthenaPackages " "--useCMake --cmtConfig x86_64-slc6-gcc49-opt ",
+        },
+    ],
+    "prodSourceLabel": "user",
+    "processingType": "panda-client-1.4.81-jedi-run",
+    "architecture": "x86_64-slc6-gcc49-opt@centos7",
+}
+
+
+c_template = {
+    "sourceURL": "https://aipanda048.cern.ch:25443",
+    "cliParams": "prun --cwl test.cwl --yaml a.yaml " "--relayHost aipanda059.cern.ch --outDS " "user.tmaeno.1f2860f0-30d6-4352-9f87-9b9dde82fd80",
+    "site": None,
+    "vo": "atlas",
+    "respectSplitRule": True,
+    "osInfo": "Linux-3.10.0-1160.36.2.el7.x86_64-x86_64-with-centos-7.9.2009-Core",
+    "log": {
+        "type": "template",
+        "param_type": "log",
+        "container": "user.tmaeno.1f2860f0-30d6-4352-9f87-9b9dde82fd80.log/",
+        "value": "user.tmaeno.1f2860f0-30d6-4352-9f87-9b9dde82fd80.log.$JEDITASKID.${SN}.log.tgz",
+        "dataset": "user.tmaeno.1f2860f0-30d6-4352-9f87-9b9dde82fd80.log/",
+    },
+    "transUses": "",
+    "excludedSite": [],
+    "nMaxFilesPerJob": 200,
+    "uniqueTaskName": True,
+    "taskName": "user.tmaeno.1f2860f0-30d6-4352-9f87-9b9dde82fd80/",
+    "transHome": None,
+    "includedSite": None,
+    "container_name": "__dummy_container__",
+    "multiStepExec": {
+        "preprocess": {"args": "--preprocess ${TRF_ARGS}", "command": "${TRF}"},
+        "containerOptions": {
+            "containerImage": "__dummy_container__",
+            "containerExec": 'echo "=== cat exec script ==="; ' "cat __run_main_exec.sh; echo; " 'echo "=== exec script ==="; ' "/bin/sh __run_main_exec.sh",
+        },
+        "postprocess": {"args": "--postprocess ${TRF_ARGS}", "command": "${TRF}"},
+    },
+    "jobParameters": [
+        {"type": "constant", "value": '-j "" --sourceURL ${SURL}'},
+        {"type": "constant", "value": "-r ."},
+        {"padding": False, "type": "constant", "value": '-p "'},
+        {"padding": False, "type": "constant", "value": "__dummy_exec_str__"},
+        {"type": "constant", "value": '"'},
+        {
+            "type": "constant",
+            "value": "-a jobO.b6bdf294-8a46-4b02-b399-46619db4088b.tar.gz",
+        },
+    ],
+    "prodSourceLabel": "user",
+    "processingType": "panda-client-1.4.81-jedi-run",
+    "architecture": "",
+}
+
+task_template = {"athena": template, "container": c_template}
+
+[node.resolve_params(task_template, id_map) for node in nodes]
+print(dump_nodes(nodes))
+
+workflow, dump_str_list = convert_nodes_to_workflow(nodes)
+print("".join(dump_str_list))
+
+for node in nodes:
+    s, o = node.verify()
+    if not s:
+        print(f"Verification error in ID:{node.id} {o}")