panda-server-source 0.8.0__py3-none-any.whl

pandaserver/workflow/snakeparser/log.py

@@ -0,0 +1,53 @@
+__author__ = "retmas"
+
+import logging
+import logging.config
+import logging.handlers
+
+
+class Singleton(type):
+    def __init__(cls, *args, **kwargs):
+        super(Singleton, cls).__init__(*args, **kwargs)
+        cls._instance = None
+
+    def __call__(cls, *args, **kwargs):
+        if cls._instance is None:
+            cls._instance = super(Singleton, cls).__call__(*args, **kwargs)
+        return cls._instance
+
+
+class Logger(object, metaclass=Singleton):
+    _config = {
+        "version": 1,
+        "disable_existing_loggers": False,
+        "formatters": {"default": {"format": "[%(asctime)s] [%(process)d] [%(levelname)s] [%(module)s] [%(funcName)s:%(lineno)d] - %(message)s"}},
+        "handlers": {
+            "console": {
+                "level": "DEBUG",
+                "class": "logging.StreamHandler",
+                "formatter": "default",
+            }
+        },
+        "loggers": {"snakeparser.log": {"handlers": ["console"], "level": "INFO"}},
+    }
+
+    def __init__(self):
+        logging.config.dictConfig(self._config)
+
+    @staticmethod
+    def set_level(level):
+        Logger.get().setLevel(level)
+
+    @staticmethod
+    def set_log_file(filename, max_bytes=0, backup_count=0, remove_console=False):
+        logger = Logger().get()
+        handler = logging.handlers.RotatingFileHandler(filename, maxBytes=max_bytes, backupCount=backup_count, encoding="utf-8")
+        handler.formatter = logger.handlers[0].formatter
+        handler.level = logging.DEBUG
+        logger.addHandler(handler)
+        if remove_console:
+            logger.removeHandler(logger.handlers[0])
+
+    @staticmethod
+    def get() -> logging.Logger:
+        return logging.getLogger(__name__)

pandaserver/workflow/snakeparser/names.py

@@ -0,0 +1,3 @@
+__author__ = "retmas"
+
+WORKFLOW_NAMES = ["prun", "phpo", "junction", "reana"]

pandaserver/workflow/snakeparser/parser.py

@@ -0,0 +1,332 @@
+__author__ = "retmas"
+
+import logging
+import os
+import os.path
+import pathlib
+import re
+from itertools import chain
+from pathlib import Path
+from types import SimpleNamespace
+
+from snakemake.api import (
+    OutputSettings,
+    ResourceSettings,
+    SnakemakeApi,
+    StorageSettings,
+)
+
+from pandaserver.workflow.snakeparser.utils import ParamRule, param_of
+from pandaserver.workflow.workflow_utils import ConditionItem, Node
+
+from .extensions import inject
+from .log import Logger
+from .names import WORKFLOW_NAMES
+
+
+class ParamNotFoundException(Exception):
+    def __init__(self, param_name, rule_name):
+        super().__init__(f"Parameter {param_name} is not found in {rule_name} rule")
+
+
+class UnknownWorkflowTypeException(Exception):
+    def __init__(self, rule_name):
+        super().__init__(f"Unknown workflow type for rule {rule_name}")
+
+
+class UnknownRuleShellException(Exception):
+    def __init__(self, rule_name):
+        super().__init__(f"Unknown shellcmd for rule {rule_name}")
+
+
+class NoRuleShellException(Exception):
+    def __init__(self, rule_name):
+        super().__init__(f"No shellcmd for rule {rule_name}")
+
+
+class UnknownConditionTokenException(Exception):
+    def __init__(self, token):
+        super().__init__(f"Unknown token {token}")
+
+
+# noinspection DuplicatedCode
+class Parser(object):
+    def __init__(self, workflow_file, level=None, logger=None):
+        self._workflow = None
+        self._dag = None
+        if logger:
+            self._logger = logger
+        else:
+            self._logger = Logger().get()
+            if level:
+                self._logger.setLevel(level)
+        snakefile = os.path.abspath(workflow_file)
+        workdir = os.path.dirname(snakefile)
+        self._logger.debug("create workflow")
+        # create workflow through API
+        with SnakemakeApi(
+            OutputSettings(
+                verbose=False,
+                show_failed_logs=True,
+            ),
+        ) as snakemake_api:
+            workflow_api = snakemake_api.workflow(
+                storage_settings=StorageSettings(),
+                resource_settings=ResourceSettings(),
+                snakefile=Path(snakefile),
+            )
+            dag_api = workflow_api.dag()
+        self._workflow = workflow_api._workflow
+        self._workflow.default_target = "all"
+        self._workflow.overwrite_workdir = None
+        current_workdir = os.getcwd()
+        try:
+            inject()
+            self._workflow.workdir(workdir)
+        finally:
+            if current_workdir:
+                os.chdir(current_workdir)
+        # build DAG
+        dag_api.unlock()
+        self._dag = self._workflow.dag
+
+    @property
+    def jobs(self):
+        if self._dag is None:
+            return list()
+        return self._dag.jobs
+
+    def parse_nodes(self, in_loop=False):
+        try:
+            return self._parse_nodes(in_loop)
+        except NoRuleShellException as ex:
+            self._logger.error(str(ex))
+            raise ex
+        except UnknownRuleShellException as ex:
+            self._logger.error(str(ex))
+            raise ex
+        except UnknownWorkflowTypeException as ex:
+            self._logger.error(str(ex))
+            raise ex
+        except ParamNotFoundException as ex:
+            self._logger.error(str(ex))
+            raise ex
+
+    def _parse_nodes(self, in_loop):
+        root_job = next(filter(lambda o: o.rule.name == self._workflow.default_target, self.jobs))
+        root_inputs = {Parser._extract_job_id(self._define_id(name)): value for name, value in root_job.params.items()}
+        root_outputs = set(
+            [
+                Parser._extract_job_id(s.id.split("#")[0] + "#" + re.sub(s.id + "/", "", s.outputSource))
+                for s in list(
+                    chain(
+                        *[
+                            map(
+                                lambda output: Parser._define_object(
+                                    {
+                                        "id": self._define_id(output),
+                                        "outputSource": self._define_id(f"{dep.name}/{output}"),
+                                    }
+                                ),
+                                dep.output,
+                            )
+                            for dep in self._dag.dependencies[root_job]
+                        ]
+                    )
+                )
+            ]
+        )
+        node_list = []
+        output_map = {}
+        serial_id = 0
+        for job in filter(lambda o: o.name != self._workflow.default_target, self.jobs):
+            if job.is_shell:
+                if job.shellcmd is None:
+                    raise NoRuleShellException(job.rule.name)
+            else:
+                raise UnknownRuleShellException(job.rule.name)
+            workflow_name = os.path.basename(job.shellcmd)
+            serial_id += 1
+            if workflow_name in WORKFLOW_NAMES:
+                node = Node(serial_id, workflow_name, None, True, job.name)
+            elif workflow_name.lower() == "Snakefile".lower():
+                node = Node(serial_id, "workflow", None, False, job.name)
+            else:
+                raise UnknownWorkflowTypeException(job.rule.name)
+            for name, value in job.params.items():
+                if isinstance(value, ParamRule):
+                    if value.rule is None:
+                        param_job = job
+                    else:
+                        param_job = next(filter(lambda o: o.rule.name == value.rule.name, self.jobs))
+                    if value.name not in param_job.rule.params.keys():
+                        raise ParamNotFoundException(value.name, param_job.rule.name)
+                    source = Parser._extract_job_id(self._define_id(f"{param_job.name}/{value.name}"))
+                    if param_job.name == self._workflow.default_target:
+                        source = source.replace(f"{param_job.name}/", "")
+                    node.inputs.update({Parser._extract_job_id(self._define_id(f"{job.name}/{name}")): {"default": None, "source": source}})
+                    continue
+                if node.inputs is None:
+                    node.inputs = dict()
+                default_value = value
+                source = None
+                if job.dependencies:
+                    if isinstance(value, list):
+                        for item in list(value):
+                            if item in job.dependencies.keys():
+                                if source is None:
+                                    source = list()
+                                source.append(Parser._extract_job_id(self._define_id(f"{job.dependencies[item].name}/{item}")))
+                        if source is not None:
+                            default_value = None
+                    else:
+                        if default_value in job.dependencies.keys():
+                            source = Parser._extract_job_id(self._define_id(f"{job.dependencies[default_value].name}/{default_value}"))
+                            default_value = None
+                node.inputs.update(
+                    {
+                        Parser._extract_job_id(self._define_id(f"{job.name}/{name}")): {
+                            "default": default_value,
+                            "source": source,
+                        }
+                    }
+                )
+            node.outputs = {Parser._extract_job_id(self._define_id(f"{job.name}/{output}")): {} for output in job.output}
+            if not node.outputs:
+                node.outputs = {Parser._extract_job_id(self._define_id(f"{job.name}/outDS")): {}}
+            output_map.update({name: serial_id for name in node.outputs})
+            scatter = getattr(job.rule, "scatter", None)
+            if scatter:
+                node.scatter = [
+                    Parser._extract_job_id(self._define_id(f"{next(filter(lambda o: o.rule.name == v.rule.name, self.jobs)).name}/{v.name}"))
+                    for v in [ParamRule(v.name, job.rule if v.rule is None else v.rule) for v in job.rule.scatter]
+                ]
+            else:
+                node.scatter = None
+            condition = getattr(job.rule, "condition", None)
+            if condition:
+                pattern = f'{param_of("")}(\w+)'
+                for param_name in re.findall(pattern, condition):
+                    if param_name not in job.rule.params.keys():
+                        raise ParamNotFoundException(param_name, job.rule.name)
+                condition = re.sub(r" *! *", r"!", condition)
+                condition = re.sub(r"\|\|", r" || ", condition)
+                condition = re.sub(r"&&", r" && ", condition)
+                tokens = condition.split()
+                param_left = None
+                param_operator = None
+                for token in tokens:
+                    token = token.strip()
+                    if token == "||":
+                        param_operator = "or"
+                        continue
+                    elif token == "&&":
+                        param_operator = "and"
+                        continue
+                    elif token.startswith(str(param_of(""))):
+                        param_right = ConditionItem(str(re.findall(pattern, token)[0]))
+                        if not param_left:
+                            param_left = param_right
+                            continue
+                    elif token.startswith(f'!{param_of("")}'):
+                        param_right = ConditionItem(str(re.findall(pattern, token)[0]), operator="not")
+                        if not param_left:
+                            param_left = param_right
+                            continue
+                    else:
+                        raise UnknownConditionTokenException(token)
+                    param_left = ConditionItem(param_left, param_right, param_operator)
+                node.condition = param_left
+                self._suppress_inputs(node.condition, node.inputs)
+            node.loop = bool(getattr(job.rule, "loop", False))
+            if node.loop or in_loop:
+                node.in_loop = True
+            if not node.is_leaf:
+                subworkflow_file = os.path.join(self._workflow.basedir, job.shellcmd)
+                subworkflow_parser = Parser(subworkflow_file, level=logging.DEBUG)
+                node.sub_nodes, node.root_inputs = subworkflow_parser.parse_nodes()
+            if root_outputs & set(node.outputs):
+                node.is_tail = True
+            node_list.append(node)
+        for node in node_list:
+            for name, data in node.inputs.items():
+                if not data["source"]:
+                    continue
+                if isinstance(data["source"], list):
+                    sources = data["source"]
+                    is_str = False
+                else:
+                    sources = [data["source"]]
+                    is_str = True
+                parent_id_list = list()
+                for source in sources:
+                    if source in output_map:
+                        parent_id = output_map[source]
+                        node.add_parent(parent_id)
+                        parent_id_list.append(parent_id)
+                if parent_id_list:
+                    if is_str:
+                        parent_id_list = parent_id_list[0]
+                    data["parent_id"] = parent_id_list
+        node_list = self._sort_node_list(node_list, set())
+        return node_list, root_inputs
+
+    def verify_workflow(self):
+        if self._workflow is None:
+            return False
+        self._workflow.check()
+        return True
+
+    def get_dot_data(self):
+        return str(self._dag)
+
+    @staticmethod
+    def _extract_job_id(job_id):
+        if not job_id:
+            return job_id
+        if not isinstance(job_id, list):
+            job_id = [job_id]
+            not_list = True
+        else:
+            not_list = False
+        items = [re.search(r"[^/]+#.+$", s).group(0) for s in job_id]
+        if not_list:
+            return items[0]
+        return items
+
+    @staticmethod
+    def _sort_node_list(node_list, visited):
+        if not node_list:
+            return []
+        new_list = []
+        new_visited = []
+        for node in node_list:
+            is_ok = True
+            for p in node.parents:
+                if p not in visited:
+                    is_ok = False
+                    break
+            if is_ok:
+                new_visited.append(node)
+                visited.add(node.id)
+            else:
+                new_list.append(node)
+        return new_visited + Parser._sort_node_list(new_list, visited)
+
+    def _define_id(self, name):
+        return f"{pathlib.Path(os.path.abspath(self._workflow.main_snakefile)).as_uri()}#{name}"
+
+    @staticmethod
+    def _define_object(dict_):
+        return SimpleNamespace(**dict_)
+
+    def _suppress_inputs(self, condition: ConditionItem, inputs):
+        if condition.right is None and condition.operator == "not" and isinstance(condition.left, str):
+            for name, data in inputs.items():
+                if condition.left == name.split("/")[-1]:
+                    data["suppressed"] = True
+        else:
+            for item in ["left", "right"]:
+                condition_item = getattr(condition, item)
+                if isinstance(condition_item, ConditionItem):
+                    self._suppress_inputs(condition_item, inputs)

pandaserver/workflow/snakeparser/utils.py

@@ -0,0 +1,29 @@
+__author__ = "retmas"
+
+from snakemake.workflow import Rule, RuleProxy
+
+
+class ParamRule(object):
+    def __init__(self, name: str, source_rule: Rule = None):
+        self._name = name
+        self._source_rule = source_rule
+
+    def __repr__(self):
+        return f"param.{self.name}"
+
+    @property
+    def name(self) -> str:
+        return self._name
+
+    @property
+    def rule(self) -> Rule:
+        return self._source_rule
+
+
+def param_of(name, source: RuleProxy = None) -> ParamRule:
+    rule = source.rule if source is not None else None
+    return ParamRule(name, rule)
+
+
+def param_exp(template):
+    return lambda wildcards: template

pandaserver/workflow/step_handler_plugins/base_step_handler.py

@@ -0,0 +1,85 @@
+from pandaserver.workflow.workflow_base import (
+    WFDataSpec,
+    WFDataStatus,
+    WFDataType,
+    WFStepSpec,
+    WFStepStatus,
+    WFStepTargetCancelResult,
+    WFStepTargetCheckResult,
+    WFStepTargetSubmitResult,
+    WFStepType,
+    WorkflowSpec,
+    WorkflowStatus,
+)
+
+
+class BaseStepHandler:
+    """
+    Base class for step handlers in the workflow.
+    This class provides a common interface and some utility methods for step handlers.
+    """
+
+    def __init__(self, task_buffer, *args, **kwargs):
+        """
+        Initialize the step handler with necessary parameters.
+
+        Args:
+            task_buffer: The task buffer interface to interact with the task database.
+            *args: Additional positional arguments.
+            **kwargs: Additional keyword arguments.
+        """
+        self.tbif = task_buffer
+
+    def submit_target(self, step_spec: WFStepSpec, **kwargs) -> WFStepTargetSubmitResult:
+        """
+        Submit a target for processing the step.
+        This method should be implemented by subclasses to handle the specifics of target submission.
+        This method should NOT modify step_spec. Any update information should be stored in the WFStepTargetSubmitResult returned instead.
+
+        Args:
+            step_spec (WFStepSpec): Specifications of the workflow step whose target is to be submitted.
+
+        Returns:
+            WFStepTargetSubmitResult: An object containing the result of the submission, including success status, target ID, and message.
+
+        """
+        raise NotImplementedError("Subclasses must implement this method.")
+
+    def check_target(self, step_spec: WFStepSpec, **kwargs) -> WFStepTargetCheckResult:
+        """
+        Check the status of the submitted target.
+        This method should be implemented by subclasses to handle the specifics of target status checking.
+        This method should NOT modify step_spec. Any update information should be stored in the WFStepTargetCheckResult returned instead.
+
+        Args:
+            step_spec (WFStepSpec): Specifications of the workflow step to be checked.
+
+        Returns:
+            WFStepTargetCheckResult: An object containing the result of the check, including success status, current step status, and message.
+        """
+        raise NotImplementedError("Subclasses must implement this method.")
+
+    def on_all_inputs_done(self, step_spec: WFStepSpec, **kwargs) -> None:
+        """
+        Hook method called when all inputs for the step are done.
+        This method can be overridden by subclasses to perform actions when all inputs are ready.
+
+        Args:
+            step_spec (WFStepSpec): Specifications of the workflow step whose inputs are done.
+            **kwargs: Additional keyword arguments.
+        """
+        raise NotImplementedError("Subclasses must implement this method.")
+
+    def cancel_target(self, step_spec: WFStepSpec, **kwargs) -> WFStepTargetCancelResult:
+        """
+        Cancel the submitted target.
+        This method can be overridden by subclasses to handle target cancellation.
+
+        Args:
+            step_spec (WFStepSpec): Specifications of the workflow step whose target is to be cancelled.
+            **kwargs: Additional keyword arguments.
+
+        Returns:
+            WFStepTargetCancelResult: An object containing the result of the cancellation, including success status and message.
+        """
+        raise NotImplementedError("Subclasses must implement this method.")