hpc-beeflow 0.1.11.dev1__tar.gz → 0.1.11.dev2__tar.gz

{hpc_beeflow-0.1.11.dev1 → hpc_beeflow-0.1.11.dev2}/PKG-INFO

@@ -1,8 +1,7 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.3
 Name: hpc-beeflow
-Version: 0.1.11.dev1
+Version: 0.1.11.dev2
 Summary: A software package for containerizing HPC applications and managing job workflows
-Home-page: https://github.com/lanl/BEE
 Keywords: bee,hpc,workflow,cluster,computing
 Author: BEE-LANL Dev Team
 Author-email: bee-dev@lanl.gov
@@ -19,7 +18,8 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Topic :: Scientific/Engineering
 Classifier: Topic :: System :: Clustering
@@ -43,7 +43,7 @@ Requires-Dist: jsonpickle (>=2.2.0,<3.0.0)
 Requires-Dist: neo4j (>=5,<6)
 Requires-Dist: networkx (==3.1)
 Requires-Dist: pre-commit (==3.5.0)
-Requires-Dist: pylint (==3.2.7)
+Requires-Dist: pylint (>=3.2.7,<4.0.0)
 Requires-Dist: python-daemon (>=2.3.1,<3.0.0)
 Requires-Dist: python-heatclient (>=3.1.0,<4.0.0) ; extra == "cloud-extras"
 Requires-Dist: python-openstackclient (>=6.0.0,<7.0.0) ; extra == "cloud-extras"
@@ -119,10 +119,12 @@ Contributors:
 * Tim Randles - `trandles-lanl <https://github.com/trandles-lanl>`_
 * Jacob Tronge - `jtronge <https://github.com/jtronge>`_
 * Kabir Vats - `kabir-vats <https://github.com/kabir-vats>`_
+* Sare Beste Zeytun - `sbzeytun <https://github.com/sbzeytun>`_
 
 Concept and Design Contributors
 
 * James Ahrens
+* Curt Canada 
 * Allen McPherson
 * Li-Ta Lo
 * Louis Vernon

{hpc_beeflow-0.1.11.dev1 → hpc_beeflow-0.1.11.dev2}/README.rst

@@ -63,10 +63,12 @@ Contributors:
 * Tim Randles - `trandles-lanl <https://github.com/trandles-lanl>`_
 * Jacob Tronge - `jtronge <https://github.com/jtronge>`_
 * Kabir Vats - `kabir-vats <https://github.com/kabir-vats>`_
+* Sare Beste Zeytun - `sbzeytun <https://github.com/sbzeytun>`_
 
 Concept and Design Contributors
 
 * James Ahrens
+* Curt Canada 
 * Allen McPherson
 * Li-Ta Lo
 * Louis Vernon

{hpc_beeflow-0.1.11.dev1 → hpc_beeflow-0.1.11.dev2}/beeflow/client/bee_client.py

@@ -231,7 +231,6 @@ def get_wf_list():
     if resp.status_code != requests.codes.okay:  # pylint: disable=no-member
         error_exit('WF Manager did not return workflow list')
 
-    logging.info('List Jobs:  {resp.text}')
     return jsonpickle.decode(resp.json()['workflow_list'])
 
 
@@ -275,10 +274,7 @@ def match_short_id(wf_id):
                          "workflow ID {matched_ids[0]}")
             long_wf_id = matched_ids[0]
             return long_wf_id
-    else:
-        sys.exit("There are currently no workflows.")
-
-    return None
+    sys.exit("There are currently no workflows.")
 
 
 def get_wf_status(wf_id):
@@ -324,6 +320,18 @@ def submit(wf_name: str = typer.Argument(..., help='the workflow name'),  # pyli
     wf_path = wf_path.resolve()
     workdir = workdir.resolve()
 
+    # Make sure the workdir is an absolute path and exists
+    workdir = os.path.expanduser(workdir)
+    workdir = os.path.abspath(workdir)
+    if not os.path.exists(workdir):
+        error_exit(f"Workflow working directory \"{workdir}\" doesn't exist")
+
+    # Make sure the workdir is not in /var or /var/tmp
+    if os.path.commonpath([os.path.realpath('/tmp'), workdir]) == os.path.realpath('/tmp'):
+        error_exit("Workflow working directory cannot be in \"/tmp\"")
+    if os.path.commonpath([os.path.realpath('/var/tmp'), workdir]) == os.path.realpath('/var/tmp'):
+        error_exit("Workflow working directory cannot be in \"/var/tmp\"")
+
     tarball_path = ""
     workflow = None
     wf_tarball = None
@@ -363,7 +371,7 @@ def submit(wf_name: str = typer.Argument(..., help='the workflow name'),  # pyli
         parser = CwlParser()
         workflow_id = generate_workflow_id()
         workflow, tasks = parser.parse_workflow(workflow_id, str(main_cwl_path),
-                                                job=str(yaml_path))
+                                                job=str(yaml_path), workdir=workdir)
         tasks = [jsonpickle.encode(task) for task in tasks]
 
         wf_tarball = open(package_path, 'rb')
@@ -373,18 +381,6 @@ def submit(wf_name: str = typer.Argument(..., help='the workflow name'),  # pyli
     else:
         error_exit(f'Workflow tarball {wf_path} cannot be found')
 
-    # Make sure the workdir is an absolute path and exists
-    workdir = os.path.expanduser(workdir)
-    workdir = os.path.abspath(workdir)
-    if not os.path.exists(workdir):
-        error_exit(f"Workflow working directory \"{workdir}\" doesn't exist")
-
-    # Make sure the workdir is not in /var or /var/tmp
-    if os.path.commonpath([os.path.realpath('/tmp'), workdir]) == os.path.realpath('/tmp'):
-        error_exit("Workflow working directory cannot be in \"/tmp\"")
-    if os.path.commonpath([os.path.realpath('/var/tmp'), workdir]) == os.path.realpath('/var/tmp'):
-        error_exit("Workflow working directory cannot be in \"/var/tmp\"")
-
     # TODO: Can all of this information be sent as a file?
     data = {
         'wf_name': wf_name.encode(),
@@ -558,8 +554,6 @@ def list_workflows():
     else:
         typer.echo("There are currently no workflows.")
 
-    logging.info('List workflows:  {resp.text}')
-
 
 @app.command()
 def query(wf_id: str = typer.Argument(..., callback=match_short_id)):

{hpc_beeflow-0.1.11.dev1 → hpc_beeflow-0.1.11.dev2}/beeflow/common/cwl/cwl.py

@@ -1,6 +1,7 @@
 """Create and manage CWL files."""
 from dataclasses import dataclass
 from io import StringIO
+from typing import Optional
 import ruamel.yaml
 
 # Create the global object for ruamel.ymal
@@ -399,6 +400,7 @@ class CheckpointRequirement:
     restart_parameters: str
     num_tries: int
     enabled: bool = True
+    add_parameters: Optional[str] = None
 
     def dump(self):
         """Dump beeflow requirement to a dictionary."""
@@ -409,6 +411,7 @@ class CheckpointRequirement:
         checkpoint_dump[req_name]['container_path'] = self.container_path
         checkpoint_dump[req_name]['file_regex'] = self.file_regex
         checkpoint_dump[req_name]['restart_parameters'] = self.restart_parameters
+        checkpoint_dump[req_name]['add_parameters'] = self.add_parameters
         checkpoint_dump[req_name]['num_tries'] = self.num_tries
         return checkpoint_dump
 
@@ -450,6 +453,27 @@ class ScriptRequirement:
         return stream.getvalue()
 
 
+@dataclass
+class TaskRequirement:
+    """Defines task requirement."""
+
+    workdir: str
+
+    def dump(self):
+        """Dump task requirement to a dictionary."""
+        key = 'beeflow:TaskRequirement'
+        this_dump = {key: {}}
+        if self.workdir:
+            this_dump[key]['workdir'] = self.workdir
+        return this_dump
+
+    def __repr__(self):
+        """Return TaskRequirement as a yaml string."""
+        stream = StringIO()
+        yaml.dump(self.dump(), stream)
+        return stream.getvalue()
+
+
 @dataclass
 class Hints:
     """Holds all the hints for a CWL workflow."""

{hpc_beeflow-0.1.11.dev1 → hpc_beeflow-0.1.11.dev2}/beeflow/common/cwl/workflow.py

@@ -6,7 +6,7 @@ from beeflow.common.cwl.cwl import (CWL, CWLInput, CWLInputs, RunInput, Inputs,
                                     Outputs, Run, RunOutput, Step, Steps, Hints,
                                     InputBinding, MPIRequirement, DockerRequirement,
                                     ScriptRequirement, SlurmRequirement,
-                                    CheckpointRequirement)
+                                    CheckpointRequirement, TaskRequirement)
 
 
 @dataclass
@@ -133,10 +133,20 @@ class Checkpoint(CheckpointRequirement):
                                      container_path=self.container_path,
                                      file_regex=self.file_regex,
                                      restart_parameters=self.restart_parameters,
+                                     add_parameters=self.add_parameters,
                                      num_tries=self.num_tries,
                                      enabled=self.enabled)
 
 
+@dataclass
+class TaskReq(TaskRequirement):
+    """Get Task Requirements."""
+
+    def requirement(self):
+        """Return a task requirement object."""
+        return TaskRequirement(workdir=self.workdir)
+
+
 @dataclass
 class Task:
     """Represents a task."""

{hpc_beeflow-0.1.11.dev1 → hpc_beeflow-0.1.11.dev2}/beeflow/common/parser/parser.py

@@ -10,6 +10,7 @@ import argparse
 import json
 import os
 import traceback
+from pathlib import Path
 import yaml
 import cwl_utils.parser.cwl_v1_2 as cwl_parser
 from schema_salad.exceptions import ValidationException
@@ -22,7 +23,8 @@ from beeflow.common.wf_data import (Workflow,
                                     StepOutput,
                                     Hint,
                                     Requirement,
-                                    generate_workflow_id)
+                                    generate_workflow_id,
+                                    get_requirement)
 
 # Map CWL types to Python types
 type_map = {
@@ -57,7 +59,7 @@ class CwlParser:
         self.steps = []
         self.params = None
 
-    def parse_workflow(self, workflow_id, cwl_path, job=None):
+    def parse_workflow(self, workflow_id, cwl_path, job=None, workdir='.'):
         """Parse a CWL Workflow file and load it into the graph database.
 
         Returns an instance of the WorkflowInterface.
@@ -117,11 +119,11 @@ class CwlParser:
 
         workflow = Workflow(workflow_name, workflow_hints, workflow_requirements, workflow_inputs,
                             workflow_outputs, workflow_id)
-        tasks = [self.parse_step(step, workflow_id) for step in self.cwl.steps]
+        tasks = [self.parse_step(step, workflow_id, workdir) for step in self.cwl.steps]
 
         return workflow, tasks
 
-    def parse_step(self, step, workflow_id):
+    def parse_step(self, step, workflow_id, workdir):
         """Parse a CWL step object.
 
         Calling this to parse a CommandLineTool file without a corresponding
@@ -153,17 +155,34 @@ class CwlParser:
         step_name = os.path.basename(step_id).split(".")[0]
         step_command = step_cwl.baseCommand
         step_inputs = self.parse_step_inputs(step.in_, step_cwl.inputs)
-        step_outputs = self.parse_step_outputs(step.out, step_cwl.outputs, step_cwl.stdout,
-                                               step_cwl.stderr)
         step_requirements = self.parse_requirements(step.requirements)
         step_requirements.extend(self.parse_requirements(step_cwl.requirements))
         step_hints = self.parse_requirements(step.hints, as_hints=True)
         step_hints.extend(self.parse_requirements(step_cwl.hints, as_hints=True))
+        step_workdir = resolve_step_workdir(
+            get_requirement(
+                step_requirements,
+                step_hints,
+                "beeflow:TaskRequirement",
+                "workdir",
+                 default=''
+            ),
+            workdir,
+        )
+
+        # When using container do not add the path to the output
+        # This can be changed when we fix binding for containers
+        if any(hint.class_ == 'DockerRequirement' for hint in step_hints):
+            step_outputs = self.parse_step_outputs(step.out, step_cwl.outputs, step_cwl.stdout,
+                                               step_cwl.stderr)
+        else:
+            step_outputs = self.parse_step_outputs(step.out, step_cwl.outputs, step_cwl.stdout,
+                                               step_cwl.stderr, workdir=step_workdir)
         step_stdout = step_cwl.stdout
         step_stderr = step_cwl.stderr
 
         return Task(step_name, step_command, step_hints, step_requirements, step_inputs,
-                    step_outputs, step_stdout, step_stderr, workflow_id)
+                    step_outputs, step_stdout, step_stderr, workflow_id, workdir=step_workdir)
 
     def parse_job(self, job):
         """Parse a CWL input job file.
@@ -229,11 +248,10 @@ class CwlParser:
                 inputs.append(StepInput(_shortname(step_input.id), input_type, None,
                                         step_input.default, source_map[_shortname(step_input.id)],
                                         None, None))
-
         return inputs
 
     @staticmethod
-    def parse_step_outputs(cwl_out, step_outputs, stdout, stderr):
+    def parse_step_outputs(cwl_out, step_outputs, stdout, stderr, workdir=None):
         """Parse step outputs from CWL step output objects.
 
         :param cwl_out: the step outputs from the Workflow file
@@ -280,8 +298,10 @@ class CwlParser:
                 if out_map[out].outputBinding:
                     glob = out_map[out].outputBinding.glob
 
-            outputs.append(StepOutput(out, output_type, None, glob))
+            if workdir is not None:
+                glob = os.path.abspath(os.path.join(workdir, glob))
 
+            outputs.append(StepOutput(out, output_type, None, glob))
         return outputs
 
     def _read_requirement_file(self, key, items):
@@ -339,7 +359,7 @@ class CwlParser:
             for req in requirements:
                 items = {}
                 for k, v in req.items():
-                    if k != 'class':
+                    if k != 'class' and v is not None:
                         if isinstance(v, (int, float)):
                             items[k] = v
                         else:
@@ -403,6 +423,18 @@ def parse_args(args=None):
     return parser.parse_args(args)
 
 
+def resolve_step_workdir(step_workdir, workdir):
+    """Resolve step workdir relative to workflow workdir."""
+    if not step_workdir or step_workdir.strip() == '':
+        return workdir
+    # should be redundant
+    workdir = Path(workdir).expanduser().resolve()
+    desired = Path(step_workdir).expanduser()
+    if desired.is_absolute():
+        return str(desired.resolve())
+    return str((workdir / desired).resolve())
+
+
 def main():
     """Run the parser on a CWL Workflow and job file directly."""
     wf_id = generate_workflow_id()

{hpc_beeflow-0.1.11.dev1 → hpc_beeflow-0.1.11.dev2}/beeflow/common/wf_data.py

@@ -96,6 +96,45 @@ class Workflow:
                 f"requirements = {self.requirements} inputs={self.inputs} outputs={self.outputs}>")
 
 
+def get_requirement(requirements, hints, req_type, req_param, default=None):
+    """Get requirement from hints or requirements, prioritizing requirements over hints.
+
+    :param requirements: list of requirements
+    :type requirements: list
+    :param hints: list of hints
+    :type hints: list
+    :param req_type: the type of requirement (e.g. 'DockerRequirement')
+    :type req_type: str
+    :param req_param: the requirement parameter (e.g. 'dockerFile')
+    :type req_param: str
+    :param default: default value if the requirement is not found
+    :type default: any
+
+    When requirements are specified hints will be ignored.
+    By default, tasks need not specify hints or requirements
+    """
+    requirements = dict(requirements)
+    requirement = default
+    # Get value if specified in requirements
+    try:
+        # Try to get Requirement
+        requirement = requirements[req_type][req_param]
+    except (KeyError, TypeError):
+        # Task Requirements are not mandatory. No docker_req_param specified in task reqs.
+        requirement = None
+    # Ignore hints if requirements available
+    if not requirement:
+        hints = dict(hints)
+        # Get value if specified in hints
+        try:
+            # Try to get Hints
+            requirement = hints[req_type][req_param]
+        except (KeyError, TypeError):
+            # Task Hints are not mandatory. No docker_req_param specified in task hints.
+            requirement = default
+    return requirement
+
+
 class Task:
     """Data structure for holding data about a single task."""
 
@@ -180,26 +219,7 @@ class Task:
         When requirements are specified hints will be ignored.
         By default, tasks need not specify hints or requirements
         """
-        requirements = dict(self.requirements)
-        requirement = default
-        # Get value if specified in requirements
-        try:
-            # Try to get Requirement
-            requirement = requirements[req_type][req_param]
-        except (KeyError, TypeError):
-            # Task Requirements are not mandatory. No docker_req_param specified in task reqs.
-            requirement = None
-        # Ignore hints if requirements available
-        if not requirement:
-            hints = dict(self.hints)
-            # Get value if specified in hints
-            try:
-                # Try to get Hints
-                requirement = hints[req_type][req_param]
-            except (KeyError, TypeError):
-                # Task Hints are not mandatory. No docker_req_param specified in task hints.
-                requirement = default
-        return requirement
+        return get_requirement(self.requirements, self.hints, req_type, req_param, default)
 
     def get_full_requirement(self, req_type):
         """Get the full requirement (or hint) for this task, if it has one.
@@ -330,6 +350,8 @@ class Task:
                         os.getenv('HOME'): os.path.join('/home', os.getenv('USER')),
                     }
                     command.append(convert_path(checkpoint_file, bind_mounts))
+                    if "add_parameters" in hint.params:
+                        command.append(hint.params["add_parameters"])
                 break
 
         return command

{hpc_beeflow-0.1.11.dev1 → hpc_beeflow-0.1.11.dev2}/beeflow/common/wf_interface.py

@@ -121,12 +121,12 @@ class WorkflowInterface:
         # Pattern match on task name
         # Append (1) if not in name already
         # Increment number on each restart
-        match = re.match(r".+\(([0-9]+)\)$", new_task.name)
+        match = re.search(r"-(\d+)$", new_task.name)
         if not match:
-            new_task.name += "(1)"
+            new_task.name += "-1"
         else:
             i = int(match.group(1))
-            new_task.name = re.sub(r"\([0-9]+\)", f"({i + 1})", new_task.name)
+            new_task.name = re.sub(r"-(\d+)$", f"-{i + 1}", new_task.name)
         metadata = self.get_task_metadata(task)
         self._gdb_driver.restart_task(task, new_task)
         self.set_task_metadata(new_task, metadata)

{hpc_beeflow-0.1.11.dev1 → hpc_beeflow-0.1.11.dev2}/beeflow/common/worker/worker.py

@@ -54,28 +54,37 @@ class Worker(ABC):
         self.workdir = bee_workdir
 
     def resolve_stdout_stderr(self, task):
-        """Reolves the path to the stderr and stdout in the task workdir."""
+        """Resolves the path to the stderr and stdout in the task workdir."""
+        task_dir = f"{task.workdir}/{task.name}-{task.id[:4]}"
+        os.makedirs(task_dir,exist_ok=True)
         if task.stdout:
             stdout_path = f"{task.workdir}/{task.stdout}"
         else:
             # If user provide no stdout or stderr name use this as a fallback
-            stdout_path = f"{task.workdir}/{task.name}-{task.id[:4]}.out"
+            stdout_path = f"{task.workdir}/{task.name}-{task.id[:4]}/{task.name}-{task.id[:4]}.out"
         if task.stderr:
             stderr_path = f"{task.workdir}/{task.stderr}"
         else:
-            stderr_path = f"{task.workdir}/{task.name}-{task.id[:4]}.err"
+            stderr_path = f"{task.workdir}/{task.name}-{task.id[:4]}/{task.name}-{task.id[:4]}.err"
         return stdout_path, stderr_path
 
 
     def task_save_path(self, task):
         """Return the task save path used for storing submission scripts output logs."""
-        return f'{self.workdir}/workflows/{task.workflow_id}/{task.name}-{task.id}'
+        return f'{self.workdir}/workflows/{task.workflow_id}/{task.name}-{task.id[:4]}'
 
     def write_script(self, task):
         """Build task script; returns filename of script."""
         task_text = self.build_text(task)
-        task_script_archive = f"{self.task_save_path(task)}/{task.name}-{task.id}.sh"
-        task_script_workdir = f"{task.workdir}/{task.name}-{task.id[:4]}.sh"
+        task_archive_dir = self.task_save_path(task)
+        os.makedirs(task_archive_dir,exist_ok=True)
+        task_script_archive = f"{self.task_save_path(task)}"\
+                f"/{task.name}-{task.id[:4]}.sh"
+        task_script_dir = f"{task.workdir}/{task.name}-{task.id[:4]}"
+        os.makedirs(task_script_dir,exist_ok=True)
+        task_script_workdir = f"{task.workdir}/{task.name}-{task.id[:4]}"\
+                f"/{task.name}-{task.id[:4]}.sh"
+
         with open(task_script_workdir, 'w', encoding="UTF-8") as workdir_script, \
              open(task_script_archive, 'w', encoding="UTF-8") as archive_script:
             workdir_script.write(task_text)
@@ -85,6 +94,7 @@ class Worker(ABC):
     def prepare(self, task):
         """Prepare for the task; create the task save directory, etc."""
         task_save_path = self.task_save_path(task)
+        os.makedirs(task.workdir, exist_ok=True)
         os.makedirs(task_save_path, exist_ok=True)
 
     @abstractmethod

{hpc_beeflow-0.1.11.dev1 → hpc_beeflow-0.1.11.dev2}/beeflow/tests/cwl_files/clamr.cwl

@@ -85,6 +85,7 @@ steps:
         container_path: checkpoint_output
         file_regex: backup[0-9]*.crx
         restart_parameters: -R
+        add_parameters:
         num_tries: 3
       beeflow:SlurmRequirement:
         timeLimit: 00:00:10

hpc_beeflow-0.1.11.dev2/beeflow/tests/cwl_files/task-req.cwl

@@ -0,0 +1,57 @@
+cwlVersion: v1.0
+class: Workflow
+
+inputs:
+  input_file: File
+  word: string
+
+outputs:
+  cat_stderr:
+    type: File
+    outputSource: cat/cat_stderr
+
+steps:
+  cat:
+    run:
+      class: CommandLineTool
+      baseCommand: cat
+      stdout: cat.txt
+      stderr: cat.err
+      inputs:
+        input_file:
+          type: File
+          inputBinding:
+            position: 1
+      outputs:
+        contents:
+          type: stdout
+        cat_stderr:
+          type: stderr
+    in:
+      input_file: input_file
+    out: [contents, cat_stderr]
+    hints:
+      beeflow:TaskRequirement:
+        workdir: cat_workdir
+  grep:
+    run:
+      class: CommandLineTool
+      baseCommand: grep
+      stdout: occur.txt
+      inputs:
+        word:
+          type: string
+          inputBinding:
+            position: 1
+        text_file:
+          type: File
+          inputBinding:
+            position: 2
+      outputs:
+        occur:
+          type: stdout
+    in:
+      word: word
+      text_file: cat/contents
+    out: [occur]
+

hpc_beeflow-0.1.11.dev2/beeflow/tests/cwl_files/task-req.yml

@@ -0,0 +1,3 @@
+input_file: lorem.txt
+word: Vivamus
+