hpc-beeflow 0.1.11.dev0__tar.gz → 0.1.11.dev1__tar.gz

{hpc_beeflow-0.1.11.dev0 → hpc_beeflow-0.1.11.dev1}/PKG-INFO

@@ -1,7 +1,8 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.1
 Name: hpc-beeflow
-Version: 0.1.11.dev0
+Version: 0.1.11.dev1
 Summary: A software package for containerizing HPC applications and managing job workflows
+Home-page: https://github.com/lanl/BEE
 Keywords: bee,hpc,workflow,cluster,computing
 Author: BEE-LANL Dev Team
 Author-email: bee-dev@lanl.gov
@@ -18,8 +19,7 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.12
-Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Topic :: Scientific/Engineering
 Classifier: Topic :: System :: Clustering

{hpc_beeflow-0.1.11.dev0 → hpc_beeflow-0.1.11.dev1}/beeflow/common/cwl/cwl.py

@@ -72,6 +72,7 @@ class InputBinding:
 
     prefix: str = None
     position: int = None
+    value_from: str = None
 
     def dump(self):
         """Dump returns dictionary that will be used by pyyaml dump."""
@@ -80,6 +81,8 @@ class InputBinding:
             binding_yaml['inputBinding']['position'] = self.position
         if self.prefix:
             binding_yaml['inputBinding']['prefix'] = self.prefix
+        if self.value_from:
+            binding_yaml['inputBinding']['valueFrom'] = self.value_from
         return binding_yaml
 
     def __repr__(self):
@@ -321,7 +324,7 @@ class DockerRequirement:
         if self.container_name:
             docker_dump['DockerRequirement']['beeflow:containerName'] = self.container_name
         if self.force_type:
-            docker_dump['DockerRequirement']['beeflow:force=Type'] = self.force_type
+            docker_dump['DockerRequirement']['beeflow:forceType'] = self.force_type
         return docker_dump
 
     def __repr__(self):
@@ -359,7 +362,7 @@ class SlurmRequirement:
     """Represents a beeflow custom MPI requirement."""
 
     account: str = None
-    time_limit: int = None
+    time_limit: str = None
     partition: str = None
     qos: str = None
     reservation: str = None
@@ -406,8 +409,8 @@ class CheckpointRequirement:
         checkpoint_dump[req_name]['container_path'] = self.container_path
         checkpoint_dump[req_name]['file_regex'] = self.file_regex
         checkpoint_dump[req_name]['restart_parameters'] = self.restart_parameters
-        checkpoint_dump[req_name]['num_tries'] = self.enabled
-        return req_name
+        checkpoint_dump[req_name]['num_tries'] = self.num_tries
+        return checkpoint_dump
 
     def __repr__(self):
         """Return CheckpointRequirement as a yaml string."""
@@ -627,6 +630,4 @@ class CWL:
 
     def __repr__(self):
         """Return CWL file as a string."""
-        stream = StringIO()
-        yaml.dump(self.dump_wf(), stream)
-        return stream.getvalue()
+        return self.dump_wf()

{hpc_beeflow-0.1.11.dev0 → hpc_beeflow-0.1.11.dev1}/beeflow/common/cwl/examples/comd.py

@@ -33,7 +33,7 @@ def main():
                         # Slurm(account="standard", time_limit=60, partition="standard",
                         #      qos="debug", reservation="standard"),
                         Script(pre_script="comd_pre.sh"),
-                        Slurm(time_limit=500),
+                        Slurm(time_limit="500"),
                         Charliecloud(docker_file="Dockerfile.comd-x86_64",
                                      container_name="comd-mpi")
                      ])

{hpc_beeflow-0.1.11.dev0 → hpc_beeflow-0.1.11.dev1}/beeflow/common/cwl/workflow.py

@@ -5,7 +5,8 @@ from dataclasses import dataclass
 from beeflow.common.cwl.cwl import (CWL, CWLInput, CWLInputs, RunInput, Inputs, CWLOutput,
                                     Outputs, Run, RunOutput, Step, Steps, Hints,
                                     InputBinding, MPIRequirement, DockerRequirement,
-                                    ScriptRequirement, SlurmRequirement)
+                                    ScriptRequirement, SlurmRequirement,
+                                    CheckpointRequirement)
 
 
 @dataclass
@@ -14,12 +15,13 @@ class Input:
     name: str
     type_: str
     # This is either a value or a source connection
-    value: str
+    value: str = None
     # The prefix or position of the argument
     # This can either be a prefix such as -f or --file
     # Or a position like 2 if the command is "foo <file>"
     prefix: str = None
     position: int = None
+    value_from: str = None
 
     pattern = re.compile(r"^[^/]+/[^/]+$")
 
@@ -35,7 +37,8 @@ class Input:
 
     def run_input(self):
         """Create a RunInput from generic Input."""
-        bindings = {'prefix': self.prefix, 'position': self.position}
+        bindings = {'prefix': self.prefix, 'position': self.position,
+                    'value_from': self.value_from}
         source = {}
         if self.has_source():
             source.update({"source": self.value})
@@ -120,6 +123,20 @@ class Script:
                                  shell=self.shell)
 
 
+@dataclass
+class Checkpoint(CheckpointRequirement):
+    """Get Checkpoint Requirements."""
+
+    def requirement(self):
+        """Return a checkpoint requirement object."""
+        return CheckpointRequirement(file_path=self.file_path,
+                                     container_path=self.container_path,
+                                     file_regex=self.file_regex,
+                                     restart_parameters=self.restart_parameters,
+                                     num_tries=self.num_tries,
+                                     enabled=self.enabled)
+
+
 @dataclass
 class Task:
     """Represents a task."""

hpc_beeflow-0.1.11.dev1/beeflow/tests/cwl_files/cat-grep-tar.cwl

@@ -0,0 +1,109 @@
+cwlVersion: v1.0
+class: Workflow
+
+inputs:
+  input_file: File
+  word0: string
+  word1: string
+  tarball_fname: string
+
+outputs:
+  cat_stderr:
+    type: File
+    outputSource: cat/cat_stderr
+  tarball:
+    type: File
+    outputSource: tar/tarball
+
+steps:
+  cat:
+    run:
+      class: CommandLineTool
+      baseCommand: cat
+      stdout: cat.txt
+      stderr: cat.err
+      inputs:
+        input_file:
+          type: File
+          inputBinding:
+            position: 1
+      outputs:
+        contents:
+          type: stdout
+        cat_stderr:
+          type: stderr
+    in:
+      input_file: input_file
+    out: [contents, cat_stderr]
+  grep0:
+    run:
+      class: CommandLineTool
+      baseCommand: grep
+      stdout: occur0.txt
+      inputs:
+        word0:
+          type: string
+          inputBinding:
+            position: 1
+        text_file:
+          type: File
+          inputBinding:
+            position: 2
+      outputs:
+        occur:
+          type: stdout
+    in:
+      word0: word0
+      text_file: cat/contents
+    out: [occur]
+  grep1:
+    run:
+      class: CommandLineTool
+      baseCommand: grep
+      stdout: occur1.txt
+      inputs:
+        word1:
+          type: string
+          inputBinding:
+            position: 1
+        text_file:
+          type: File
+          inputBinding:
+            position: 2
+      outputs:
+        occur:
+          type: stdout
+    in:
+      word1: word1
+      text_file: cat/contents
+    out: [occur]
+  tar:
+    run:
+      class: CommandLineTool
+      baseCommand: tar
+      stdout: occur1.txt
+      inputs:
+        tarball_fname:
+          type: string
+          inputBinding:
+            position: 1
+            prefix: -cf
+        file0:
+          type: File
+          inputBinding:
+            position: 2
+        file1:
+          type: File
+          inputBinding:
+            position: 3
+      outputs:
+        tarball:
+          type: File
+          outputBinding:
+            glob: $(inputs.tarball_fname)
+    in:
+      tarball_fname: tarball_fname
+      file0: grep0/occur
+      file1: grep1/occur
+    out: [tarball]
+

hpc_beeflow-0.1.11.dev1/beeflow/tests/cwl_files/cat-grep-tar.yml

@@ -0,0 +1,5 @@
+input_file: lorem.txt
+word0: Vivamus
+word1: pulvinar
+tarball_fname: out.tgz
+

hpc_beeflow-0.1.11.dev1/beeflow/tests/cwl_files/clamr.cwl

@@ -0,0 +1,94 @@
+cwlVersion: v1.0
+class: Workflow
+
+inputs:
+  max_levels: int
+  grid_resolution: int
+  steps_between_output: int
+  steps_between_graphics: int
+  time_steps: int
+  graphics_type: string
+  checkpoint_disk_interval: int
+
+outputs:
+  clamr_stdout:
+    type: File
+    outputSource: clamr/clamr_stdout
+  clamr_time_log:
+    type: File
+    outputSource: clamr/time_log
+
+steps:
+  clamr:
+    run:
+      class: CommandLineTool
+      baseCommand: /CLAMR/clamr_cpuonly
+      stdout: clamr.txt
+      stderr: clamr.err
+      inputs:
+        max_levels:
+          type: int
+          inputBinding:
+            prefix: -l
+        grid_resolution:
+          type: int
+          inputBinding:
+            prefix: -n
+        steps_between_output:
+          type: int
+          inputBinding:
+            prefix: -i
+        steps_between_graphics:
+          type: int
+          inputBinding:
+            prefix: -g
+        time_steps:
+          type: int
+          inputBinding:
+            prefix: -t
+        graphics_type:
+          type: string
+          inputBinding:
+            prefix: -G
+        checkpoint_disk_interval:
+          type: int
+          inputBinding:
+            prefix: -c
+      outputs:
+        clamr_stdout:
+          type: File
+        outdir:
+          type: Directory
+          outputBinding:
+            glob: graphics_output/graph%05d.png
+        checkpoint_dir:
+          type: Directory
+          outputBinding:
+            glob: checkpoint_output/backup%05d.crx
+        clamr_time_log:
+          type: File
+          outputBinding:
+            glob: total_execution_time.log
+    in:
+      max_levels: max_levels
+      grid_resolution: grid_resolution
+      steps_between_output: steps_between_output
+      steps_between_graphics: steps_between_graphics
+      time_steps: time_steps
+      graphics_type: graphics_type
+      checkpoint_disk_interval: checkpoint_disk_interval
+    out: [clamr_stdout, outdir, checkpoint_dir, clamr_time_log]
+    hints:
+      beeflow:CheckpointRequirement:
+        enabled: true
+        file_path: checkpoint_output
+        container_path: checkpoint_output
+        file_regex: backup[0-9]*.crx
+        restart_parameters: -R
+        num_tries: 3
+      beeflow:SlurmRequirement:
+        timeLimit: 00:00:10
+      DockerRequirement:
+        dockerFile: Dockerfile.clamr-ffmpeg
+        beeflow:containerName: clamr-ffmpeg
+

hpc_beeflow-0.1.11.dev1/beeflow/tests/cwl_files/clamr.yml

@@ -0,0 +1,8 @@
+max_levels: 3
+grid_resolution: 32
+steps_between_output: 10
+steps_between_graphics: 25
+time_steps: 10000
+graphics_type: png
+checkpoint_disk_interval: 50
+

hpc_beeflow-0.1.11.dev1/beeflow/tests/cwl_files/comd.cwl

@@ -0,0 +1,79 @@
+cwlVersion: v1.0
+class: Workflow
+
+inputs:
+  i: int
+  j: int
+  k: int
+  x: int
+  y: int
+  z: int
+  pot_dir: string
+
+outputs:
+  comd_stdout:
+    type: File
+    outputSource: comd/comd_stdout
+
+steps:
+  comd:
+    run:
+      class: CommandLineTool
+      baseCommand: /CoMD/bin/CoMD-mpi -e
+      stdout: comd.txt
+      stderr: comd.err
+      inputs:
+        i:
+          type: int
+          inputBinding:
+            prefix: -i
+        j:
+          type: int
+          inputBinding:
+            prefix: -j
+        k:
+          type: int
+          inputBinding:
+            prefix: -k
+        x:
+          type: int
+          inputBinding:
+            prefix: -x
+        y:
+          type: int
+          inputBinding:
+            prefix: -y
+        z:
+          type: int
+          inputBinding:
+            prefix: -z
+        pot_dir:
+          type: string
+          inputBinding:
+            prefix: --potDir
+      outputs:
+        comd_stdout:
+          type: File
+    in:
+      i: i
+      j: j
+      k: k
+      x: x
+      y: y
+      z: z
+      pot_dir: pot_dir
+    out: [comd_stdout]
+    hints:
+      beeflow:MPIRequirement:
+        nodes: 4
+        ntasks: 8
+      beeflow:ScriptRequirement:
+        pre_script: comd_pre.sh
+        enabled: true
+        shell: /bin/bash
+      beeflow:SlurmRequirement:
+        timeLimit: 500
+      DockerRequirement:
+        dockerFile: Dockerfile.comd-x86_64
+        beeflow:containerName: comd-mpi
+

hpc_beeflow-0.1.11.dev1/beeflow/tests/cwl_files/comd.yml

@@ -0,0 +1,8 @@
+i: 2
+j: 2
+k: 2
+x: 40
+y: 40
+z: 40
+pot_dir: /CoMD/pots
+