hpc-beeflow 0.1.11.dev5__tar.gz → 0.1.12.dev0__tar.gz

{hpc_beeflow-0.1.11.dev5 → hpc_beeflow-0.1.12.dev0}/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: hpc-beeflow
-Version: 0.1.11.dev5
-Summary: A software package for containerizing HPC applications and managing job workflows
+Version: 0.1.12.dev0
+Summary: Orchestrates complex simulation workflows through coordination, execution, and monitoring, with support for containerized applications.
 License-File: LICENSE
 Keywords: bee,hpc,workflow,cluster,computing
 Author: BEE-LANL Dev Team
@@ -32,6 +32,7 @@ Requires-Dist: Flask (>=2.0,<3.0)
 Requires-Dist: PyYAML (>=6.0.1,<7.0.0)
 Requires-Dist: celery[redis,sqlalchemy] (>=5.3.4,<6.0.0)
 Requires-Dist: cffi (>=1.15.1,<2.0.0)
+Requires-Dist: click (==8.1.7)
 Requires-Dist: cwl-utils (>=0.16,<0.17)
 Requires-Dist: docutils (==0.18.1)
 Requires-Dist: fastapi (==0.115.8)
@@ -106,7 +107,7 @@ BEE Sites
      - Documentation
 
    * - Stable release
-     - ``0.1.10``
+     - ``0.1.11``
      - `https://lanl.github.io/BEE/ <https://lanl.github.io/BEE/>`__
    * - Pre-release
      - ``0.1.11dev5``

{hpc_beeflow-0.1.11.dev5 → hpc_beeflow-0.1.12.dev0}/README.rst

@@ -47,7 +47,7 @@ BEE Sites
      - Documentation
 
    * - Stable release
-     - ``0.1.10``
+     - ``0.1.11``
      - `https://lanl.github.io/BEE/ <https://lanl.github.io/BEE/>`__
    * - Pre-release
      - ``0.1.11dev5``

{hpc_beeflow-0.1.11.dev5 → hpc_beeflow-0.1.12.dev0}/beeflow/client/core.py

@@ -49,15 +49,15 @@ def patch_attribute_section():
     sched = config.get('DEFAULT', 'workload_scheduler', fallback='Slurm')
     if sched == 'Flux':
         section = 'flux attributes'
-        default_attrs = 'queue,runtime,nodelist'
+        default_attrs = 'id,queue,runtime,nodelist'
     elif sched == 'Slurm':
         use_cmds = config.get('slurm', 'use_commands', fallback='True').strip().lower() == 'true'
         if use_cmds:
             section = 'slurm command attributes'
-            default_attrs = 'Partition,RunTime,NodeList'
+            default_attrs = 'JobId,Partition,RunTime,NodeList,Command'
         else:
             section = 'slurm attributes'
-            default_attrs = 'partition,nodes'
+            default_attrs = 'job_id,partition,nodes,command'
     else:
         return
 
@@ -197,9 +197,10 @@ def open_log(component):
 
 
 def need_slurmrestd():
-    """Check if slurmrestd is needed."""
+    """Check if slurmrestd is needed and that it exists in the current environment."""
+    have_slurmrestd = shutil.which('slurmrestd')
     return (bc.get('DEFAULT', 'workload_scheduler') == 'Slurm'
-            and not bc.get('slurm', 'use_commands'))
+            and not bc.get('slurm', 'use_commands') and have_slurmrestd)
 
 
 def need_neo4j():

{hpc_beeflow-0.1.11.dev5 → hpc_beeflow-0.1.12.dev0}/beeflow/common/config_driver.py

@@ -367,18 +367,18 @@ flux_attr = [
 VALIDATOR.section('slurm attributes',info='Available task information for the slurm scheduler\n',
         depends_on=('slurm', 'use_commands', 'False'))
 VALIDATOR.option('slurm attributes','attributes',validator=validate_attributes,prompt=False,
-        default='partition,nodes',info='Enter task attributes (comma-separated)')
+        default='job_id,partition,nodes,command',info='Enter task attributes (comma-separated)')
 
 VALIDATOR.section('slurm command attributes',info='Available task information for sacct\n',
         depends_on=('slurm', 'use_commands', 'True'))
 VALIDATOR.option('slurm command attributes','attributes',validator=validate_attributes,
-        prompt=False,default='Partition,RunTime,NodeList',
+        prompt=False,default='JobId,Partition,RunTime,NodeList,Command',
         info='Enter task attributes (comma-separated)')
 
 VALIDATOR.section('flux attributes', info = 'Available task information for the flux scheduler\n',
         depends_on=('DEFAULT', 'workload_scheduler', 'Flux'))
 VALIDATOR.option('flux attributes','attributes',validator=validate_attributes,prompt=False,
-        default='queue,runtime,nodelist',info='Enter task attributes (comma-separated).')
+        default='id,queue,runtime,nodelist',info='Enter task attributes (comma-separated).')
 
 def validate_chrun_opts(opts):
     """Ensure that chrun_opts don't contain options that'll conflict with BEE."""

{hpc_beeflow-0.1.11.dev5 → hpc_beeflow-0.1.12.dev0}/beeflow/common/config_utils.py

@@ -25,7 +25,7 @@ def filter_and_validate(config, validator,config_path):
         if not config.has_section('flux attributes'):
             config.add_section('flux attributes')
         if not config.get('flux attributes','attributes',fallback='').strip():
-            config.set('flux attributes','attributes','queue,runtime,nodelist')
+            config.set('flux attributes','attributes','id,queue,runtime,nodelist')
     elif scheduler == 'Slurm':
         if use_commands:
             sections_to_remove = ['flux attributes','slurm attributes']

{hpc_beeflow-0.1.11.dev5 → hpc_beeflow-0.1.12.dev0}/beeflow/common/cwl/cwl.py

@@ -377,6 +377,7 @@ class SlurmRequirement:
 
     account: str = None
     time_limit: str = None
+    signal: str = None
     partition: str = None
     qos: str = None
     reservation: str = None
@@ -395,6 +396,8 @@ class SlurmRequirement:
             sched_dump['beeflow:SlurmRequirement']['qos'] = self.qos
         if self.reservation:
             sched_dump['beeflow:SlurmRequirement']['reservation'] = self.reservation
+        if self.signal:
+            sched_dump['beeflow:SlurmRequirement']['signal'] = self.signal
         if self.load_from_file:
             sched_dump['beeflow:SlurmRequirement']['load_from_file'] = self.load_from_file
         return sched_dump

{hpc_beeflow-0.1.11.dev5 → hpc_beeflow-0.1.12.dev0}/beeflow/common/cwl/workflow.py

@@ -90,6 +90,7 @@ class Slurm(SlurmRequirement):
         """Return a scheduler requirement object."""
         return SlurmRequirement(time_limit=self.time_limit, account=self.account,
                 partition=self.partition, qos=self.qos, reservation=self.reservation,
+                          signal=self.signal,
                           load_from_file=self.load_from_file)
 
 

{hpc_beeflow-0.1.11.dev5 → hpc_beeflow-0.1.12.dev0}/beeflow/common/worker/slurm_worker.py

@@ -81,6 +81,8 @@ class BaseSlurmWorker(Worker):
         reservation = self.get_req(task, 'beeflow:SlurmRequirement', 'reservation', config,
                                    default=self.default_reservation)
 
+        signal = self.get_req(task, 'beeflow:SlurmRequirement', 'signal', config,
+                             default=None)
         shell = self.get_req(task, 'beeflow:ScriptRequirement', 'shell', config,
                              default="/bin/bash")
         scripts_enabled = self.get_req(task, 'beeflow:ScriptRequirement', 'enabled', config,
@@ -96,6 +98,7 @@ class BaseSlurmWorker(Worker):
             'qos': qos,
             'reservation': reservation,
             'shell': shell,
+            'signal': signal,
             'scripts_enabled': scripts_enabled
         }
         return requirements
@@ -122,6 +125,8 @@ class BaseSlurmWorker(Worker):
             header.append(f'#SBATCH --qos {requirements["qos"]}')
         if requirements['reservation']:
             header.append(f'#SBATCH --reservation {requirements["reservation"]}')
+        if requirements['signal']:
+            header.append(f'#SBATCH --signal={requirements["signal"]}')
         # Return immediately on error
         if requirements["shell"] == "/bin/bash":
             header.append('set -e')

{hpc_beeflow-0.1.11.dev5 → hpc_beeflow-0.1.12.dev0}/beeflow/common/worker/utils.py

@@ -22,7 +22,7 @@ def get_state_sacct(job_id):
         data = resp.stdout.splitlines()
         header = data[0]
         header = header.split('|')
-        job_id_idx = header.index('JobID')
+        job_id_idx = header.index('JobId')
         rows = [row.split('|') for row in data[1:]]
         job_ids = [row[job_id_idx] for row in rows]
         info = rows[job_ids.index(job_id)]

{hpc_beeflow-0.1.11.dev5 → hpc_beeflow-0.1.12.dev0}/pyproject.toml

@@ -1,7 +1,7 @@
 [tool.poetry]
 name = "hpc-beeflow"
-version = "0.1.11dev5"
-description = "A software package for containerizing HPC applications and managing job workflows"
+version = "0.1.12dev"
+description = "Orchestrates complex simulation workflows through coordination, execution, and monitoring, with support for containerized applications."
 
 
 authors = [
@@ -70,6 +70,8 @@ gunicorn = ">=20.1,<23.0"
 # typer version 0.6 and above seem to be throwing an AssertionError with no
 # attached info
 typer = "^0.5.0"
+# setting click below 8.3.0 avoids a crashing bug with typer version
+click = "8.1.7"
 # Seems to be required for Flux
 cffi = "^1.15.1"
 celery = { version = "^5.3.4", extras = ["redis", "sqlalchemy"] }
@@ -91,7 +93,7 @@ mpi4py = "^4.1.1"
 [tool.poetry.extras]
 cloud_extras = ["google-api-python-client", "python-openstackclient", "python-heatclient"]
 
-[tool.poetry.dev-dependencies]
+[tool.poetry.group.dev.dependencies]
 # Developer dependencies
 pylint = "3.3.6"
 pytest = "7.2.0"