hpc-beeflow 0.1.11.dev5__tar.gz → 0.1.12.dev1__tar.gz

{hpc_beeflow-0.1.11.dev5 → hpc_beeflow-0.1.12.dev1}/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: hpc-beeflow
-Version: 0.1.11.dev5
-Summary: A software package for containerizing HPC applications and managing job workflows
+Version: 0.1.12.dev1
+Summary: Orchestrates complex simulation workflows through coordination, execution, and monitoring, with support for containerized applications.
 License-File: LICENSE
 Keywords: bee,hpc,workflow,cluster,computing
 Author: BEE-LANL Dev Team
@@ -32,6 +32,7 @@ Requires-Dist: Flask (>=2.0,<3.0)
 Requires-Dist: PyYAML (>=6.0.1,<7.0.0)
 Requires-Dist: celery[redis,sqlalchemy] (>=5.3.4,<6.0.0)
 Requires-Dist: cffi (>=1.15.1,<2.0.0)
+Requires-Dist: click (==8.1.7)
 Requires-Dist: cwl-utils (>=0.16,<0.17)
 Requires-Dist: docutils (==0.18.1)
 Requires-Dist: fastapi (==0.115.8)
@@ -106,11 +107,11 @@ BEE Sites
      - Documentation
 
    * - Stable release
-     - ``0.1.10``
+     - ``0.1.11``
      - `https://lanl.github.io/BEE/ <https://lanl.github.io/BEE/>`__
    * - Pre-release
-     - ``0.1.11dev5``
-     - `https://lanl.github.io/BEE/0.1.11dev5/ <https://lanl.github.io/BEE/0.1.11dev5/>`__
+     - ``0.1.12dev0``
+     - `https://lanl.github.io/BEE/0.1.12dev1/ <https://lanl.github.io/BEE/0.1.12dev1/>`__
 
 
 Contact

{hpc_beeflow-0.1.11.dev5 → hpc_beeflow-0.1.12.dev1}/README.rst

@@ -47,11 +47,11 @@ BEE Sites
      - Documentation
 
    * - Stable release
-     - ``0.1.10``
+     - ``0.1.11``
      - `https://lanl.github.io/BEE/ <https://lanl.github.io/BEE/>`__
    * - Pre-release
-     - ``0.1.11dev5``
-     - `https://lanl.github.io/BEE/0.1.11dev5/ <https://lanl.github.io/BEE/0.1.11dev5/>`__
+     - ``0.1.12dev0``
+     - `https://lanl.github.io/BEE/0.1.12dev1/ <https://lanl.github.io/BEE/0.1.12dev1/>`__
 
 
 Contact

{hpc_beeflow-0.1.11.dev5 → hpc_beeflow-0.1.12.dev1}/beeflow/client/core.py

@@ -49,15 +49,15 @@ def patch_attribute_section():
     sched = config.get('DEFAULT', 'workload_scheduler', fallback='Slurm')
     if sched == 'Flux':
         section = 'flux attributes'
-        default_attrs = 'queue,runtime,nodelist'
+        default_attrs = 'id,queue,runtime,nodelist'
     elif sched == 'Slurm':
         use_cmds = config.get('slurm', 'use_commands', fallback='True').strip().lower() == 'true'
         if use_cmds:
             section = 'slurm command attributes'
-            default_attrs = 'Partition,RunTime,NodeList'
+            default_attrs = 'JobId,Partition,RunTime,NodeList,Command'
         else:
             section = 'slurm attributes'
-            default_attrs = 'partition,nodes'
+            default_attrs = 'job_id,partition,nodes,command'
     else:
         return
 
@@ -197,9 +197,10 @@ def open_log(component):
 
 
 def need_slurmrestd():
-    """Check if slurmrestd is needed."""
+    """Check if slurmrestd is needed and that it exists in the current environment."""
+    have_slurmrestd = shutil.which('slurmrestd')
     return (bc.get('DEFAULT', 'workload_scheduler') == 'Slurm'
-            and not bc.get('slurm', 'use_commands'))
+            and not bc.get('slurm', 'use_commands') and have_slurmrestd)
 
 
 def need_neo4j():
@@ -214,7 +215,7 @@ def init_components(remote=False):
     # Slurmrestd will be started only if we're running with Slurm and
     # slurm::use_commands is not True
 
-    @mgr.component('wf_manager', ('scheduler', 'celery'))
+    @mgr.component('wf_manager', ('celery',))
     def start_wfm():
         """Start the WFM."""
         fp = open_log('wf_manager')
@@ -232,14 +233,6 @@ def init_components(remote=False):
         return launch_with_gunicorn('beeflow.task_manager.task_manager:create_app()',
                                     paths.tm_socket(), stdout=fp, stderr=fp)
 
-    @mgr.component('scheduler', ())
-    def start_scheduler():
-        """Start the scheduler."""
-        fp = open_log('scheduler')
-        # Using a function here because of the funny way that the scheduler's written
-        return launch_with_gunicorn('beeflow.scheduler.scheduler:create_app()',
-                                    paths.sched_socket(), stdout=fp, stderr=fp)
-
     if remote:
         @mgr.component('remote_api', ('wf_manager', 'task_manager'))
         def start_remote_api():
@@ -495,7 +488,7 @@ def start(foreground: bool = typer.Option(False, '--foreground', '-F',
     # Create the log path if it doesn't exist yet
     path = paths.log_path()
     os.makedirs(path, exist_ok=True)
-    base_components = ['wf_manager', 'task_manager', 'scheduler']
+    base_components = ['wf_manager', 'task_manager']
     if foreground:
         try:
             Beeflow(mgr, base_components).loop()
@@ -597,14 +590,31 @@ def handle_rm_error(err, dir_to_check, wf_list):
 def reset(archive: bool = typer.Option(False, '--archive', '-a',
                                        help='Archive bee_workdir  before removal')):
     """Stop all components and delete the bee_workdir directory."""
-    # Check workflow states; warn if there are active states.
     reset_hn = socket.gethostname()
-    bee_client.check_hostname(reset_hn)
-    workflow_list = bee_client.get_wf_list()
+
+    # Check if beeflow is running
+    sock_path = paths.beeflow_socket()
+    beeflow_running = False
+    workflow_list = []
+
+    if os.path.exists(sock_path):
+        try:
+            resp = cli_connection.send(sock_path, {'type': 'status'})
+            if resp is not None:
+                beeflow_running = True
+                bee_client.check_hostname(reset_hn)
+                workflow_list = bee_client.get_wf_list()
+        except (ConnectionResetError, ConnectionRefusedError,
+                cli_connection.BeeflowConnectionError, OSError):
+            # Beeflow socket exists but is not responsive
+            pass
+
+    # Check workflow states; warn if there are active states.
     active_states = {'Running', 'Paused', 'Initializing', 'Waiting'}
     caution = ""
-    if {item for row in workflow_list for item in row}.intersection(active_states):
-        caution = """
+    if beeflow_running and workflow_list:
+        if {item for row in workflow_list for item in row}.intersection(active_states):
+            caution = """
         **************************************************************
           Caution: There are active workflows! They will be removed!
           Try 'beeflow list' to view them.
@@ -615,9 +625,16 @@ def reset(archive: bool = typer.Option(False, '--archive', '-a',
     warn(f"\n    A reset will remove this directory: {dir_to_delete}\n")
     if archive:
         print("    Archive flag is set: logs, workflows and containers will be backed up.")
+    else:
+        warn("    If you want to save logs, workflows and containers:")
+        warn("      answer no and rerun with --archive flag.")
+
+    status_msg = "running" if beeflow_running else "not running"
+    print(f"\n    Beeflow is currently {status_msg}.")
+
     print("""
     A reset will:
-        Shutdown beeflow and all BEE components.
+        Shutdown beeflow and all BEE components (if running).
         Delete the bee_workdir directory which results in:
             Removing the archive of all workflows.
                 (unless archives is configured elsewhere).
@@ -635,11 +652,14 @@ def reset(archive: bool = typer.Option(False, '--archive', '-a',
             sys.exit()
         elif absolutely_sure in ("y", "yes"):
             # Stop all of the beeflow processes
-            stop("quiet")
-            print("Beeflow is shutting down.")
-            print("Waiting for components to cleanly stop.")
-            # This wait is essential. It takes a minute to shut down.
-            time.sleep(5)
+            if beeflow_running:
+                stop("quiet")
+                print("Beeflow is shutting down.")
+                print("Waiting for components to cleanly stop.")
+                # This wait is essential. It takes a minute to shut down.
+                time.sleep(5)
+            else:
+                print("Beeflow is not running, proceeding to remove bee_workdir.")
 
             # Save the bee_workdir directory if the archive option was set
             if archive:

{hpc_beeflow-0.1.11.dev5 → hpc_beeflow-0.1.12.dev1}/beeflow/common/config_driver.py

@@ -367,18 +367,18 @@ flux_attr = [
 VALIDATOR.section('slurm attributes',info='Available task information for the slurm scheduler\n',
         depends_on=('slurm', 'use_commands', 'False'))
 VALIDATOR.option('slurm attributes','attributes',validator=validate_attributes,prompt=False,
-        default='partition,nodes',info='Enter task attributes (comma-separated)')
+        default='job_id,partition,nodes,command',info='Enter task attributes (comma-separated)')
 
 VALIDATOR.section('slurm command attributes',info='Available task information for sacct\n',
         depends_on=('slurm', 'use_commands', 'True'))
 VALIDATOR.option('slurm command attributes','attributes',validator=validate_attributes,
-        prompt=False,default='Partition,RunTime,NodeList',
+        prompt=False,default='JobId,Partition,RunTime,NodeList,Command',
         info='Enter task attributes (comma-separated)')
 
 VALIDATOR.section('flux attributes', info = 'Available task information for the flux scheduler\n',
         depends_on=('DEFAULT', 'workload_scheduler', 'Flux'))
 VALIDATOR.option('flux attributes','attributes',validator=validate_attributes,prompt=False,
-        default='queue,runtime,nodelist',info='Enter task attributes (comma-separated).')
+        default='id,queue,runtime,nodelist',info='Enter task attributes (comma-separated).')
 
 def validate_chrun_opts(opts):
     """Ensure that chrun_opts don't contain options that'll conflict with BEE."""

{hpc_beeflow-0.1.11.dev5 → hpc_beeflow-0.1.12.dev1}/beeflow/common/config_utils.py

@@ -25,7 +25,7 @@ def filter_and_validate(config, validator,config_path):
         if not config.has_section('flux attributes'):
             config.add_section('flux attributes')
         if not config.get('flux attributes','attributes',fallback='').strip():
-            config.set('flux attributes','attributes','queue,runtime,nodelist')
+            config.set('flux attributes','attributes','id,queue,runtime,nodelist')
     elif scheduler == 'Slurm':
         if use_commands:
             sections_to_remove = ['flux attributes','slurm attributes']

{hpc_beeflow-0.1.11.dev5 → hpc_beeflow-0.1.12.dev1}/beeflow/common/cwl/cwl.py

@@ -377,6 +377,7 @@ class SlurmRequirement:
 
     account: str = None
     time_limit: str = None
+    signal: str = None
     partition: str = None
     qos: str = None
     reservation: str = None
@@ -395,6 +396,8 @@ class SlurmRequirement:
             sched_dump['beeflow:SlurmRequirement']['qos'] = self.qos
         if self.reservation:
             sched_dump['beeflow:SlurmRequirement']['reservation'] = self.reservation
+        if self.signal:
+            sched_dump['beeflow:SlurmRequirement']['signal'] = self.signal
         if self.load_from_file:
             sched_dump['beeflow:SlurmRequirement']['load_from_file'] = self.load_from_file
         return sched_dump

{hpc_beeflow-0.1.11.dev5 → hpc_beeflow-0.1.12.dev1}/beeflow/common/cwl/workflow.py

@@ -90,6 +90,7 @@ class Slurm(SlurmRequirement):
         """Return a scheduler requirement object."""
         return SlurmRequirement(time_limit=self.time_limit, account=self.account,
                 partition=self.partition, qos=self.qos, reservation=self.reservation,
+                          signal=self.signal,
                           load_from_file=self.load_from_file)
 
 

{hpc_beeflow-0.1.11.dev5 → hpc_beeflow-0.1.12.dev1}/beeflow/common/log.py

@@ -18,6 +18,6 @@ def setup(name):
     log.setLevel(LEVEL)
     handler = logging.StreamHandler()
     handler.setLevel(logging.DEBUG)
-    handler.setFormatter(logging.Formatter('[%(asctime)s] %(name)s:%(funcName)s(): %(msg)s'))
+    handler.setFormatter(logging.Formatter('[%(asctime)s] %(name)s:%(funcName)s(): %(message)s'))
     log.addHandler(handler)
     return log

{hpc_beeflow-0.1.11.dev5 → hpc_beeflow-0.1.12.dev1}/beeflow/common/paths.py

@@ -35,11 +35,6 @@ def remote_socket():
     return os.path.join(_sockdir(), 'remote.sock')
 
 
-def sched_socket():
-    """Get the scheduler socket."""
-    return os.path.join(_sockdir(), 'scheduler.sock')
-
-
 def slurm_socket():
     """Get the slurm socket (for slurmrestd)."""
     return os.path.join(_sockdir(), 'slurmrestd.sock')

{hpc_beeflow-0.1.11.dev5 → hpc_beeflow-0.1.12.dev1}/beeflow/common/worker/slurm_worker.py

@@ -81,6 +81,8 @@ class BaseSlurmWorker(Worker):
         reservation = self.get_req(task, 'beeflow:SlurmRequirement', 'reservation', config,
                                    default=self.default_reservation)
 
+        signal = self.get_req(task, 'beeflow:SlurmRequirement', 'signal', config,
+                             default=None)
         shell = self.get_req(task, 'beeflow:ScriptRequirement', 'shell', config,
                              default="/bin/bash")
         scripts_enabled = self.get_req(task, 'beeflow:ScriptRequirement', 'enabled', config,
@@ -96,6 +98,7 @@ class BaseSlurmWorker(Worker):
             'qos': qos,
             'reservation': reservation,
             'shell': shell,
+            'signal': signal,
             'scripts_enabled': scripts_enabled
         }
         return requirements
@@ -122,6 +125,8 @@ class BaseSlurmWorker(Worker):
             header.append(f'#SBATCH --qos {requirements["qos"]}')
         if requirements['reservation']:
             header.append(f'#SBATCH --reservation {requirements["reservation"]}')
+        if requirements['signal']:
+            header.append(f'#SBATCH --signal={requirements["signal"]}')
         # Return immediately on error
         if requirements["shell"] == "/bin/bash":
             header.append('set -e')

{hpc_beeflow-0.1.11.dev5 → hpc_beeflow-0.1.12.dev1}/beeflow/common/worker/utils.py

@@ -22,7 +22,7 @@ def get_state_sacct(job_id):
         data = resp.stdout.splitlines()
         header = data[0]
         header = header.split('|')
-        job_id_idx = header.index('JobID')
+        job_id_idx = header.index('JobId')
         rows = [row.split('|') for row in data[1:]]
         job_ids = [row[job_id_idx] for row in rows]
         info = rows[job_ids.index(job_id)]

{hpc_beeflow-0.1.11.dev5 → hpc_beeflow-0.1.12.dev1}/beeflow/tests/test_remote.py

@@ -76,7 +76,6 @@ def test_get_core_status(mocker):
     # Mocked response from the daemon
     mock_response = {
         "components": {
-            "scheduler": "running",
             "wf_manager": "running",
             "task_manager": "running"
         }
@@ -89,7 +88,6 @@ def test_get_core_status(mocker):
 
     assert response.status_code == 200
     assert response.json() == {
-        "scheduler": "running",
         "wf_manager": "running",
         "task_manager": "running"
     }

{hpc_beeflow-0.1.11.dev5 → hpc_beeflow-0.1.12.dev1}/beeflow/tests/test_remote_client.py

@@ -243,7 +243,7 @@ def test_core_status_success(mocker):
     mock_run = mocker.patch("subprocess.run")
     mock_run.return_value = subprocess.CompletedProcess(
             ["curl", "ssh_target:1234/core/status/"], returncode=0,
-            stdout='{"redis":"RUNNING","scheduler":"RUNNING","celery":"RUNNING",'
+            stdout='{"redis":"RUNNING","celery":"RUNNING",'
             '"slurmrestd":"RUNNING","wf_manager":"RUNNING","task_manager":"RUNNING",'
             '"remote_api":"RUNNING","neo4j-database":"RUNNING"}',
             stderr=""
@@ -252,7 +252,7 @@ def test_core_status_success(mocker):
     result = runner.invoke(app, ["core-status", "ssh_target"])
 
     assert result.exit_code == 0
-    expected_output = '{"redis":"RUNNING","scheduler":"RUNNING","celery":"RUNNING",' \
+    expected_output = '{"redis":"RUNNING","celery":"RUNNING",' \
                       '"slurmrestd":"RUNNING","wf_manager":"RUNNING","task_manager":"RUNNING",' \
                       '"remote_api":"RUNNING","neo4j-database":"RUNNING"}'
     assert expected_output in result.output

{hpc_beeflow-0.1.11.dev5 → hpc_beeflow-0.1.12.dev1}/beeflow/tests/test_wf_manager.py

@@ -140,13 +140,13 @@ def test_start_workflow(client, mocker):
     """Test starting a workflow."""
     mocker.patch('beeflow.wf_manager.resources.wf_utils.get_workflow_interface',
                  return_value=MockWFI())
-    mocker.patch('beeflow.wf_manager.resources.wf_utils.submit_tasks_tm', return_value=None)
-    mocker.patch('beeflow.wf_manager.resources.wf_utils.submit_tasks_scheduler', return_value=None)
+    submit_tasks_tm = mocker.patch('beeflow.wf_manager.resources.wf_utils.submit_tasks_tm', return_value=None)
     mocker.patch('beeflow.wf_manager.resources.wf_utils.update_wf_status', return_value=None)
     mocker.patch('beeflow.tests.mocks.MockWFI.get_workflow_state', return_value='No Start')
     resp = client().post(f'/bee_wfm/v1/jobs/{WF_ID}')
     assert resp.status_code == 200
     assert resp.json['msg'] == 'Workflow started successfully'
+    submit_tasks_tm.assert_called()
 
 
 def test_workflow_status(client, mocker, setup_teardown_workflow):
@@ -266,8 +266,7 @@ def test_resume_workflow(client, mocker, setup_teardown_workflow, temp_db):
                  return_value=MockWFI())
     mocker.patch('beeflow.tests.mocks.MockWFI.get_workflow_state',
                  return_value='Paused')
-    mocker.patch('beeflow.wf_manager.resources.wf_utils.submit_tasks_tm', return_value=None)
-    mocker.patch('beeflow.wf_manager.resources.wf_utils.submit_tasks_scheduler', return_value=None)
+    submit_tasks_tm = mocker.patch('beeflow.wf_manager.resources.wf_utils.submit_tasks_tm', return_value=None)
     mocker.patch('beeflow.wf_manager.resources.wf_utils.update_wf_status', return_value=None)
 
     wf_utils.update_wf_status(WF_ID, 'Paused')
@@ -275,3 +274,4 @@ def test_resume_workflow(client, mocker, setup_teardown_workflow, temp_db):
     resp = client().patch(f'/bee_wfm/v1/jobs/{WF_ID}', json=request)
     assert resp.json['msg'] == 'Workflow Resumed'
     assert resp.status_code == 200
+    submit_tasks_tm.assert_called()

{hpc_beeflow-0.1.11.dev5 → hpc_beeflow-0.1.12.dev1}/beeflow/wf_manager/resources/wf_actions.py

@@ -134,7 +134,7 @@ class WFActions(Resource):
             wf_utils.update_wf_status(wf_id, "Running")
             wfi = wf_utils.get_workflow_interface(wf_id)
             tasks = wfi.get_ready_tasks()
-            wf_utils.schedule_submit_tasks(wf_id, tasks)
+            wf_utils.submit_tasks_tm(wf_id, tasks)
 
             log.info(f"Workflow {wf_id} Resumed")
             resp = WorkflowActionResponse(msg="Workflow Resumed").model_dump(), 200

{hpc_beeflow-0.1.11.dev5 → hpc_beeflow-0.1.12.dev1}/beeflow/wf_manager/resources/wf_update.py

@@ -135,7 +135,7 @@ class WFUpdate(Resource):
                 return True
             # Submit the restart task
             tasks = [new_task]
-            wf_utils.schedule_submit_tasks(state_update.wf_id, tasks)
+            wf_utils.submit_tasks_tm(state_update.wf_id, tasks)
             log.info(f'Task {state_update.task_id} restarted')
             return True
         return False
@@ -153,7 +153,7 @@ class WFUpdate(Resource):
             wf_utils.copy_task_output(task)
             tasks = wfi.finalize_task(task)
             if tasks and wf_state not in ('Paused', 'Cancelled'):
-                wf_utils.schedule_submit_tasks(state_update.wf_id, tasks)
+                wf_utils.submit_tasks_tm(state_update.wf_id, tasks)
 
         # If the job failed, fail the dependent tasks
         # TIMEOUT states should only be seen here if they can't restart

{hpc_beeflow-0.1.11.dev5 → hpc_beeflow-0.1.12.dev1}/beeflow/wf_manager/resources/wf_utils.py

@@ -17,12 +17,6 @@ from beeflow.common.connection import Connection
 from beeflow.common import paths
 from beeflow.common.db import wfm_db
 from beeflow.common.db.bdb import connect_db
-from beeflow.scheduler.models import (
-    ScheduleTasksRequest,
-    ScheduleTasksResponse,
-    SchedulerTask,
-)
-
 from beeflow.task_manager.models import SubmitTasksRequest
 from beeflow.common.deps.neo4j_manager import connect_neo4j_driver
 
@@ -170,7 +164,6 @@ def tm_url():
 
 # Base URLs for the TM and the Scheduler
 TM_URL = "bee_tm/v1/task/"
-SCHED_URL = "bee_sched/v1/"
 
 
 def _connect_tm():
@@ -178,32 +171,13 @@ def _connect_tm():
     return Connection(paths.tm_socket())
 
 
-def sched_url():
-    """Get Scheduler url."""
-    db = connect_db(wfm_db, get_db_path())
-    scheduler = "bee_sched/v1/"
-    # sched_listen_port = bc.get('scheduler', 'listen_port')
-    sched_listen_port = db.info.get_port("sched")
-    return f"http://127.0.0.1:{sched_listen_port}/{scheduler}"
-
-
-def _connect_scheduler():
-    """Return a connection to the Scheduler."""
-    return Connection(paths.sched_socket())
-
-
-def _resource(component, tag=""):
-    """Access Task Manager or Scheduler."""
-    if component == "tm":
-        return TM_URL + str(tag)
-    if component == "sched":
-        return SCHED_URL + str(tag)
-
-    raise ValueError(f"Invalid component: {component}")
+def _taskmanager(tag=""):
+    """Access Task Manager resources."""
+    return TM_URL + str(tag)
 
 
 # Submit tasks to the TM
-def submit_tasks_tm(wf_id, tasks, allocation):  # pylint: disable=W0613
+def submit_tasks_tm(wf_id, tasks):
     """Submit a task to the task manager."""
     wfi = get_workflow_interface(wf_id)
     # Serialize task with json
@@ -212,7 +186,7 @@ def submit_tasks_tm(wf_id, tasks, allocation):  # pylint: disable=W0613
     try:
         conn = _connect_tm()
         resp = conn.post(
-            _resource("tm"),
+            _taskmanager(),
             json=SubmitTasksRequest(tasks=tasks).model_dump(),
             timeout=5,
         )
@@ -228,48 +202,6 @@ def submit_tasks_tm(wf_id, tasks, allocation):  # pylint: disable=W0613
         log.info("Submit task to TM returned bad status: %s", resp.status_code)
 
 
-def tasks_to_sched(tasks):
-    """Convert gdb tasks to sched tasks."""
-    sched_tasks = []
-    for task in tasks:
-        sched_task = SchedulerTask(
-            workflow_name="workflow",
-            task_name=task.name,
-            requirements={"max_runtime": 1, "nodes": 1},
-        )
-        sched_tasks.append(sched_task)
-    return sched_tasks
-
-
-def submit_tasks_scheduler(tasks):
-    """Submit a list of tasks to the scheduler."""
-    sched_tasks = tasks_to_sched(tasks)
-    # The workflow name will eventually be added to the wfi workflow object
-    try:
-        conn = _connect_scheduler()
-        resp = conn.put(
-            _resource("sched", "workflows/workflow/jobs"),
-            json=ScheduleTasksRequest(tasks=sched_tasks).model_dump(),
-            timeout=5,
-        )
-    except requests.exceptions.ConnectionError:
-        log.error("Unable to connect to scheduler to submit tasks.")
-        return "Did not work"
-
-    if resp.status_code != 200:
-        log.info("Something bad happened %s", resp.status_code)
-        return "Did not work"
-    return ScheduleTasksResponse.model_validate(resp.json()).tasks
-
-
-def schedule_submit_tasks(wf_id, tasks):
-    """Schedule and then submit tasks to the TM."""
-    # Submit ready tasks to the scheduler
-    allocation = submit_tasks_scheduler(tasks)
-    # Submit tasks to TM
-    submit_tasks_tm(wf_id, tasks, allocation)
-
-
 def setup_workflow(wf_id, wf_name, wf_dir, wf_workdir, no_start, workflow=None, # pylint: disable=W0613
                    tasks=None):
     """Initialize Workflow and Tasks then start workflow in separate process"""
@@ -318,7 +250,7 @@ def start_workflow(wf_id):
             wfi.set_task_state(task.id, "WAITING")
     wfi.execute_workflow()
     tasks = wfi.get_ready_tasks()
-    schedule_submit_tasks(wf_id, tasks)
+    submit_tasks_tm(wf_id, tasks)
     update_wf_status(wf_id, "Running")
     return True
 

{hpc_beeflow-0.1.11.dev5 → hpc_beeflow-0.1.12.dev1}/pyproject.toml

@@ -1,7 +1,7 @@
 [tool.poetry]
 name = "hpc-beeflow"
-version = "0.1.11dev5"
-description = "A software package for containerizing HPC applications and managing job workflows"
+version = "0.1.12dev1"
+description = "Orchestrates complex simulation workflows through coordination, execution, and monitoring, with support for containerized applications."
 
 
 authors = [
@@ -70,6 +70,8 @@ gunicorn = ">=20.1,<23.0"
 # typer version 0.6 and above seem to be throwing an AssertionError with no
 # attached info
 typer = "^0.5.0"
+# setting click below 8.3.0 avoids a crashing bug with typer version
+click = "8.1.7"
 # Seems to be required for Flux
 cffi = "^1.15.1"
 celery = { version = "^5.3.4", extras = ["redis", "sqlalchemy"] }
@@ -91,7 +93,7 @@ mpi4py = "^4.1.1"
 [tool.poetry.extras]
 cloud_extras = ["google-api-python-client", "python-openstackclient", "python-heatclient"]
 
-[tool.poetry.dev-dependencies]
+[tool.poetry.group.dev.dependencies]
 # Developer dependencies
 pylint = "3.3.6"
 pytest = "7.2.0"

hpc_beeflow-0.1.11.dev5/beeflow/common/db/sched_db.py

@@ -1,58 +0,0 @@
-"""Scheduler database code."""
-
-import jsonpickle
-from beeflow.common.db import bdb
-
-
-class Resources:
-    """Resources wrapper class."""
-
-    def __init__(self, db_file):
-        """Create the resources class."""
-        self.db_file = db_file
-
-    def __iter__(self):
-        """Create an iterator over the resources."""
-        stmt = 'SELECT id, resource FROM resources'
-        result = bdb.getall(self.db_file, stmt)
-        for rslt in result:
-            resource = jsonpickle.decode(rslt[1])
-            yield resource
-
-    def extend(self, resources):
-        """Add a list of new resources."""
-        for resource in resources:
-            data = jsonpickle.encode(resource)
-            stmt = 'INSERT INTO resources (resource) VALUES (?)'
-            bdb.run(self.db_file, stmt, [data])
-
-    def clear(self):
-        """Remove all resources."""
-        clear_stmt = 'DELETE FROM resources'
-        bdb.run(self.db_file, clear_stmt)
-
-
-class SchedDB:
-    """Scheduler database."""
-
-    def __init__(self, db_file):
-        """Construct a new scheduler database connection."""
-        self.db_file = db_file
-        self._init_tables()
-
-    def _init_tables(self):
-        """Initialze the scheduler tables if they don't exist."""
-        resource_stmt = """CREATE TABLE IF NOT EXISTS resources(
-                        id INTEGER PRIMARY KEY ASC,
-                        resource TEXT);"""
-        bdb.create_table(self.db_file, resource_stmt)
-
-    @property
-    def resources(self):
-        """Get resources from the database."""
-        return Resources(self.db_file)
-
-
-def open_db(db_file):
-    """Open and return a new database."""
-    return SchedDB(db_file)

hpc_beeflow-0.1.11.dev5/beeflow/scheduler/README.md

@@ -1,40 +0,0 @@
-# Scheduler
-
-This directory contains the initial scheduler implementation which will
-connect to the other BEE components through a REST API.
-
-- `scheduler.py` - REST API code
-- `sched_types.py` - Task, workflow and resource representation code
-- `allocation.py` - Base scheduling code
-- `algorithms.py` - Code implementing the various scheduling algorithms
-
-The scheduler is currently designed to allow for scheduling one set of
-independent tasks at a time. So if a first set of tasks must run before a
-second set of tasks, then that first set must be scheduled first. When the
-first set of tasks has been scheduled and the tasks have completed, then the
-second set of tasks may be scheduled to run.
-
-Each task can have a dict of requirements which specify the resources
-that they need to run on. Supported requirement values, right now, are:
-
-- `max_runtime` - `int`
-- `cores` - `int`
-
-## MARS
-
-The MARS part of the scheduler uses reinforcement learning to schedule tasks.
-Before using this algorithm you must have a trained model. The default trained
-model is located in `./model` but can be changed in the config file.
-
-To train a model you will need to run `beeflow/scheduler/train.py`. This will
-require a schedule log that is produced by running the scheduler on a number
-of workflows. If defaults are used, training can be as simple as running
-`python beeflow/scheduler/train.py` which will output the model in the
-default directory `./model`.
-
-### Evaluation
-
-To evaluate MARS you can run `python beeflow/scheduler/evaluate.py` on a
-specific workflow log file. This will output a number of graphs showing a
-performance comparison of the MARS scheduler with other algorithms (like FCFS
-and Backfill).