hpc-beeflow 0.1.12.dev0__tar.gz → 0.1.12.dev1__tar.gz

{hpc_beeflow-0.1.12.dev0 → hpc_beeflow-0.1.12.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hpc-beeflow
-Version: 0.1.12.dev0
+Version: 0.1.12.dev1
 Summary: Orchestrates complex simulation workflows through coordination, execution, and monitoring, with support for containerized applications.
 License-File: LICENSE
 Keywords: bee,hpc,workflow,cluster,computing
@@ -110,8 +110,8 @@ BEE Sites
      - ``0.1.11``
      - `https://lanl.github.io/BEE/ <https://lanl.github.io/BEE/>`__
    * - Pre-release
-     - ``0.1.11dev5``
-     - `https://lanl.github.io/BEE/0.1.11dev5/ <https://lanl.github.io/BEE/0.1.11dev5/>`__
+     - ``0.1.12dev0``
+     - `https://lanl.github.io/BEE/0.1.12dev1/ <https://lanl.github.io/BEE/0.1.12dev1/>`__
 
 
 Contact

{hpc_beeflow-0.1.12.dev0 → hpc_beeflow-0.1.12.dev1}/README.rst

@@ -50,8 +50,8 @@ BEE Sites
      - ``0.1.11``
      - `https://lanl.github.io/BEE/ <https://lanl.github.io/BEE/>`__
    * - Pre-release
-     - ``0.1.11dev5``
-     - `https://lanl.github.io/BEE/0.1.11dev5/ <https://lanl.github.io/BEE/0.1.11dev5/>`__
+     - ``0.1.12dev0``
+     - `https://lanl.github.io/BEE/0.1.12dev1/ <https://lanl.github.io/BEE/0.1.12dev1/>`__
 
 
 Contact

{hpc_beeflow-0.1.12.dev0 → hpc_beeflow-0.1.12.dev1}/beeflow/client/core.py

@@ -215,7 +215,7 @@ def init_components(remote=False):
     # Slurmrestd will be started only if we're running with Slurm and
     # slurm::use_commands is not True
 
-    @mgr.component('wf_manager', ('scheduler', 'celery'))
+    @mgr.component('wf_manager', ('celery',))
     def start_wfm():
         """Start the WFM."""
         fp = open_log('wf_manager')
@@ -233,14 +233,6 @@ def init_components(remote=False):
         return launch_with_gunicorn('beeflow.task_manager.task_manager:create_app()',
                                     paths.tm_socket(), stdout=fp, stderr=fp)
 
-    @mgr.component('scheduler', ())
-    def start_scheduler():
-        """Start the scheduler."""
-        fp = open_log('scheduler')
-        # Using a function here because of the funny way that the scheduler's written
-        return launch_with_gunicorn('beeflow.scheduler.scheduler:create_app()',
-                                    paths.sched_socket(), stdout=fp, stderr=fp)
-
     if remote:
         @mgr.component('remote_api', ('wf_manager', 'task_manager'))
         def start_remote_api():
@@ -496,7 +488,7 @@ def start(foreground: bool = typer.Option(False, '--foreground', '-F',
     # Create the log path if it doesn't exist yet
     path = paths.log_path()
     os.makedirs(path, exist_ok=True)
-    base_components = ['wf_manager', 'task_manager', 'scheduler']
+    base_components = ['wf_manager', 'task_manager']
     if foreground:
         try:
             Beeflow(mgr, base_components).loop()
@@ -598,14 +590,31 @@ def handle_rm_error(err, dir_to_check, wf_list):
 def reset(archive: bool = typer.Option(False, '--archive', '-a',
                                        help='Archive bee_workdir  before removal')):
     """Stop all components and delete the bee_workdir directory."""
-    # Check workflow states; warn if there are active states.
     reset_hn = socket.gethostname()
-    bee_client.check_hostname(reset_hn)
-    workflow_list = bee_client.get_wf_list()
+
+    # Check if beeflow is running
+    sock_path = paths.beeflow_socket()
+    beeflow_running = False
+    workflow_list = []
+
+    if os.path.exists(sock_path):
+        try:
+            resp = cli_connection.send(sock_path, {'type': 'status'})
+            if resp is not None:
+                beeflow_running = True
+                bee_client.check_hostname(reset_hn)
+                workflow_list = bee_client.get_wf_list()
+        except (ConnectionResetError, ConnectionRefusedError,
+                cli_connection.BeeflowConnectionError, OSError):
+            # Beeflow socket exists but is not responsive
+            pass
+
+    # Check workflow states; warn if there are active states.
     active_states = {'Running', 'Paused', 'Initializing', 'Waiting'}
     caution = ""
-    if {item for row in workflow_list for item in row}.intersection(active_states):
-        caution = """
+    if beeflow_running and workflow_list:
+        if {item for row in workflow_list for item in row}.intersection(active_states):
+            caution = """
         **************************************************************
           Caution: There are active workflows! They will be removed!
           Try 'beeflow list' to view them.
@@ -616,9 +625,16 @@ def reset(archive: bool = typer.Option(False, '--archive', '-a',
     warn(f"\n    A reset will remove this directory: {dir_to_delete}\n")
     if archive:
         print("    Archive flag is set: logs, workflows and containers will be backed up.")
+    else:
+        warn("    If you want to save logs, workflows and containers:")
+        warn("      answer no and rerun with --archive flag.")
+
+    status_msg = "running" if beeflow_running else "not running"
+    print(f"\n    Beeflow is currently {status_msg}.")
+
     print("""
     A reset will:
-        Shutdown beeflow and all BEE components.
+        Shutdown beeflow and all BEE components (if running).
         Delete the bee_workdir directory which results in:
             Removing the archive of all workflows.
                 (unless archives is configured elsewhere).
@@ -636,11 +652,14 @@ def reset(archive: bool = typer.Option(False, '--archive', '-a',
             sys.exit()
         elif absolutely_sure in ("y", "yes"):
             # Stop all of the beeflow processes
-            stop("quiet")
-            print("Beeflow is shutting down.")
-            print("Waiting for components to cleanly stop.")
-            # This wait is essential. It takes a minute to shut down.
-            time.sleep(5)
+            if beeflow_running:
+                stop("quiet")
+                print("Beeflow is shutting down.")
+                print("Waiting for components to cleanly stop.")
+                # This wait is essential. It takes a minute to shut down.
+                time.sleep(5)
+            else:
+                print("Beeflow is not running, proceeding to remove bee_workdir.")
 
             # Save the bee_workdir directory if the archive option was set
             if archive:

{hpc_beeflow-0.1.12.dev0 → hpc_beeflow-0.1.12.dev1}/beeflow/common/log.py

@@ -18,6 +18,6 @@ def setup(name):
     log.setLevel(LEVEL)
     handler = logging.StreamHandler()
     handler.setLevel(logging.DEBUG)
-    handler.setFormatter(logging.Formatter('[%(asctime)s] %(name)s:%(funcName)s(): %(msg)s'))
+    handler.setFormatter(logging.Formatter('[%(asctime)s] %(name)s:%(funcName)s(): %(message)s'))
     log.addHandler(handler)
     return log

{hpc_beeflow-0.1.12.dev0 → hpc_beeflow-0.1.12.dev1}/beeflow/common/paths.py

@@ -35,11 +35,6 @@ def remote_socket():
     return os.path.join(_sockdir(), 'remote.sock')
 
 
-def sched_socket():
-    """Get the scheduler socket."""
-    return os.path.join(_sockdir(), 'scheduler.sock')
-
-
 def slurm_socket():
     """Get the slurm socket (for slurmrestd)."""
     return os.path.join(_sockdir(), 'slurmrestd.sock')

{hpc_beeflow-0.1.12.dev0 → hpc_beeflow-0.1.12.dev1}/beeflow/tests/test_remote.py

@@ -76,7 +76,6 @@ def test_get_core_status(mocker):
     # Mocked response from the daemon
     mock_response = {
         "components": {
-            "scheduler": "running",
             "wf_manager": "running",
             "task_manager": "running"
         }
@@ -89,7 +88,6 @@ def test_get_core_status(mocker):
 
     assert response.status_code == 200
     assert response.json() == {
-        "scheduler": "running",
         "wf_manager": "running",
         "task_manager": "running"
     }

{hpc_beeflow-0.1.12.dev0 → hpc_beeflow-0.1.12.dev1}/beeflow/tests/test_remote_client.py

@@ -243,7 +243,7 @@ def test_core_status_success(mocker):
     mock_run = mocker.patch("subprocess.run")
     mock_run.return_value = subprocess.CompletedProcess(
             ["curl", "ssh_target:1234/core/status/"], returncode=0,
-            stdout='{"redis":"RUNNING","scheduler":"RUNNING","celery":"RUNNING",'
+            stdout='{"redis":"RUNNING","celery":"RUNNING",'
             '"slurmrestd":"RUNNING","wf_manager":"RUNNING","task_manager":"RUNNING",'
             '"remote_api":"RUNNING","neo4j-database":"RUNNING"}',
             stderr=""
@@ -252,7 +252,7 @@ def test_core_status_success(mocker):
     result = runner.invoke(app, ["core-status", "ssh_target"])
 
     assert result.exit_code == 0
-    expected_output = '{"redis":"RUNNING","scheduler":"RUNNING","celery":"RUNNING",' \
+    expected_output = '{"redis":"RUNNING","celery":"RUNNING",' \
                       '"slurmrestd":"RUNNING","wf_manager":"RUNNING","task_manager":"RUNNING",' \
                       '"remote_api":"RUNNING","neo4j-database":"RUNNING"}'
     assert expected_output in result.output

{hpc_beeflow-0.1.12.dev0 → hpc_beeflow-0.1.12.dev1}/beeflow/tests/test_wf_manager.py

@@ -140,13 +140,13 @@ def test_start_workflow(client, mocker):
     """Test starting a workflow."""
     mocker.patch('beeflow.wf_manager.resources.wf_utils.get_workflow_interface',
                  return_value=MockWFI())
-    mocker.patch('beeflow.wf_manager.resources.wf_utils.submit_tasks_tm', return_value=None)
-    mocker.patch('beeflow.wf_manager.resources.wf_utils.submit_tasks_scheduler', return_value=None)
+    submit_tasks_tm = mocker.patch('beeflow.wf_manager.resources.wf_utils.submit_tasks_tm', return_value=None)
     mocker.patch('beeflow.wf_manager.resources.wf_utils.update_wf_status', return_value=None)
     mocker.patch('beeflow.tests.mocks.MockWFI.get_workflow_state', return_value='No Start')
     resp = client().post(f'/bee_wfm/v1/jobs/{WF_ID}')
     assert resp.status_code == 200
     assert resp.json['msg'] == 'Workflow started successfully'
+    submit_tasks_tm.assert_called()
 
 
 def test_workflow_status(client, mocker, setup_teardown_workflow):
@@ -266,8 +266,7 @@ def test_resume_workflow(client, mocker, setup_teardown_workflow, temp_db):
                  return_value=MockWFI())
     mocker.patch('beeflow.tests.mocks.MockWFI.get_workflow_state',
                  return_value='Paused')
-    mocker.patch('beeflow.wf_manager.resources.wf_utils.submit_tasks_tm', return_value=None)
-    mocker.patch('beeflow.wf_manager.resources.wf_utils.submit_tasks_scheduler', return_value=None)
+    submit_tasks_tm = mocker.patch('beeflow.wf_manager.resources.wf_utils.submit_tasks_tm', return_value=None)
     mocker.patch('beeflow.wf_manager.resources.wf_utils.update_wf_status', return_value=None)
 
     wf_utils.update_wf_status(WF_ID, 'Paused')
@@ -275,3 +274,4 @@ def test_resume_workflow(client, mocker, setup_teardown_workflow, temp_db):
     resp = client().patch(f'/bee_wfm/v1/jobs/{WF_ID}', json=request)
     assert resp.json['msg'] == 'Workflow Resumed'
     assert resp.status_code == 200
+    submit_tasks_tm.assert_called()

{hpc_beeflow-0.1.12.dev0 → hpc_beeflow-0.1.12.dev1}/beeflow/wf_manager/resources/wf_actions.py

@@ -134,7 +134,7 @@ class WFActions(Resource):
             wf_utils.update_wf_status(wf_id, "Running")
             wfi = wf_utils.get_workflow_interface(wf_id)
             tasks = wfi.get_ready_tasks()
-            wf_utils.schedule_submit_tasks(wf_id, tasks)
+            wf_utils.submit_tasks_tm(wf_id, tasks)
 
             log.info(f"Workflow {wf_id} Resumed")
             resp = WorkflowActionResponse(msg="Workflow Resumed").model_dump(), 200

{hpc_beeflow-0.1.12.dev0 → hpc_beeflow-0.1.12.dev1}/beeflow/wf_manager/resources/wf_update.py

@@ -135,7 +135,7 @@ class WFUpdate(Resource):
                 return True
             # Submit the restart task
             tasks = [new_task]
-            wf_utils.schedule_submit_tasks(state_update.wf_id, tasks)
+            wf_utils.submit_tasks_tm(state_update.wf_id, tasks)
             log.info(f'Task {state_update.task_id} restarted')
             return True
         return False
@@ -153,7 +153,7 @@ class WFUpdate(Resource):
             wf_utils.copy_task_output(task)
             tasks = wfi.finalize_task(task)
             if tasks and wf_state not in ('Paused', 'Cancelled'):
-                wf_utils.schedule_submit_tasks(state_update.wf_id, tasks)
+                wf_utils.submit_tasks_tm(state_update.wf_id, tasks)
 
         # If the job failed, fail the dependent tasks
         # TIMEOUT states should only be seen here if they can't restart

{hpc_beeflow-0.1.12.dev0 → hpc_beeflow-0.1.12.dev1}/beeflow/wf_manager/resources/wf_utils.py

@@ -17,12 +17,6 @@ from beeflow.common.connection import Connection
 from beeflow.common import paths
 from beeflow.common.db import wfm_db
 from beeflow.common.db.bdb import connect_db
-from beeflow.scheduler.models import (
-    ScheduleTasksRequest,
-    ScheduleTasksResponse,
-    SchedulerTask,
-)
-
 from beeflow.task_manager.models import SubmitTasksRequest
 from beeflow.common.deps.neo4j_manager import connect_neo4j_driver
 
@@ -170,7 +164,6 @@ def tm_url():
 
 # Base URLs for the TM and the Scheduler
 TM_URL = "bee_tm/v1/task/"
-SCHED_URL = "bee_sched/v1/"
 
 
 def _connect_tm():
@@ -178,32 +171,13 @@ def _connect_tm():
     return Connection(paths.tm_socket())
 
 
-def sched_url():
-    """Get Scheduler url."""
-    db = connect_db(wfm_db, get_db_path())
-    scheduler = "bee_sched/v1/"
-    # sched_listen_port = bc.get('scheduler', 'listen_port')
-    sched_listen_port = db.info.get_port("sched")
-    return f"http://127.0.0.1:{sched_listen_port}/{scheduler}"
-
-
-def _connect_scheduler():
-    """Return a connection to the Scheduler."""
-    return Connection(paths.sched_socket())
-
-
-def _resource(component, tag=""):
-    """Access Task Manager or Scheduler."""
-    if component == "tm":
-        return TM_URL + str(tag)
-    if component == "sched":
-        return SCHED_URL + str(tag)
-
-    raise ValueError(f"Invalid component: {component}")
+def _taskmanager(tag=""):
+    """Access Task Manager resources."""
+    return TM_URL + str(tag)
 
 
 # Submit tasks to the TM
-def submit_tasks_tm(wf_id, tasks, allocation):  # pylint: disable=W0613
+def submit_tasks_tm(wf_id, tasks):
     """Submit a task to the task manager."""
     wfi = get_workflow_interface(wf_id)
     # Serialize task with json
@@ -212,7 +186,7 @@ def submit_tasks_tm(wf_id, tasks, allocation):  # pylint: disable=W0613
     try:
         conn = _connect_tm()
         resp = conn.post(
-            _resource("tm"),
+            _taskmanager(),
             json=SubmitTasksRequest(tasks=tasks).model_dump(),
             timeout=5,
         )
@@ -228,48 +202,6 @@ def submit_tasks_tm(wf_id, tasks, allocation):  # pylint: disable=W0613
         log.info("Submit task to TM returned bad status: %s", resp.status_code)
 
 
-def tasks_to_sched(tasks):
-    """Convert gdb tasks to sched tasks."""
-    sched_tasks = []
-    for task in tasks:
-        sched_task = SchedulerTask(
-            workflow_name="workflow",
-            task_name=task.name,
-            requirements={"max_runtime": 1, "nodes": 1},
-        )
-        sched_tasks.append(sched_task)
-    return sched_tasks
-
-
-def submit_tasks_scheduler(tasks):
-    """Submit a list of tasks to the scheduler."""
-    sched_tasks = tasks_to_sched(tasks)
-    # The workflow name will eventually be added to the wfi workflow object
-    try:
-        conn = _connect_scheduler()
-        resp = conn.put(
-            _resource("sched", "workflows/workflow/jobs"),
-            json=ScheduleTasksRequest(tasks=sched_tasks).model_dump(),
-            timeout=5,
-        )
-    except requests.exceptions.ConnectionError:
-        log.error("Unable to connect to scheduler to submit tasks.")
-        return "Did not work"
-
-    if resp.status_code != 200:
-        log.info("Something bad happened %s", resp.status_code)
-        return "Did not work"
-    return ScheduleTasksResponse.model_validate(resp.json()).tasks
-
-
-def schedule_submit_tasks(wf_id, tasks):
-    """Schedule and then submit tasks to the TM."""
-    # Submit ready tasks to the scheduler
-    allocation = submit_tasks_scheduler(tasks)
-    # Submit tasks to TM
-    submit_tasks_tm(wf_id, tasks, allocation)
-
-
 def setup_workflow(wf_id, wf_name, wf_dir, wf_workdir, no_start, workflow=None, # pylint: disable=W0613
                    tasks=None):
     """Initialize Workflow and Tasks then start workflow in separate process"""
@@ -318,7 +250,7 @@ def start_workflow(wf_id):
             wfi.set_task_state(task.id, "WAITING")
     wfi.execute_workflow()
     tasks = wfi.get_ready_tasks()
-    schedule_submit_tasks(wf_id, tasks)
+    submit_tasks_tm(wf_id, tasks)
     update_wf_status(wf_id, "Running")
     return True
 

{hpc_beeflow-0.1.12.dev0 → hpc_beeflow-0.1.12.dev1}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "hpc-beeflow"
-version = "0.1.12dev"
+version = "0.1.12dev1"
 description = "Orchestrates complex simulation workflows through coordination, execution, and monitoring, with support for containerized applications."
 
 

hpc_beeflow-0.1.12.dev0/beeflow/common/db/sched_db.py

@@ -1,58 +0,0 @@
-"""Scheduler database code."""
-
-import jsonpickle
-from beeflow.common.db import bdb
-
-
-class Resources:
-    """Resources wrapper class."""
-
-    def __init__(self, db_file):
-        """Create the resources class."""
-        self.db_file = db_file
-
-    def __iter__(self):
-        """Create an iterator over the resources."""
-        stmt = 'SELECT id, resource FROM resources'
-        result = bdb.getall(self.db_file, stmt)
-        for rslt in result:
-            resource = jsonpickle.decode(rslt[1])
-            yield resource
-
-    def extend(self, resources):
-        """Add a list of new resources."""
-        for resource in resources:
-            data = jsonpickle.encode(resource)
-            stmt = 'INSERT INTO resources (resource) VALUES (?)'
-            bdb.run(self.db_file, stmt, [data])
-
-    def clear(self):
-        """Remove all resources."""
-        clear_stmt = 'DELETE FROM resources'
-        bdb.run(self.db_file, clear_stmt)
-
-
-class SchedDB:
-    """Scheduler database."""
-
-    def __init__(self, db_file):
-        """Construct a new scheduler database connection."""
-        self.db_file = db_file
-        self._init_tables()
-
-    def _init_tables(self):
-        """Initialze the scheduler tables if they don't exist."""
-        resource_stmt = """CREATE TABLE IF NOT EXISTS resources(
-                        id INTEGER PRIMARY KEY ASC,
-                        resource TEXT);"""
-        bdb.create_table(self.db_file, resource_stmt)
-
-    @property
-    def resources(self):
-        """Get resources from the database."""
-        return Resources(self.db_file)
-
-
-def open_db(db_file):
-    """Open and return a new database."""
-    return SchedDB(db_file)

hpc_beeflow-0.1.12.dev0/beeflow/scheduler/README.md

@@ -1,40 +0,0 @@
-# Scheduler
-
-This directory contains the initial scheduler implementation which will
-connect to the other BEE components through a REST API.
-
-- `scheduler.py` - REST API code
-- `sched_types.py` - Task, workflow and resource representation code
-- `allocation.py` - Base scheduling code
-- `algorithms.py` - Code implementing the various scheduling algorithms
-
-The scheduler is currently designed to allow for scheduling one set of
-independent tasks at a time. So if a first set of tasks must run before a
-second set of tasks, then that first set must be scheduled first. When the
-first set of tasks has been scheduled and the tasks have completed, then the
-second set of tasks may be scheduled to run.
-
-Each task can have a dict of requirements which specify the resources
-that they need to run on. Supported requirement values, right now, are:
-
-- `max_runtime` - `int`
-- `cores` - `int`
-
-## MARS
-
-The MARS part of the scheduler uses reinforcement learning to schedule tasks.
-Before using this algorithm you must have a trained model. The default trained
-model is located in `./model` but can be changed in the config file.
-
-To train a model you will need to run `beeflow/scheduler/train.py`. This will
-require a schedule log that is produced by running the scheduler on a number
-of workflows. If defaults are used, training can be as simple as running
-`python beeflow/scheduler/train.py` which will output the model in the
-default directory `./model`.
-
-### Evaluation
-
-To evaluate MARS you can run `python beeflow/scheduler/evaluate.py` on a
-specific workflow log file. This will output a number of graphs showing a
-performance comparison of the MARS scheduler with other algorithms (like FCFS
-and Backfill).

hpc_beeflow-0.1.12.dev0/beeflow/scheduler/__init__.py

@@ -1 +0,0 @@
-"""beeflow scheduler module."""