hpc-beeflow 0.1.9.dev4__tar.gz → 0.1.10.dev0__tar.gz

{hpc_beeflow-0.1.9.dev4 → hpc_beeflow-0.1.10.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hpc-beeflow
-Version: 0.1.9.dev4
+Version: 0.1.10.dev0
 Summary: A software package for containerizing HPC applications and managing job workflows
 Home-page: https://github.com/lanl/BEE
 Keywords: bee,hpc,workflow,cluster,computing
@@ -34,17 +34,21 @@ Requires-Dist: celery[redis,sqlalchemy] (>=5.3.4,<6.0.0)
 Requires-Dist: cffi (>=1.15.1,<2.0.0)
 Requires-Dist: cwl-utils (>=0.16,<0.17)
 Requires-Dist: docutils (==0.18.1)
+Requires-Dist: fastapi (==0.109.2)
 Requires-Dist: flask_restful (==0.3.9)
 Requires-Dist: google-api-python-client (>=2.66.0,<3.0.0) ; extra == "cloud-extras"
+Requires-Dist: graphviz (>=0.20.3,<0.21.0)
 Requires-Dist: gunicorn (>=20.1,<23.0)
 Requires-Dist: jsonpickle (>=2.2.0,<3.0.0)
 Requires-Dist: neo4j (>=5,<6)
+Requires-Dist: networkx (==3.1)
 Requires-Dist: python-daemon (>=2.3.1,<3.0.0)
 Requires-Dist: python-heatclient (>=3.1.0,<4.0.0) ; extra == "cloud-extras"
 Requires-Dist: python-openstackclient (>=6.0.0,<7.0.0) ; extra == "cloud-extras"
-Requires-Dist: requests (<2.29.0)
-Requires-Dist: requests-unixsocket (>=0.3.0,<0.4.0)
+Requires-Dist: requests (>=2.32.3,<3.0.0)
+Requires-Dist: requests-unixsocket2 (>=0.4.2,<0.5.0)
 Requires-Dist: typer (>=0.5.0,<0.6.0)
+Requires-Dist: uvicorn (==0.27.1)
 Project-URL: Repository, https://github.com/lanl/BEE
 Description-Content-Type: text/x-rst
 

{hpc_beeflow-0.1.9.dev4 → hpc_beeflow-0.1.10.dev0}/beeflow/client/bee_client.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 """bee-client.
 
-This script provides an client interface to the user to manage workflows.
+This script provides a client interface to the user to manage workflows.
 Capablities include submitting, starting, listing, pausing and cancelling workflows.
 """
 import os
@@ -11,6 +11,7 @@ import inspect
 import pathlib
 import shutil
 import subprocess
+import tarfile
 import tempfile
 import textwrap
 import time
@@ -18,6 +19,7 @@ import importlib.metadata
 import jsonpickle
 import requests
 import typer
+import yaml
 
 from beeflow.common import config_driver
 from beeflow.common.cli import NaturalOrderGroup
@@ -27,6 +29,8 @@ from beeflow.common.parser import CwlParser
 from beeflow.common.wf_data import generate_workflow_id
 from beeflow.client import core
 from beeflow.wf_manager.resources import wf_utils
+from beeflow.common.db import client_db
+from beeflow.common.db import bdb
 
 # Length of a shortened workflow ID
 short_id_len = 6 #noqa: Not a constant
@@ -39,6 +43,7 @@ _INTERACTIVE = False
 
 
 logging.basicConfig(level=logging.WARNING)
+logging.getLogger("neo4j").setLevel(logging.WARNING)
 WORKFLOW_MANAGER = 'bee_wfm/v1/jobs/'
 
 
@@ -50,6 +55,41 @@ class ClientError(Exception):
         self.args = args
 
 
+def warn(*pargs):
+    """Print a red warning message."""
+    typer.secho(' '.join(pargs), fg=typer.colors.RED, file=sys.stderr)
+
+
+def db_path():
+    """Return the client database path."""
+    bee_workdir = config_driver.BeeConfig.get('DEFAULT', 'bee_workdir')
+    return os.path.join(bee_workdir, 'client.db')
+
+
+def setup_hostname(start_hn):
+    """Set up front end name when beeflow core start is returned."""
+    db = bdb.connect_db(client_db, db_path())
+    db.info.set_hostname(start_hn)
+
+
+def get_hostname():
+    """Check if beeflow is running somewhere else."""
+    db = bdb.connect_db(client_db, db_path())
+    curr_hn = db.info.get_hostname()
+    return curr_hn
+
+
+def check_hostname(curr_hn):
+    """Check current front end name matches the one beeflow was started on."""
+    db = bdb.connect_db(client_db, db_path())
+    start_hn = db.info.get_hostname()
+    if start_hn and curr_hn != start_hn:  # noqa: don't use set instead
+        warn(f'beeflow was started on "{start_hn}" and you are trying to '
+             f'run a command on "{curr_hn}".')
+    if start_hn == "":
+        warn('beeflow has not been started!')
+
+
 def error_exit(msg, include_caller=True):
     """Print a message and exit or raise an error with that message."""
     if include_caller:
@@ -112,7 +152,11 @@ def get_wf_list():
         conn = _wfm_conn()
         resp = conn.get(_url(), timeout=60)
     except requests.exceptions.ConnectionError:
-        error_exit('Could not reach WF Manager.')
+        if get_hostname() == "":
+            warn('beeflow has not been started!')
+            sys.exit(1)
+        else:
+            error_exit('Could not reach WF Manager.')
 
     if resp.status_code != requests.codes.okay:  # pylint: disable=no-member
         error_exit('WF Manager did not return workflow list')
@@ -192,7 +236,7 @@ def submit(wf_name: str = typer.Argument(..., help='the workflow name'),  # pyli
            main_cwl: str = typer.Argument(...,
            help='filename of main CWL (if using CWL tarball), '
            + 'path of main CWL (if using CWL directory)'),
-           yaml: str = typer.Argument(...,
+           yaml_file: str = typer.Argument(...,
            help='filename of yaml file (if using CWL tarball), '
            + 'path of yaml file (if using CWL directory)'),
            workdir: pathlib.Path = typer.Argument(...,
@@ -215,16 +259,17 @@ def submit(wf_name: str = typer.Argument(..., help='the workflow name'),  # pyli
         if os.path.isdir(wf_path):
             print("Detected directory instead of packaged workflow. Packaging Directory...")
             main_cwl_path = pathlib.Path(main_cwl).resolve()
-            yaml_path = pathlib.Path(yaml).resolve()
+            yaml_path = pathlib.Path(yaml_file).resolve()
 
             if not main_cwl_path.exists():
                 error_exit(f'Main CWL file {main_cwl} does not exist')
             if not yaml_path.exists():
-                error_exit(f'YAML file {yaml} does not exist')
+                error_exit(f'YAML file {yaml_file} does not exist')
 
             # Packaging in temp dir, after copying alternate cwl_main or yaml file
             cwl_indir = is_parent(wf_path, main_cwl_path)
             yaml_indir = is_parent(wf_path, yaml_path)
+
             # Always create temp dir for the workflow
             tempdir_path = pathlib.Path(tempfile.mkdtemp())
             tempdir_wf_path = pathlib.Path(tempdir_path / wf_name)
@@ -232,15 +277,16 @@ def submit(wf_name: str = typer.Argument(..., help='the workflow name'),  # pyli
             if not cwl_indir:
                 shutil.copy2(main_cwl, tempdir_wf_path)
             if not yaml_indir:
-                shutil.copy2(yaml, tempdir_wf_path)
+                shutil.copy2(yaml_file, tempdir_wf_path)
             package_path = package(tempdir_wf_path, tempdir_path)
         else:
             package_path = wf_path
+
         # Untar and parse workflow
         untar_path = pathlib.Path(tempfile.mkdtemp())
         untar_wf_path = unpackage(package_path, untar_path)
         main_cwl_path = untar_wf_path / pathlib.Path(main_cwl).name
-        yaml_path = untar_wf_path / pathlib.Path(yaml).name
+        yaml_path = untar_wf_path / pathlib.Path(yaml_file).name
         parser = CwlParser()
         workflow_id = generate_workflow_id()
         workflow, tasks = parser.parse_workflow(workflow_id, str(main_cwl_path),
@@ -304,16 +350,19 @@ def submit(wf_name: str = typer.Argument(..., help='the workflow name'),  # pyli
 
     # Store provided arguments in text file for future reference
     wf_dir = wf_utils.get_workflow_dir(wf_id)
-    sub_wf_dir = wf_dir + "/submit_command_args.txt"
+    sub_wf_dir = wf_dir + "/submit_command_args.yaml"
+
+    cmd = {
+        'wf_name': wf_name,
+        'wf_path': str(wf_path),
+        'main_cwl': main_cwl,
+        'yaml': yaml_file,
+        'workdir': workdir,
+        'wf_id': wf_id
+    }
 
-    f_name = open(sub_wf_dir, "w", encoding="utf-8")
-    f_name.write(f"wf_name: {wf_name}\n")
-    f_name.write(f"wf_path: {wf_path}\n")
-    f_name.write(f"main_cwl: {main_cwl}\n")
-    f_name.write(f"yaml: {yaml}\n")
-    f_name.write(f"workdir: {workdir}\n")
-    f_name.write(f"wf_id: {wf_id}")
-    f_name.close()
+    with open(sub_wf_dir, "w", encoding='utf-8') as command_file:
+        yaml.dump(cmd, command_file)
 
     return wf_id
 
@@ -457,7 +506,10 @@ def query(wf_id: str = typer.Argument(..., callback=match_short_id)):
     wf_status = resp.json()['wf_status']
     typer.echo(wf_status)
     for _task_id, task_name, task_state in tasks_status:
-        typer.echo(f'{task_name}--{task_state}')
+        if wf_status == 'No Start':
+            typer.echo(f'{task_name}')
+        else:
+            typer.echo(f'{task_name}--{task_state}')
 
     logging.info('Query workflow:  {resp.text}')
     return wf_status, tasks_status
@@ -518,7 +570,7 @@ def cancel(wf_id: str = typer.Argument(..., callback=match_short_id)):
     """Cancel a paused or running workflow."""
     long_wf_id = wf_id
     wf_status = get_wf_status(wf_id)
-    if wf_status in ('Running', 'Paused'):
+    if wf_status in ('Running', 'Paused', 'No Start'):
         try:
             conn = _wfm_conn()
             resp = conn.delete(_resource(long_wf_id), json={'option': 'cancel'}, timeout=60)
@@ -570,6 +622,26 @@ def reexecute(wf_name: str = typer.Argument(..., help='The workflow name'),
     workdir = os.path.abspath(workdir)
     if not os.path.exists(workdir):
         error_exit(f"Workflow working directory \"{workdir}\" doesn't exist")
+    cwl_path = pathlib.Path(tempfile.mkdtemp())
+    archive_id = str(wf_path.stem)
+    with tarfile.open(wf_path) as archive:
+        archive_cmd = yaml.load(archive.extractfile(
+            str(pathlib.Path(archive_id) / 'submit_command_args.yaml')).read(),
+            Loader=yaml.Loader)
+
+        cwl_files = [
+            tarinfo for tarinfo in archive.getmembers()
+            if tarinfo.name.startswith(archive_id + '/cwl_files/')
+            and tarinfo.isreg()
+        ]
+        for path in cwl_files:
+            path.name = os.path.basename(path.name)
+        archive.extractall(path=cwl_path, members=cwl_files)
+
+        main_cwl = cwl_path / pathlib.Path(archive_cmd['main_cwl']).name
+        yaml_file = cwl_path / pathlib.Path(archive_cmd['yaml']).name
+
+        return submit(wf_name, pathlib.Path(cwl_path), main_cwl, yaml_file, pathlib.Path(workdir))
 
     data = {
         'wf_filename': os.path.basename(wf_path).encode(),
@@ -593,6 +665,42 @@ def reexecute(wf_name: str = typer.Argument(..., help='The workflow name'),
     return wf_id
 
 
+@app.command()
+def dag(wf_id: str = typer.Argument(..., callback=match_short_id),
+        output_dir: pathlib.Path = typer.Argument(...,
+        help='Path to the where the dag output will be'),
+        no_dag_dir: bool = typer.Option(False, '--no-dag-dir',
+        help='do not make a subdirectory within ouput_dir for the dags')):
+    """Export a DAG of the workflow to a GraphML file."""
+    output_dir = output_dir.resolve()
+    # Make sure output_dir is an absolute path and exists
+    output_dir = os.path.expanduser(output_dir)
+    output_dir = os.path.abspath(output_dir)
+    if not os.path.exists(output_dir):
+        error_exit(f"Path for dag directory \"{output_dir}\" doesn't exist")
+
+    # output_dir must be a string
+    output_dir = str(output_dir)
+    # Check if the workflow is archived
+    wf_status = get_wf_status(wf_id)
+    if wf_status == 'Archived':
+        copy_dag_in_archive = False
+        bee_workdir = wf_utils.get_bee_workdir()
+        mount_dir = os.path.join(bee_workdir, 'gdb_mount')
+        graphmls_dir = mount_dir + '/graphmls'
+        typer.secho("Workflow has been archived. All new DAGs will look the same as the one "
+                    "in the archive directory.",
+                    fg=typer.colors.MAGENTA)
+    else:
+        copy_dag_in_archive = True
+        wf_dir = wf_utils.get_workflow_dir(wf_id)
+        graphmls_dir = wf_dir + '/graphmls'
+        os.makedirs(graphmls_dir, exist_ok=True)
+    wf_utils.export_dag(wf_id, output_dir, graphmls_dir, no_dag_dir, copy_dag_in_archive)
+    typer.secho(f"DAG for workflow {_short_id(wf_id)} has been exported successfully.",
+                fg=typer.colors.GREEN)
+
+
 @app.callback(invoke_without_command=True)
 def version_callback(version: bool = False):
     """Beeflow."""
@@ -615,4 +723,5 @@ if __name__ == "__main__":
 
 # Ignore W0511: This allows us to have TODOs in the code
 # Ignore R1732: Significant code restructuring required to fix
+# Ignore R1714: Not using a set instead
 # pylama:ignore=W0511,R1732

{hpc_beeflow-0.1.9.dev4 → hpc_beeflow-0.1.10.dev0}/beeflow/client/core.py

@@ -9,6 +9,7 @@ services is specified using the appropriate flag(s) then ONLY those services
 will be started.
 """
 import os
+import re
 import signal
 import subprocess
 import socket
@@ -17,12 +18,16 @@ import shutil
 import datetime
 import time
 import importlib.metadata
+from pathlib import Path
+import packaging
 
 import daemon
 import typer
 
+
 from beeflow.client import bee_client
 from beeflow.common.config_driver import BeeConfig as bc
+from beeflow.common.config_driver import AlterConfig
 from beeflow.common import cli_connection
 from beeflow.common import paths
 from beeflow.wf_manager.resources import wf_utils
@@ -32,6 +37,9 @@ from beeflow.common.deps import neo4j_manager
 from beeflow.common.deps import redis_manager
 
 
+REPO_PATH = Path(*Path(__file__).parts[:-3])
+
+
 class ComponentManager:
     """Component manager class."""
 
@@ -44,7 +52,11 @@ class ComponentManager:
         """Return a decorator function to be called."""
 
         def wrap(fn):
-            """Add the component to the list."""
+            """Check to see if any components are disabled."""
+            if bc.get('DEFAULT', 'remote_api') is False and 'remote_api' in name:
+                return
+
+            # Add the component to the list.
             self.components[name] = {
                 'fn': fn,
                 'deps': deps,
@@ -156,6 +168,22 @@ def need_slurmrestd():
             and not bc.get('slurm', 'use_commands'))
 
 
+def get_slurmrestd_version():
+    """Get the newest slurmrestd version."""
+    resp = subprocess.run(["slurmrestd", "-s", "list"], check=True, stderr=subprocess.PIPE,
+                          text=True).stderr
+    resp = resp.split("\n")
+    # Confirm slurmrestd format is the same
+    # If the slurmrestd list outputs has changed potentially something else has broken
+    if "Possible OpenAPI plugins" not in resp[0]:
+        print("Slurmrestd OpenAPI format has changed and things may break")
+    api_versions = [line.split('/')[1] for line in resp[1:] if re.search(r"openapi/v\d+\.\d+\.\d+",
+                                                                         line)]
+    # Sort the versions and grab the newest one
+    newest_api = sorted(api_versions, key=packaging.version.Version, reverse=True)[0]
+    return newest_api
+
+
 def init_components():
     """Initialize the components and component manager."""
     mgr = ComponentManager()
@@ -189,6 +217,13 @@ def init_components():
         return launch_with_gunicorn('beeflow.scheduler.scheduler:create_app()',
                                     paths.sched_socket(), stdout=fp, stderr=fp)
 
+    @mgr.component('remote_api', ('wf_manager', 'task_manager'))
+    def start_remote_api():
+        """Start the remote API."""
+        fp = open_log('remote_api')
+        return launch_with_gunicorn('beeflow.remote.remote:create_app()',
+                                    paths.remote_socket(), stdout=fp, stderr=fp)
+
     @mgr.component('celery', ('redis',))
     def celery():
         """Start the celery task queue."""
@@ -231,6 +266,9 @@ def init_components():
             bee_workdir = bc.get('DEFAULT', 'bee_workdir')
             slurmrestd_log = '/'.join([bee_workdir, 'logs', 'restd.log'])
             openapi_version = bc.get('slurm', 'openapi_version')
+            if not openapi_version:
+                # Detect the newest version of the slurmrestd API
+                openapi_version = get_slurmrestd_version()
             slurm_args = f'-s openapi/{openapi_version}'
             # The following adds the db plugin we opted not to use for now
             # slurm_args = f'-s openapi/{openapi_version},openapi/db{openapi_version}'
@@ -285,8 +323,13 @@ def load_check_charliecloud():
         sys.exit(1)
 
 
-def check_dependencies():
+def check_dependencies(backend=False):
     """Check for various dependencies in the environment."""
+    # Check if running on compute node under Slurm scheduler
+    if not backend and os.environ.get('SLURM_JOB_NODELIST') is not None:
+        warn('Slurm job node detected! Beeflow should not be run on a compute node.')
+        warn(f'SLURM_JOB_NODELIST = {os.environ.get("SLURM_JOB_NODELIST")}')
+        sys.exit(1)
     print('Checking dependencies...')
     # Check for Charliecloud and its version
     load_check_charliecloud()
@@ -369,9 +412,19 @@ app = typer.Typer(no_args_is_help=True)
 
 @app.command()
 def start(foreground: bool = typer.Option(False, '--foreground', '-F',
-          help='run in the foreground')):
+          help='run in the foreground'), backend: bool = typer.Option(False, '--backend',
+          '-B', help='allow to run on a backend node')):
     """Start all BEE components."""
-    check_dependencies()
+    start_hn = socket.gethostname()  # hostname when beeflow starts
+    if bee_client.get_hostname() == "":
+        bee_client.setup_hostname(start_hn)  # add to client db
+    elif bee_client.get_hostname() != start_hn:
+        warn(f'Error: beeflow is already running on "{bee_client.get_hostname()}."')
+        sys.exit(1)
+    if backend:  # allow beeflow to run on backend node
+        check_dependencies(backend=True)
+    else:
+        check_dependencies()
     mgr = init_components()
     beeflow_log = paths.log_fname('beeflow')
     sock_path = paths.beeflow_socket()
@@ -397,6 +450,7 @@ def start(foreground: bool = typer.Option(False, '--foreground', '-F',
 
     version = importlib.metadata.version("hpc-beeflow")
     print(f'Starting beeflow {version}...')
+    print(f'Running beeflow on {start_hn}')
     if not foreground:
         print('Run `beeflow core status` for more information.')
     # Create the log path if it doesn't exist yet
@@ -415,20 +469,29 @@ def start(foreground: bool = typer.Option(False, '--foreground', '-F',
 @app.command()
 def status():
     """Check the status of beeflow and the components."""
+    status_hn = socket.gethostname()  # hostname when beeflow core status returned
     resp = cli_connection.send(paths.beeflow_socket(), {'type': 'status'})
     if resp is None:
-        beeflow_log = paths.log_fname('beeflow')
-        warn('Cannot connect to the beeflow daemon, is it running? Check the '
-             f'log at "{beeflow_log}".')
+        bee_client.check_hostname(status_hn)
         sys.exit(1)
     print('beeflow components:')
     for comp, stat in resp['components'].items():
         print(f'{comp} ... {stat}')
 
 
+@app.command()
+def info():
+    """Get information about beeflow's installation."""
+    version = importlib.metadata.version("hpc-beeflow")
+    print(f"Beeflow version: {version}")
+    print(f"bee_workflow directory: {paths.workdir()}")
+    print(f"Log path: {paths.log_path()}")
+
+
 @app.command()
 def stop(query='yes'):
     """Stop the current running beeflow daemon."""
+    stop_hn = socket.gethostname()  # hostname when beeflow core stop returned
     # Check workflow states; warn if there are active states, pause running workflows
     workflow_list = bee_client.get_wf_list()
     concern_states = {'Running', 'Initializing', 'Waiting'}
@@ -451,15 +514,13 @@ def stop(query='yes'):
             bee_client.pause(wf_id)
     resp = cli_connection.send(paths.beeflow_socket(), {'type': 'quit'})
     if resp is None:
-        beeflow_log = paths.log_fname('beeflow')
-        warn('Error: beeflow is not running on this system. It could be '
-             'running on a different front end.\n'
-             f'       Check the beeflow log: "{beeflow_log}".')
+        bee_client.check_hostname(stop_hn)
         sys.exit(1)
     # As long as it returned something, we should be good
     beeflow_log = paths.log_fname('beeflow')
     if query == "yes":
         print(f'Beeflow has stopped. Check the log at "{beeflow_log}".')
+        bee_client.setup_hostname("")
 
 
 def archive_dir(dir_to_archive):
@@ -496,6 +557,8 @@ def reset(archive: bool = typer.Option(False, '--archive', '-a',
                                        help='Archive bee_workdir  before removal')):
     """Stop all components and delete the bee_workdir directory."""
     # Check workflow states; warn if there are active states.
+    reset_hn = socket.gethostname()
+    bee_client.check_hostname(reset_hn)
     workflow_list = bee_client.get_wf_list()
     active_states = {'Running', 'Paused', 'Initializing', 'Waiting'}
     caution = ""
@@ -563,19 +626,26 @@ def pull_to_tar(ref, tarball):
     subprocess.check_call(['ch-convert', '-i', 'ch-image', '-o', 'tar', ref, tarball])
 
 
+def build_to_tar(tag, dockerfile, tarball):
+    """Build a container from a Dockerfile and convert to tarball."""
+    subprocess.check_call(['ch-image', 'build', '-t', tag, '-f', dockerfile, '.'])
+    subprocess.check_call(['ch-convert', '-i', 'ch-image', '-o', 'tar', tag, tarball])
+
+
 @app.command()
 def pull_deps(outdir: str = typer.Option('.', '--outdir', '-o',
                                          help='directory to store containers in')):
     """Pull required BEE containers and store in outdir."""
     load_check_charliecloud()
     neo4j_path = os.path.join(os.path.realpath(outdir), 'neo4j.tar.gz')
-    pull_to_tar('neo4j:5.17', neo4j_path)
+    neo4j_dockerfile = str(Path(REPO_PATH, "beeflow/data/dockerfiles/Dockerfile.neo4j"))
+    build_to_tar('neo4j_image', neo4j_dockerfile, neo4j_path)
     redis_path = os.path.join(os.path.realpath(outdir), 'redis.tar.gz')
     pull_to_tar('redis', redis_path)
-    print()
-    print('The BEE dependency containers have been successfully downloaded. '
-          'Please make sure to set the following options in your config:')
-    print()
-    print('[DEFAULT]')
-    print('neo4j_image =', neo4j_path)
-    print('redis_image =', redis_path)
+
+    AlterConfig(changes={'DEFAULT': {'neo4j_image': neo4j_path,
+                                     'redis_image': redis_path}}).save()
+
+    dep_dir = container_manager.get_dep_dir()
+    if os.path.isdir(dep_dir):
+        shutil.rmtree(dep_dir)