hpc-beeflow 0.1.11.dev2__tar.gz → 0.1.11.dev3__tar.gz

{hpc_beeflow-0.1.11.dev2 → hpc_beeflow-0.1.11.dev3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: hpc-beeflow
-Version: 0.1.11.dev2
+Version: 0.1.11.dev3
 Summary: A software package for containerizing HPC applications and managing job workflows
 Keywords: bee,hpc,workflow,cluster,computing
 Author: BEE-LANL Dev Team

{hpc_beeflow-0.1.11.dev2 → hpc_beeflow-0.1.11.dev3}/beeflow/client/bee_client.py

@@ -739,14 +739,21 @@ def dag(wf_id: str = typer.Argument(..., callback=match_short_id),
         typer.secho("Workflow has been archived. All new DAGs will look the same as the one "
                     "in the archive directory.",
                     fg=typer.colors.MAGENTA)
-        wf_utils.export_dag(wf_id, output_dir, graphmls_dir, no_dag_dir)
+        dot_avail = wf_utils.export_dag(wf_id, output_dir, graphmls_dir, no_dag_dir)
     else:
         wf_dir = wf_utils.get_workflow_dir(wf_id)
         graphmls_dir = wf_dir + '/graphmls'
         os.makedirs(graphmls_dir, exist_ok=True)
-        wf_utils.export_dag(wf_id, output_dir, graphmls_dir, no_dag_dir, wf_dir)
-    typer.secho(f"DAG for workflow {_short_id(wf_id)} has been exported successfully.",
-                fg=typer.colors.GREEN)
+        dot_avail = wf_utils.export_dag(wf_id, output_dir, graphmls_dir, no_dag_dir, wf_dir)
+    if dot_avail:
+        typer.secho(f"DAG for workflow {_short_id(wf_id)} has been exported successfully.",
+                    fg=typer.colors.GREEN)
+    else:
+        typer.secho(
+            f"GraphML for workflow {_short_id(wf_id)} has been exported successfully."
+            "(Graphviz not found, so no rendered DAG was produced.)",
+            fg=typer.colors.GREEN
+        )
 
 
 @app.callback(invoke_without_command=True)

{hpc_beeflow-0.1.11.dev2 → hpc_beeflow-0.1.11.dev3}/beeflow/client/core.py

@@ -344,6 +344,9 @@ class Beeflow:
     def loop(self):
         """Run the main loop."""
         print(f'Running on {socket.gethostname()}')
+        # Check for Graphviz:
+        if not shutil.which("dot"):
+            print('Graphviz was not found on PATH.')
         self.mgr.run(self.base_components)
         with cli_connection.server(paths.beeflow_socket()) as server:
             while not self.quit:

{hpc_beeflow-0.1.11.dev2 → hpc_beeflow-0.1.11.dev3}/beeflow/common/build/container_drivers.py

@@ -45,47 +45,53 @@ class CharliecloudBuildDriver(ContainerBuildDriver):
         :param task: the task to build for
         :type task: Task
         """
-        # Store build container archive pased on config file or relative to bee_workdir if not set.
-        container_archive = bc.get('builder', 'container_archive')
-        self.container_archive = bc.resolve_path(container_archive)
-        os.makedirs(self.container_archive, exist_ok=True)
-        # Deploy build tarballs relative to /var/tmp/username/beeflow by default
-        deployed_image_root = bc.get('builder', 'deployed_image_root')
-        # Make sure conf_file path exists
-        os.makedirs(deployed_image_root, exist_ok=True)
-        # Make sure path is absolute
-        deployed_image_root = bc.resolve_path(deployed_image_root)
-        self.deployed_image_root = deployed_image_root
-        os.makedirs(self.deployed_image_root, exist_ok=True)
-        # Set container-relative output directory based on BeeConfig, or use '/'
-        container_output_path = bc.get('builder', 'container_output_path')
-        self.container_output_path = container_output_path
-        # record that a Charliecloud builder was used
-        # bc.modify_section('user', 'builder', {'container_type': 'charliecloud'})
-        self.task = task
-        self.docker_image_id = None
-        self.container_name = None
-        docker_requirements = set()
-        try:
-            requirement_docker_requirements = self.task.requirements['DockerRequirement'].keys()
-            docker_requirements = docker_requirements.union(requirement_docker_requirements)
-            req_string = f'{set(requirement_docker_requirements)}'
-            log.info(f'task {self.task.id} requirement DockerRequirements: {req_string}')
-        except (TypeError, KeyError):
-            log.info(f'task {self.task.name} {self.task.id} no DockerRequirements in requirement')
-        try:
-            hint_docker_requirements = self.task.hints['DockerRequirement'].keys()
-            docker_requirements = docker_requirements.union(hint_docker_requirements)
-            hint_str = f'{set(hint_docker_requirements)}'
-            log.info(f'task {self.task.name} {self.task.id} hint DockerRequirements: {hint_str}')
-        except (TypeError, KeyError):
-            log.info(f'task {self.task.name} {self.task.id} hints has no DockerRequirements')
-        log.info(f'task {self.task.id} union DockerRequirements : {docker_requirements}')
-        exec_superset = self.resolve_priority()
-        self.exec_list = [i for i in exec_superset if i[1] in docker_requirements]
-        log_exec_list = [i[1] for i in self.exec_list]
-        log.info(f'task {self.task.id} DockerRequirement execution order will be: {log_exec_list}')
-        log.info('Execution order pre-empts hint/requirement status.')
+        has_container = task.get_full_requirement('DockerRequirement')
+        if has_container:  # only try to build if task has a container
+            # Store build container archive pased on config file or relative to bee_workdir
+            # if not set.
+            container_archive = bc.get('builder', 'container_archive')
+            self.container_archive = bc.resolve_path(container_archive)
+            os.makedirs(self.container_archive, exist_ok=True)
+            # Deploy build tarballs relative to /var/tmp/username/beeflow by default
+            deployed_image_root = bc.get('builder', 'deployed_image_root')
+            # Make sure conf_file path exists
+            os.makedirs(deployed_image_root, exist_ok=True)
+            # Make sure path is absolute
+            deployed_image_root = bc.resolve_path(deployed_image_root)
+            self.deployed_image_root = deployed_image_root
+            os.makedirs(self.deployed_image_root, exist_ok=True)
+            # Set container-relative output directory based on BeeConfig, or use '/'
+            container_output_path = bc.get('builder', 'container_output_path')
+            self.container_output_path = container_output_path
+            # record that a Charliecloud builder was used
+            # bc.modify_section('user', 'builder', {'container_type': 'charliecloud'})
+            self.task = task
+            self.docker_image_id = None
+            self.container_name = None
+            docker_requirements = set()
+            try:
+                req_docker_requirements = self.task.requirements['DockerRequirement'].keys()
+                docker_requirements = docker_requirements.union(req_docker_requirements)
+                req_string = f'{set(req_docker_requirements)}'
+                log.info(f'task {self.task.id} requirement DockerRequirements: {req_string}')
+            except (TypeError, KeyError):
+                log.info(f'task {self.task.name} {self.task.id} '
+                         'no DockerRequirements in requirement')
+            try:
+                hint_docker_requirements = self.task.hints['DockerRequirement'].keys()
+                docker_requirements = docker_requirements.union(hint_docker_requirements)
+                hint_str = f'{set(hint_docker_requirements)}'
+                log.info(f'task {self.task.name} {self.task.id} hint '
+                         f'DockerRequirements: {hint_str}')
+            except (TypeError, KeyError):
+                log.info(f'task {self.task.name} {self.task.id} hints has no DockerRequirements')
+            log.info(f'task {self.task.id} union DockerRequirements : {docker_requirements}')
+            exec_superset = self.resolve_priority()
+            self.exec_list = [i for i in exec_superset if i[1] in docker_requirements]
+            log_exec_list = [i[1] for i in self.exec_list]
+            log.info(f'task {self.task.id} DockerRequirement execution order '
+                     f'will be: {log_exec_list}')
+            log.info('Execution order pre-empts hint/requirement status.')
 
     def get_docker_req(self, docker_req_param):
         """Get dockerRequirement, prioritizing requirements over hints.

{hpc_beeflow-0.1.11.dev2 → hpc_beeflow-0.1.11.dev3}/beeflow/common/deps/neo4j_manager.py

@@ -5,8 +5,13 @@ import shutil
 import re
 import subprocess
 import time
+import logging
+
+from beeflow.common.gdb import neo4j_driver
+from beeflow.common.gdb.neo4j_driver import (DEFAULT_BOLT_PORT)
+
+from beeflow.common.db.bdb import connect_db
 
-from beeflow.wf_manager.resources import wf_utils
 from beeflow.common.db import wfm_db
 
 from beeflow.common import paths
@@ -15,6 +20,7 @@ from beeflow.common.deps import container_manager
 from beeflow.common.config_driver import BeeConfig as bc
 from beeflow.common import log as bee_logging
 
+
 BEE_WORKDIR = None
 MOUNT_DIR = None
 DATA_DIR = None
@@ -23,9 +29,9 @@ RUN_DIR = None
 CERTS_DIR = None
 CONFS_DIR = None
 GRAPHMLS_DIR = None
-CONTAINER_PATH = Nonelog = bee_logging.setup('neo4j')
-log = bee_logging.setup('neo4j')
+CONTAINER_PATH = None
 
+log = None  # pylint: disable=C0103
 
 def get_open_port():
     """Return an open ephemeral port."""
@@ -44,7 +50,7 @@ def setup_ports():
     http_port = get_open_port()
     https_port = get_open_port()
 
-    db = wf_utils.connect_db(wfm_db, wf_utils.get_db_path())
+    db = connect_db(wfm_db, bc.get('DEFAULT', 'bee_workdir') + "/wfm.db")
 
     db.info.set_port('bolt', bolt_port)
     db.info.set_port('http', http_port)
@@ -141,7 +147,7 @@ def create_credentials():
         log.error("neo4j-admin set-initial-password failed")
 
 
-def create_database():
+def create_database(bolt_port):
     """Create the neo4j database and return the process."""
     try:
         command = ['neo4j', 'console']
@@ -157,7 +163,7 @@ def create_database():
             "-W", "-b", GRAPHMLS_DIR + ":/var/lib/neo4j/import",
             CONTAINER_PATH, "--", *command
         ], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        wait_gdb()
+        wait_gdb(bolt_port=bolt_port)
         return proc
 
     except FileNotFoundError:
@@ -167,6 +173,12 @@ def create_database():
 
 def start():
     """Start the graph database."""
+
+    global log
+    log = bee_logging.setup("neo4j")
+    log.setLevel(logging.INFO)
+    log.propagate = False
+
     log.info('Starting Neo4j Database')
 
     bolt_port, http_port, https_port = setup_ports()
@@ -179,17 +191,23 @@ def start():
 
     create_credentials()
 
-    return create_database()
+    return create_database(bolt_port=bolt_port)
 
 
-def wait_gdb():
-    """Need to wait for the GDB. Currently, we're using the sleep time paramater.
-
-    We'd like to remove that in the future.
+def wait_gdb(bolt_port=DEFAULT_BOLT_PORT):
+    """
+    Block until the Neo4j instance answers a Bolt handshake.
     """
-    gdb_sleep_time = bc.get('graphdb', 'sleep_time')
-    print(f'waiting {gdb_sleep_time}s for GDB to come up')
-    time.sleep(gdb_sleep_time)
+    connect_neo4j_driver(bolt_port)
+
+
+def connect_neo4j_driver(bolt_port):
+    """Create a neo4j driver to a gdb through bolt port."""
+    driver = neo4j_driver.Neo4jDriver()
+    driver.connect(user="neo4j", bolt_port=bolt_port,
+                   db_hostname=bc.get("graphdb", "hostname"),
+                   password=bc.get("graphdb", "dbpass"))
+    driver.create_bee_node()
 
 
 def remove_gdb():

{hpc_beeflow-0.1.11.dev2 → hpc_beeflow-0.1.11.dev3}/beeflow/common/gdb/graphml_key_updater.py

@@ -31,7 +31,18 @@ default_key_definitions = {
 }
 
 
-def update_graphml(wf_id, graphmls_dir):
+def backup_graphml(path, graphmls_dir, short_id):
+    """Backup graphmls."""
+    if os.path.exists(path):
+        i = 1
+        backup_path = f'{graphmls_dir}/{short_id}_v{i}.graphml'
+        while os.path.exists(backup_path):
+            i += 1
+            backup_path = f'{graphmls_dir}/{short_id}_v{i}.graphml'
+        shutil.copy(path, backup_path)
+
+
+def update_graphml(wf_id, graphmls_dir, extra_output_dir=None, no_dag_dir=False):
     """Update GraphML file by ensuring required keys are present and updating its structure."""
     short_id = wf_id[:6]
     # Define paths
@@ -41,13 +52,7 @@ def update_graphml(wf_id, graphmls_dir):
     gdb_graphml_path = gdb_graphmls_dir + "/" + short_id + ".graphml"
     output_graphml_path = graphmls_dir + "/" + short_id + ".graphml"
     # Handle making multiple versions of the graphmls without overriding old ones
-    if os.path.exists(output_graphml_path):
-        i = 1
-        backup_path = f'{graphmls_dir}/{short_id}_v{i}.graphml'
-        while os.path.exists(backup_path):
-            i += 1
-            backup_path = f'{graphmls_dir}/{short_id}_v{i}.graphml'
-        shutil.copy(output_graphml_path, backup_path)
+    backup_graphml(output_graphml_path, graphmls_dir, short_id)
     # Parse the GraphML file and preserve namespaces
     tree = ET.parse(gdb_graphml_path)
     root = tree.getroot()
@@ -69,3 +74,13 @@ def update_graphml(wf_id, graphmls_dir):
 
     # Save the updated GraphML file
     tree.write(output_graphml_path, encoding='UTF-8', xml_declaration=True)
+
+    # Saving the GraphML file to an extra user specified location
+    if extra_output_dir:
+        if not no_dag_dir:
+            extra_output_dir += "/" + short_id + "-graphmls"
+        extra_graphml_path = extra_output_dir + "/" + short_id + ".graphml"
+        os.makedirs(extra_output_dir, exist_ok=True)
+        if extra_graphml_path != output_graphml_path:
+            backup_graphml(extra_graphml_path, extra_output_dir, short_id)
+            tree.write(extra_graphml_path, encoding='UTF-8', xml_declaration=True)

{hpc_beeflow-0.1.11.dev2 → hpc_beeflow-0.1.11.dev3}/beeflow/common/gdb/neo4j_driver.py

@@ -10,6 +10,7 @@ either standardized or read from a config file.
 # Disable E1129: External module is missing proper resource context manager methods.
 # pylint:disable=E1129
 
+import time
 from neo4j import GraphDatabase as Neo4jDatabase
 from neo4j.exceptions import ServiceUnavailable
 
@@ -27,6 +28,8 @@ DEFAULT_HOSTNAME = "localhost"
 DEFAULT_BOLT_PORT = "7687"
 DEFAULT_USER = "neo4j"
 DEFAULT_PASSWORD = "password"
+MAX_WAIT_SECS = 60
+RETRY_DELAY_SECS = 5
 
 
 class Neo4jNotRunning(Exception):
@@ -61,14 +64,19 @@ class Neo4jDriver(GraphDatabaseDriver):
         bolt_port = kwargs.get("bolt_port", DEFAULT_BOLT_PORT)
         password = kwargs.get("db_pass", DEFAULT_PASSWORD)
         uri = f"bolt://{db_hostname}:{bolt_port}"
-        try:
-            # Connect to the Neo4j database using the Neo4j proprietary driver
-            self._driver = Neo4jDatabase.driver(uri, auth=(user, password)) # pylint: disable=W0201
-            # Checks the connection and returns ServiceUnavailable if something is wrong
-            self._driver.verify_connectivity()
-        except ServiceUnavailable as sue:
-            log.error("Neo4j database is unavailable")
-            raise Neo4jNotRunning("Neo4j database is unavailable") from sue
+        start_time = time.monotonic()
+        self._driver = Neo4jDatabase.driver(uri, auth=(user, password)) # pylint: disable=W0201
+        while True:
+            try:
+                self._driver.verify_connectivity()
+                return
+            except ServiceUnavailable as sue:
+                elapsed = time.monotonic() - start_time
+                if elapsed >= MAX_WAIT_SECS:
+                    log.error("Neo4j database is unavailable")
+                    raise Neo4jNotRunning("Neo4j database is unavailable") from sue
+            log.info(f"Waiting {RETRY_DELAY_SECS} seconds for Neo4j to become ready...")
+            time.sleep(RETRY_DELAY_SECS)
 
     def create_bee_node(self):
         """Create the "BEE" node for all workflows to connect to."""

{hpc_beeflow-0.1.11.dev2 → hpc_beeflow-0.1.11.dev3}/beeflow/common/parser/parser.py

@@ -281,7 +281,7 @@ class CwlParser:
                 raise CwlParseError(f"specified step output {out} not produced by CommandLineTool")
 
             output_type = out_map[out].type
-            glob = None
+            glob = ""
             if output_type == "stdout":
                 if not stdout:
                     raise CwlParseError(f"stdout capture required for step output {out} "

{hpc_beeflow-0.1.11.dev2 → hpc_beeflow-0.1.11.dev3}/beeflow/task_manager/background.py

@@ -40,9 +40,11 @@ def resolve_environment(task):
 def submit_task(db, worker, task):
     """Submit (or resubmit) a task."""
     try:
-        log.info(f'Resolving environment for task {task.name}')
-        resolve_environment(task)
-        log.info(f'Environment preparation complete for task {task.name}')
+        has_container = task.get_full_requirement('DockerRequirement')
+        if has_container:
+            log.info(f'Resolving environment for task {task.name}')
+            resolve_environment(task)
+            log.info(f'Environment preparation complete for task {task.name}')
         job_id, job_state = worker.submit_task(task)
         log.info(f"Job Submitted '{task.name}' job_id: {job_id} job_state: {job_state}")
         # place job in queue to monitor

hpc_beeflow-0.1.11.dev3/beeflow/tests/test_wf_utils.py

@@ -0,0 +1,61 @@
+"""Tests for wf_utils module."""
+
+import pytest
+from beeflow.wf_manager.resources import wf_utils
+
+from beeflow.tests.mocks import MockWFI
+
+
+@pytest.mark.parametrize(
+    "which_return, no_dag_flag, expected_dot_avail",
+    [
+        ("/usr/bin/dot", True, True),
+        ("/usr/bin/dot", False, True),
+        (None, True, False),
+        (None, False, False),
+    ],
+)
+def test_export_dag(mocker, which_return, no_dag_flag, expected_dot_avail):
+    """Regression test export_dag based on graphviz availability."""
+    wf_id = "abc123"
+    output_dir = "/path/to/out"
+    graphmls_dir = "/path/to/gmls"
+    workflow_dir = "/path/to/wfdir"
+    
+    mock_wfi = mocker.MagicMock()
+    mocker.patch('beeflow.wf_manager.resources.wf_utils.get_workflow_interface', return_value=mock_wfi)
+    mocker.patch('beeflow.common.wf_interface.WorkflowInterface.export_graphml')
+    mocker.patch('beeflow.wf_manager.resources.wf_utils.shutil.which',
+                 return_value=which_return)
+    mock_update = mocker.patch('beeflow.wf_manager.resources.wf_utils.update_graphml')
+    mock_viz = mocker.patch('beeflow.wf_manager.resources.wf_utils.generate_viz')
+
+    result =  wf_utils.export_dag(
+        wf_id,
+        output_dir,
+        graphmls_dir,
+        no_dag_flag,
+        workflow_dir
+    )
+
+    # always export the GraphML first
+    mock_wfi.export_graphml.assert_called_once_with()
+
+    # return value is bool(shutil.which)
+    assert result is expected_dot_avail
+
+    if expected_dot_avail:
+        mock_update.assert_called_once_with(wf_id, graphmls_dir)
+        mock_viz.assert_called_once_with(
+            wf_id, output_dir, graphmls_dir, no_dag_flag, workflow_dir
+        )
+    else:
+        mock_update.assert_called_once_with(
+            wf_id, graphmls_dir, output_dir, no_dag_flag
+        )
+        mock_viz.assert_not_called()
+
+
+
+
+

{hpc_beeflow-0.1.11.dev2 → hpc_beeflow-0.1.11.dev3}/beeflow/wf_manager/resources/wf_utils.py

@@ -17,6 +17,7 @@ from beeflow.common.connection import Connection
 from beeflow.common import paths
 from beeflow.common.db import wfm_db
 from beeflow.common.db.bdb import connect_db
+from beeflow.common.deps.neo4j_manager import connect_neo4j_driver
 
 
 log = bee_logging.setup(__name__)
@@ -261,15 +262,6 @@ def schedule_submit_tasks(wf_id, tasks):
     submit_tasks_tm(wf_id, tasks, allocation)
 
 
-def connect_neo4j_driver(bolt_port):
-    """Create a neo4j driver to a gdb through bolt port."""
-    driver = neo4j_driver.Neo4jDriver()
-    driver.connect(user="neo4j", bolt_port=bolt_port,
-                   db_hostname=bc.get("graphdb", "hostname"),
-                   password=bc.get("graphdb", "dbpass"))
-    driver.create_bee_node()
-
-
 def setup_workflow(wf_id, wf_name, wf_dir, wf_workdir, no_start, workflow=None, # pylint: disable=W0613
                    tasks=None):
     """Initialize Workflow in Separate Process."""
@@ -303,8 +295,13 @@ def export_dag(wf_id, output_dir, graphmls_dir, no_dag_dir, workflow_dir=None):
     """Export the DAG of the workflow."""
     wfi = get_workflow_interface(wf_id)
     wfi.export_graphml()
-    update_graphml(wf_id, graphmls_dir)
-    generate_viz(wf_id, output_dir, graphmls_dir, no_dag_dir, workflow_dir)
+    dot_avail = bool(shutil.which("dot"))
+    if dot_avail:
+        update_graphml(wf_id, graphmls_dir)
+        generate_viz(wf_id, output_dir, graphmls_dir, no_dag_dir, workflow_dir)
+    else:
+        update_graphml(wf_id, graphmls_dir, output_dir, no_dag_dir)
+    return dot_avail
 
 
 def start_workflow(wf_id):

{hpc_beeflow-0.1.11.dev2 → hpc_beeflow-0.1.11.dev3}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "hpc-beeflow"
-version = "0.1.11dev2"
+version = "0.1.11dev3"
 description = "A software package for containerizing HPC applications and managing job workflows"