hpc-beeflow 0.1.11.dev1__tar.gz → 0.1.11.dev3__tar.gz

{hpc_beeflow-0.1.11.dev1 → hpc_beeflow-0.1.11.dev3}/PKG-INFO

@@ -1,8 +1,7 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.3
 Name: hpc-beeflow
-Version: 0.1.11.dev1
+Version: 0.1.11.dev3
 Summary: A software package for containerizing HPC applications and managing job workflows
-Home-page: https://github.com/lanl/BEE
 Keywords: bee,hpc,workflow,cluster,computing
 Author: BEE-LANL Dev Team
 Author-email: bee-dev@lanl.gov
@@ -19,7 +18,8 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Topic :: Scientific/Engineering
 Classifier: Topic :: System :: Clustering
@@ -43,7 +43,7 @@ Requires-Dist: jsonpickle (>=2.2.0,<3.0.0)
 Requires-Dist: neo4j (>=5,<6)
 Requires-Dist: networkx (==3.1)
 Requires-Dist: pre-commit (==3.5.0)
-Requires-Dist: pylint (==3.2.7)
+Requires-Dist: pylint (>=3.2.7,<4.0.0)
 Requires-Dist: python-daemon (>=2.3.1,<3.0.0)
 Requires-Dist: python-heatclient (>=3.1.0,<4.0.0) ; extra == "cloud-extras"
 Requires-Dist: python-openstackclient (>=6.0.0,<7.0.0) ; extra == "cloud-extras"
@@ -119,10 +119,12 @@ Contributors:
 * Tim Randles - `trandles-lanl <https://github.com/trandles-lanl>`_
 * Jacob Tronge - `jtronge <https://github.com/jtronge>`_
 * Kabir Vats - `kabir-vats <https://github.com/kabir-vats>`_
+* Sare Beste Zeytun - `sbzeytun <https://github.com/sbzeytun>`_
 
 Concept and Design Contributors
 
 * James Ahrens
+* Curt Canada 
 * Allen McPherson
 * Li-Ta Lo
 * Louis Vernon

{hpc_beeflow-0.1.11.dev1 → hpc_beeflow-0.1.11.dev3}/README.rst

@@ -63,10 +63,12 @@ Contributors:
 * Tim Randles - `trandles-lanl <https://github.com/trandles-lanl>`_
 * Jacob Tronge - `jtronge <https://github.com/jtronge>`_
 * Kabir Vats - `kabir-vats <https://github.com/kabir-vats>`_
+* Sare Beste Zeytun - `sbzeytun <https://github.com/sbzeytun>`_
 
 Concept and Design Contributors
 
 * James Ahrens
+* Curt Canada 
 * Allen McPherson
 * Li-Ta Lo
 * Louis Vernon

{hpc_beeflow-0.1.11.dev1 → hpc_beeflow-0.1.11.dev3}/beeflow/client/bee_client.py

@@ -231,7 +231,6 @@ def get_wf_list():
     if resp.status_code != requests.codes.okay:  # pylint: disable=no-member
         error_exit('WF Manager did not return workflow list')
 
-    logging.info('List Jobs:  {resp.text}')
     return jsonpickle.decode(resp.json()['workflow_list'])
 
 
@@ -275,10 +274,7 @@ def match_short_id(wf_id):
                          "workflow ID {matched_ids[0]}")
             long_wf_id = matched_ids[0]
             return long_wf_id
-    else:
-        sys.exit("There are currently no workflows.")
-
-    return None
+    sys.exit("There are currently no workflows.")
 
 
 def get_wf_status(wf_id):
@@ -324,6 +320,18 @@ def submit(wf_name: str = typer.Argument(..., help='the workflow name'),  # pyli
     wf_path = wf_path.resolve()
     workdir = workdir.resolve()
 
+    # Make sure the workdir is an absolute path and exists
+    workdir = os.path.expanduser(workdir)
+    workdir = os.path.abspath(workdir)
+    if not os.path.exists(workdir):
+        error_exit(f"Workflow working directory \"{workdir}\" doesn't exist")
+
+    # Make sure the workdir is not in /var or /var/tmp
+    if os.path.commonpath([os.path.realpath('/tmp'), workdir]) == os.path.realpath('/tmp'):
+        error_exit("Workflow working directory cannot be in \"/tmp\"")
+    if os.path.commonpath([os.path.realpath('/var/tmp'), workdir]) == os.path.realpath('/var/tmp'):
+        error_exit("Workflow working directory cannot be in \"/var/tmp\"")
+
     tarball_path = ""
     workflow = None
     wf_tarball = None
@@ -363,7 +371,7 @@ def submit(wf_name: str = typer.Argument(..., help='the workflow name'),  # pyli
         parser = CwlParser()
         workflow_id = generate_workflow_id()
         workflow, tasks = parser.parse_workflow(workflow_id, str(main_cwl_path),
-                                                job=str(yaml_path))
+                                                job=str(yaml_path), workdir=workdir)
         tasks = [jsonpickle.encode(task) for task in tasks]
 
         wf_tarball = open(package_path, 'rb')
@@ -373,18 +381,6 @@ def submit(wf_name: str = typer.Argument(..., help='the workflow name'),  # pyli
     else:
         error_exit(f'Workflow tarball {wf_path} cannot be found')
 
-    # Make sure the workdir is an absolute path and exists
-    workdir = os.path.expanduser(workdir)
-    workdir = os.path.abspath(workdir)
-    if not os.path.exists(workdir):
-        error_exit(f"Workflow working directory \"{workdir}\" doesn't exist")
-
-    # Make sure the workdir is not in /var or /var/tmp
-    if os.path.commonpath([os.path.realpath('/tmp'), workdir]) == os.path.realpath('/tmp'):
-        error_exit("Workflow working directory cannot be in \"/tmp\"")
-    if os.path.commonpath([os.path.realpath('/var/tmp'), workdir]) == os.path.realpath('/var/tmp'):
-        error_exit("Workflow working directory cannot be in \"/var/tmp\"")
-
     # TODO: Can all of this information be sent as a file?
     data = {
         'wf_name': wf_name.encode(),
@@ -558,8 +554,6 @@ def list_workflows():
     else:
         typer.echo("There are currently no workflows.")
 
-    logging.info('List workflows:  {resp.text}')
-
 
 @app.command()
 def query(wf_id: str = typer.Argument(..., callback=match_short_id)):
@@ -745,14 +739,21 @@ def dag(wf_id: str = typer.Argument(..., callback=match_short_id),
         typer.secho("Workflow has been archived. All new DAGs will look the same as the one "
                     "in the archive directory.",
                     fg=typer.colors.MAGENTA)
-        wf_utils.export_dag(wf_id, output_dir, graphmls_dir, no_dag_dir)
+        dot_avail = wf_utils.export_dag(wf_id, output_dir, graphmls_dir, no_dag_dir)
     else:
         wf_dir = wf_utils.get_workflow_dir(wf_id)
         graphmls_dir = wf_dir + '/graphmls'
         os.makedirs(graphmls_dir, exist_ok=True)
-        wf_utils.export_dag(wf_id, output_dir, graphmls_dir, no_dag_dir, wf_dir)
-    typer.secho(f"DAG for workflow {_short_id(wf_id)} has been exported successfully.",
-                fg=typer.colors.GREEN)
+        dot_avail = wf_utils.export_dag(wf_id, output_dir, graphmls_dir, no_dag_dir, wf_dir)
+    if dot_avail:
+        typer.secho(f"DAG for workflow {_short_id(wf_id)} has been exported successfully.",
+                    fg=typer.colors.GREEN)
+    else:
+        typer.secho(
+            f"GraphML for workflow {_short_id(wf_id)} has been exported successfully."
+            "(Graphviz not found, so no rendered DAG was produced.)",
+            fg=typer.colors.GREEN
+        )
 
 
 @app.callback(invoke_without_command=True)

{hpc_beeflow-0.1.11.dev1 → hpc_beeflow-0.1.11.dev3}/beeflow/client/core.py

@@ -344,6 +344,9 @@ class Beeflow:
     def loop(self):
         """Run the main loop."""
         print(f'Running on {socket.gethostname()}')
+        # Check for Graphviz:
+        if not shutil.which("dot"):
+            print('Graphviz was not found on PATH.')
         self.mgr.run(self.base_components)
         with cli_connection.server(paths.beeflow_socket()) as server:
             while not self.quit:

{hpc_beeflow-0.1.11.dev1 → hpc_beeflow-0.1.11.dev3}/beeflow/common/build/container_drivers.py

@@ -45,47 +45,53 @@ class CharliecloudBuildDriver(ContainerBuildDriver):
         :param task: the task to build for
         :type task: Task
         """
-        # Store build container archive pased on config file or relative to bee_workdir if not set.
-        container_archive = bc.get('builder', 'container_archive')
-        self.container_archive = bc.resolve_path(container_archive)
-        os.makedirs(self.container_archive, exist_ok=True)
-        # Deploy build tarballs relative to /var/tmp/username/beeflow by default
-        deployed_image_root = bc.get('builder', 'deployed_image_root')
-        # Make sure conf_file path exists
-        os.makedirs(deployed_image_root, exist_ok=True)
-        # Make sure path is absolute
-        deployed_image_root = bc.resolve_path(deployed_image_root)
-        self.deployed_image_root = deployed_image_root
-        os.makedirs(self.deployed_image_root, exist_ok=True)
-        # Set container-relative output directory based on BeeConfig, or use '/'
-        container_output_path = bc.get('builder', 'container_output_path')
-        self.container_output_path = container_output_path
-        # record that a Charliecloud builder was used
-        # bc.modify_section('user', 'builder', {'container_type': 'charliecloud'})
-        self.task = task
-        self.docker_image_id = None
-        self.container_name = None
-        docker_requirements = set()
-        try:
-            requirement_docker_requirements = self.task.requirements['DockerRequirement'].keys()
-            docker_requirements = docker_requirements.union(requirement_docker_requirements)
-            req_string = f'{set(requirement_docker_requirements)}'
-            log.info(f'task {self.task.id} requirement DockerRequirements: {req_string}')
-        except (TypeError, KeyError):
-            log.info(f'task {self.task.name} {self.task.id} no DockerRequirements in requirement')
-        try:
-            hint_docker_requirements = self.task.hints['DockerRequirement'].keys()
-            docker_requirements = docker_requirements.union(hint_docker_requirements)
-            hint_str = f'{set(hint_docker_requirements)}'
-            log.info(f'task {self.task.name} {self.task.id} hint DockerRequirements: {hint_str}')
-        except (TypeError, KeyError):
-            log.info(f'task {self.task.name} {self.task.id} hints has no DockerRequirements')
-        log.info(f'task {self.task.id} union DockerRequirements : {docker_requirements}')
-        exec_superset = self.resolve_priority()
-        self.exec_list = [i for i in exec_superset if i[1] in docker_requirements]
-        log_exec_list = [i[1] for i in self.exec_list]
-        log.info(f'task {self.task.id} DockerRequirement execution order will be: {log_exec_list}')
-        log.info('Execution order pre-empts hint/requirement status.')
+        has_container = task.get_full_requirement('DockerRequirement')
+        if has_container:  # only try to build if task has a container
+            # Store build container archive pased on config file or relative to bee_workdir
+            # if not set.
+            container_archive = bc.get('builder', 'container_archive')
+            self.container_archive = bc.resolve_path(container_archive)
+            os.makedirs(self.container_archive, exist_ok=True)
+            # Deploy build tarballs relative to /var/tmp/username/beeflow by default
+            deployed_image_root = bc.get('builder', 'deployed_image_root')
+            # Make sure conf_file path exists
+            os.makedirs(deployed_image_root, exist_ok=True)
+            # Make sure path is absolute
+            deployed_image_root = bc.resolve_path(deployed_image_root)
+            self.deployed_image_root = deployed_image_root
+            os.makedirs(self.deployed_image_root, exist_ok=True)
+            # Set container-relative output directory based on BeeConfig, or use '/'
+            container_output_path = bc.get('builder', 'container_output_path')
+            self.container_output_path = container_output_path
+            # record that a Charliecloud builder was used
+            # bc.modify_section('user', 'builder', {'container_type': 'charliecloud'})
+            self.task = task
+            self.docker_image_id = None
+            self.container_name = None
+            docker_requirements = set()
+            try:
+                req_docker_requirements = self.task.requirements['DockerRequirement'].keys()
+                docker_requirements = docker_requirements.union(req_docker_requirements)
+                req_string = f'{set(req_docker_requirements)}'
+                log.info(f'task {self.task.id} requirement DockerRequirements: {req_string}')
+            except (TypeError, KeyError):
+                log.info(f'task {self.task.name} {self.task.id} '
+                         'no DockerRequirements in requirement')
+            try:
+                hint_docker_requirements = self.task.hints['DockerRequirement'].keys()
+                docker_requirements = docker_requirements.union(hint_docker_requirements)
+                hint_str = f'{set(hint_docker_requirements)}'
+                log.info(f'task {self.task.name} {self.task.id} hint '
+                         f'DockerRequirements: {hint_str}')
+            except (TypeError, KeyError):
+                log.info(f'task {self.task.name} {self.task.id} hints has no DockerRequirements')
+            log.info(f'task {self.task.id} union DockerRequirements : {docker_requirements}')
+            exec_superset = self.resolve_priority()
+            self.exec_list = [i for i in exec_superset if i[1] in docker_requirements]
+            log_exec_list = [i[1] for i in self.exec_list]
+            log.info(f'task {self.task.id} DockerRequirement execution order '
+                     f'will be: {log_exec_list}')
+            log.info('Execution order pre-empts hint/requirement status.')
 
     def get_docker_req(self, docker_req_param):
         """Get dockerRequirement, prioritizing requirements over hints.

{hpc_beeflow-0.1.11.dev1 → hpc_beeflow-0.1.11.dev3}/beeflow/common/cwl/cwl.py

@@ -1,6 +1,7 @@
 """Create and manage CWL files."""
 from dataclasses import dataclass
 from io import StringIO
+from typing import Optional
 import ruamel.yaml
 
 # Create the global object for ruamel.ymal
@@ -399,6 +400,7 @@ class CheckpointRequirement:
     restart_parameters: str
     num_tries: int
     enabled: bool = True
+    add_parameters: Optional[str] = None
 
     def dump(self):
         """Dump beeflow requirement to a dictionary."""
@@ -409,6 +411,7 @@ class CheckpointRequirement:
         checkpoint_dump[req_name]['container_path'] = self.container_path
         checkpoint_dump[req_name]['file_regex'] = self.file_regex
         checkpoint_dump[req_name]['restart_parameters'] = self.restart_parameters
+        checkpoint_dump[req_name]['add_parameters'] = self.add_parameters
         checkpoint_dump[req_name]['num_tries'] = self.num_tries
         return checkpoint_dump
 
@@ -450,6 +453,27 @@ class ScriptRequirement:
         return stream.getvalue()
 
 
+@dataclass
+class TaskRequirement:
+    """Defines task requirement."""
+
+    workdir: str
+
+    def dump(self):
+        """Dump task requirement to a dictionary."""
+        key = 'beeflow:TaskRequirement'
+        this_dump = {key: {}}
+        if self.workdir:
+            this_dump[key]['workdir'] = self.workdir
+        return this_dump
+
+    def __repr__(self):
+        """Return TaskRequirement as a yaml string."""
+        stream = StringIO()
+        yaml.dump(self.dump(), stream)
+        return stream.getvalue()
+
+
 @dataclass
 class Hints:
     """Holds all the hints for a CWL workflow."""

{hpc_beeflow-0.1.11.dev1 → hpc_beeflow-0.1.11.dev3}/beeflow/common/cwl/workflow.py

@@ -6,7 +6,7 @@ from beeflow.common.cwl.cwl import (CWL, CWLInput, CWLInputs, RunInput, Inputs,
                                     Outputs, Run, RunOutput, Step, Steps, Hints,
                                     InputBinding, MPIRequirement, DockerRequirement,
                                     ScriptRequirement, SlurmRequirement,
-                                    CheckpointRequirement)
+                                    CheckpointRequirement, TaskRequirement)
 
 
 @dataclass
@@ -133,10 +133,20 @@ class Checkpoint(CheckpointRequirement):
                                      container_path=self.container_path,
                                      file_regex=self.file_regex,
                                      restart_parameters=self.restart_parameters,
+                                     add_parameters=self.add_parameters,
                                      num_tries=self.num_tries,
                                      enabled=self.enabled)
 
 
+@dataclass
+class TaskReq(TaskRequirement):
+    """Get Task Requirements."""
+
+    def requirement(self):
+        """Return a task requirement object."""
+        return TaskRequirement(workdir=self.workdir)
+
+
 @dataclass
 class Task:
     """Represents a task."""

{hpc_beeflow-0.1.11.dev1 → hpc_beeflow-0.1.11.dev3}/beeflow/common/deps/neo4j_manager.py

@@ -5,8 +5,13 @@ import shutil
 import re
 import subprocess
 import time
+import logging
+
+from beeflow.common.gdb import neo4j_driver
+from beeflow.common.gdb.neo4j_driver import (DEFAULT_BOLT_PORT)
+
+from beeflow.common.db.bdb import connect_db
 
-from beeflow.wf_manager.resources import wf_utils
 from beeflow.common.db import wfm_db
 
 from beeflow.common import paths
@@ -15,6 +20,7 @@ from beeflow.common.deps import container_manager
 from beeflow.common.config_driver import BeeConfig as bc
 from beeflow.common import log as bee_logging
 
+
 BEE_WORKDIR = None
 MOUNT_DIR = None
 DATA_DIR = None
@@ -23,9 +29,9 @@ RUN_DIR = None
 CERTS_DIR = None
 CONFS_DIR = None
 GRAPHMLS_DIR = None
-CONTAINER_PATH = Nonelog = bee_logging.setup('neo4j')
-log = bee_logging.setup('neo4j')
+CONTAINER_PATH = None
 
+log = None  # pylint: disable=C0103
 
 def get_open_port():
     """Return an open ephemeral port."""
@@ -44,7 +50,7 @@ def setup_ports():
     http_port = get_open_port()
     https_port = get_open_port()
 
-    db = wf_utils.connect_db(wfm_db, wf_utils.get_db_path())
+    db = connect_db(wfm_db, bc.get('DEFAULT', 'bee_workdir') + "/wfm.db")
 
     db.info.set_port('bolt', bolt_port)
     db.info.set_port('http', http_port)
@@ -141,7 +147,7 @@ def create_credentials():
         log.error("neo4j-admin set-initial-password failed")
 
 
-def create_database():
+def create_database(bolt_port):
     """Create the neo4j database and return the process."""
     try:
         command = ['neo4j', 'console']
@@ -157,7 +163,7 @@ def create_database():
             "-W", "-b", GRAPHMLS_DIR + ":/var/lib/neo4j/import",
             CONTAINER_PATH, "--", *command
         ], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        wait_gdb()
+        wait_gdb(bolt_port=bolt_port)
         return proc
 
     except FileNotFoundError:
@@ -167,6 +173,12 @@ def create_database():
 
 def start():
     """Start the graph database."""
+
+    global log
+    log = bee_logging.setup("neo4j")
+    log.setLevel(logging.INFO)
+    log.propagate = False
+
     log.info('Starting Neo4j Database')
 
     bolt_port, http_port, https_port = setup_ports()
@@ -179,17 +191,23 @@ def start():
 
     create_credentials()
 
-    return create_database()
+    return create_database(bolt_port=bolt_port)
 
 
-def wait_gdb():
-    """Need to wait for the GDB. Currently, we're using the sleep time paramater.
-
-    We'd like to remove that in the future.
+def wait_gdb(bolt_port=DEFAULT_BOLT_PORT):
+    """
+    Block until the Neo4j instance answers a Bolt handshake.
     """
-    gdb_sleep_time = bc.get('graphdb', 'sleep_time')
-    print(f'waiting {gdb_sleep_time}s for GDB to come up')
-    time.sleep(gdb_sleep_time)
+    connect_neo4j_driver(bolt_port)
+
+
+def connect_neo4j_driver(bolt_port):
+    """Create a neo4j driver to a gdb through bolt port."""
+    driver = neo4j_driver.Neo4jDriver()
+    driver.connect(user="neo4j", bolt_port=bolt_port,
+                   db_hostname=bc.get("graphdb", "hostname"),
+                   password=bc.get("graphdb", "dbpass"))
+    driver.create_bee_node()
 
 
 def remove_gdb():

{hpc_beeflow-0.1.11.dev1 → hpc_beeflow-0.1.11.dev3}/beeflow/common/gdb/graphml_key_updater.py

@@ -31,7 +31,18 @@ default_key_definitions = {
 }
 
 
-def update_graphml(wf_id, graphmls_dir):
+def backup_graphml(path, graphmls_dir, short_id):
+    """Backup graphmls."""
+    if os.path.exists(path):
+        i = 1
+        backup_path = f'{graphmls_dir}/{short_id}_v{i}.graphml'
+        while os.path.exists(backup_path):
+            i += 1
+            backup_path = f'{graphmls_dir}/{short_id}_v{i}.graphml'
+        shutil.copy(path, backup_path)
+
+
+def update_graphml(wf_id, graphmls_dir, extra_output_dir=None, no_dag_dir=False):
     """Update GraphML file by ensuring required keys are present and updating its structure."""
     short_id = wf_id[:6]
     # Define paths
@@ -41,13 +52,7 @@ def update_graphml(wf_id, graphmls_dir):
     gdb_graphml_path = gdb_graphmls_dir + "/" + short_id + ".graphml"
     output_graphml_path = graphmls_dir + "/" + short_id + ".graphml"
     # Handle making multiple versions of the graphmls without overriding old ones
-    if os.path.exists(output_graphml_path):
-        i = 1
-        backup_path = f'{graphmls_dir}/{short_id}_v{i}.graphml'
-        while os.path.exists(backup_path):
-            i += 1
-            backup_path = f'{graphmls_dir}/{short_id}_v{i}.graphml'
-        shutil.copy(output_graphml_path, backup_path)
+    backup_graphml(output_graphml_path, graphmls_dir, short_id)
     # Parse the GraphML file and preserve namespaces
     tree = ET.parse(gdb_graphml_path)
     root = tree.getroot()
@@ -69,3 +74,13 @@ def update_graphml(wf_id, graphmls_dir):
 
     # Save the updated GraphML file
     tree.write(output_graphml_path, encoding='UTF-8', xml_declaration=True)
+
+    # Saving the GraphML file to an extra user specified location
+    if extra_output_dir:
+        if not no_dag_dir:
+            extra_output_dir += "/" + short_id + "-graphmls"
+        extra_graphml_path = extra_output_dir + "/" + short_id + ".graphml"
+        os.makedirs(extra_output_dir, exist_ok=True)
+        if extra_graphml_path != output_graphml_path:
+            backup_graphml(extra_graphml_path, extra_output_dir, short_id)
+            tree.write(extra_graphml_path, encoding='UTF-8', xml_declaration=True)

{hpc_beeflow-0.1.11.dev1 → hpc_beeflow-0.1.11.dev3}/beeflow/common/gdb/neo4j_driver.py

@@ -10,6 +10,7 @@ either standardized or read from a config file.
 # Disable E1129: External module is missing proper resource context manager methods.
 # pylint:disable=E1129
 
+import time
 from neo4j import GraphDatabase as Neo4jDatabase
 from neo4j.exceptions import ServiceUnavailable
 
@@ -27,6 +28,8 @@ DEFAULT_HOSTNAME = "localhost"
 DEFAULT_BOLT_PORT = "7687"
 DEFAULT_USER = "neo4j"
 DEFAULT_PASSWORD = "password"
+MAX_WAIT_SECS = 60
+RETRY_DELAY_SECS = 5
 
 
 class Neo4jNotRunning(Exception):
@@ -61,14 +64,19 @@ class Neo4jDriver(GraphDatabaseDriver):
         bolt_port = kwargs.get("bolt_port", DEFAULT_BOLT_PORT)
         password = kwargs.get("db_pass", DEFAULT_PASSWORD)
         uri = f"bolt://{db_hostname}:{bolt_port}"
-        try:
-            # Connect to the Neo4j database using the Neo4j proprietary driver
-            self._driver = Neo4jDatabase.driver(uri, auth=(user, password)) # pylint: disable=W0201
-            # Checks the connection and returns ServiceUnavailable if something is wrong
-            self._driver.verify_connectivity()
-        except ServiceUnavailable as sue:
-            log.error("Neo4j database is unavailable")
-            raise Neo4jNotRunning("Neo4j database is unavailable") from sue
+        start_time = time.monotonic()
+        self._driver = Neo4jDatabase.driver(uri, auth=(user, password)) # pylint: disable=W0201
+        while True:
+            try:
+                self._driver.verify_connectivity()
+                return
+            except ServiceUnavailable as sue:
+                elapsed = time.monotonic() - start_time
+                if elapsed >= MAX_WAIT_SECS:
+                    log.error("Neo4j database is unavailable")
+                    raise Neo4jNotRunning("Neo4j database is unavailable") from sue
+            log.info(f"Waiting {RETRY_DELAY_SECS} seconds for Neo4j to become ready...")
+            time.sleep(RETRY_DELAY_SECS)
 
     def create_bee_node(self):
         """Create the "BEE" node for all workflows to connect to."""