hpc-beeflow 0.1.9.dev3__tar.gz → 0.1.9.dev4__tar.gz

{hpc_beeflow-0.1.9.dev3 → hpc_beeflow-0.1.9.dev4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hpc-beeflow
-Version: 0.1.9.dev3
+Version: 0.1.9.dev4
 Summary: A software package for containerizing HPC applications and managing job workflows
 Home-page: https://github.com/lanl/BEE
 Keywords: bee,hpc,workflow,cluster,computing

{hpc_beeflow-0.1.9.dev3 → hpc_beeflow-0.1.9.dev4}/beeflow/client/core.py

@@ -27,9 +27,9 @@ from beeflow.common import cli_connection
 from beeflow.common import paths
 from beeflow.wf_manager.resources import wf_utils
 
-from beeflow.common.db import wfm_db
-from beeflow.common.db.bdb import connect_db
-from beeflow.wf_manager.common import dep_manager
+from beeflow.common.deps import container_manager
+from beeflow.common.deps import neo4j_manager
+from beeflow.common.deps import redis_manager
 
 
 class ComponentManager:
@@ -194,49 +194,34 @@ def init_components():
         """Start the celery task queue."""
         log = open_log('celery')
         # Setting --pool=solo to avoid preforking multiple processes
-        return subprocess.Popen(['celery', '-A', 'beeflow.common.celery', 'worker', '--pool=solo'],
-                                stdout=log, stderr=log)
+        return subprocess.Popen(['celery', '-A', 'beeflow.common.deps.celery_manager',
+                                 'worker', '--pool=solo'], stdout=log, stderr=log)
 
     # Run this before daemonizing in order to avoid slow background start
-    container_path = paths.redis_container()
+    # container_path = paths.redis_container()
     # If it exists, we assume that it actually has a valid container
-    if not os.path.exists(container_path):
+    # if not os.path.exists(container_path):
+        # print('Unpacking Redis image...')
+        # subprocess.check_call(['ch-convert', '-i', 'tar', '-o', 'dir',
+        #                       bc.get('DEFAULT', 'redis_image'), container_path])
+    if not container_manager.check_container_dir('redis'):
         print('Unpacking Redis image...')
-        subprocess.check_call(['ch-convert', '-i', 'tar', '-o', 'dir',
-                               bc.get('DEFAULT', 'redis_image'), container_path])
+        container_manager.create_image('redis')
+
+    if not container_manager.check_container_dir('neo4j'):
+        print('Unpacking Neo4j image...')
+        container_manager.create_image('neo4j')
+
+    @mgr.component('neo4j-database', ('wf_manager',))
+    def start_neo4j():
+        """Start the neo4j graph database."""
+        return neo4j_manager.start()
 
     @mgr.component('redis', ())
-    def redis():
+    def start_redis():
         """Start redis."""
-        data_dir = 'data'
-        os.makedirs(os.path.join(paths.redis_root(), data_dir), exist_ok=True)
-        conf_name = 'redis.conf'
-        container_path = paths.redis_container()
-        # Dump the config
-        conf_path = os.path.join(paths.redis_root(), conf_name)
-        if not os.path.exists(conf_path):
-            with open(conf_path, 'w', encoding='utf-8') as fp:
-                # Don't listen on TCP
-                print('port 0', file=fp)
-                print('dir', os.path.join('/mnt', data_dir), file=fp)
-                print('maxmemory 2mb', file=fp)
-                print('unixsocket', os.path.join('/mnt', paths.redis_sock_fname()), file=fp)
-                print('unixsocketperm 700', file=fp)
-        cmd = [
-            'ch-run',
-            f'--bind={paths.redis_root()}:/mnt',
-            container_path,
-            'redis-server',
-            '/mnt/redis.conf',
-        ]
         log = open_log('redis')
-        # Ran into a strange "Failed to configure LOCALE for invalid locale name."
-        # from Redis, so setting LANG=C. This could have consequences for UTF-8
-        # strings.
-        env = dict(os.environ)
-        env['LANG'] = 'C'
-        env['LC_ALL'] = 'C'
-        return subprocess.Popen(cmd, env=env, stdout=log, stderr=log)
+        return redis_manager.start(log)
 
     # Workflow manager and task manager need to be opened with PIPE for their stdout/stderr
     if need_slurmrestd():
@@ -477,23 +462,6 @@ def stop(query='yes'):
         print(f'Beeflow has stopped. Check the log at "{beeflow_log}".')
 
 
-def kill_active_workflows(active_states, workflow_list):
-    """Kill workflows with active states."""
-    db_path = wf_utils.get_db_path()
-    db = connect_db(wfm_db, db_path)
-    success = True
-    for name, wf_id, state in workflow_list:
-        if state in active_states:
-            pid = db.workflows.get_gdb_pid(wf_id)
-            if pid > 0:
-                dep_manager.kill_gdb(pid)
-            else:
-                # Failure most likely caused by an Initializing workflow.
-                print(f"No process for {name}, {wf_id}, {state}.")
-                success = False
-    return success
-
-
 def archive_dir(dir_to_archive):
     """Archive directories for archive flag in reset."""
     archive_dirs = ['logs', 'container_archive', 'archives', 'workflows']
@@ -561,9 +529,6 @@ def reset(archive: bool = typer.Option(False, '--archive', '-a',
             # Exit out if the user didn't really mean to do a reset
             sys.exit()
         elif absolutely_sure in ("y", "yes"):
-            # First stop all active workflow processes
-            workflow_list = bee_client.get_wf_list()
-            kill_active_workflows(active_states, workflow_list)
             # Stop all of the beeflow processes
             stop("quiet")
             print("Beeflow is shutting down.")

{hpc_beeflow-0.1.9.dev3 → hpc_beeflow-0.1.9.dev4}/beeflow/common/config_driver.py

@@ -301,7 +301,7 @@ VALIDATOR.option('graphdb', 'https_port', default=DEFAULT_HTTPS_PORT,
                  info='HTTPS port used for the graph database')
 VALIDATOR.option('graphdb', 'gdb_image_mntdir', default=join_path('/tmp', USER),
                  info='graph database image mount directory', validator=validation.make_dir)
-VALIDATOR.option('graphdb', 'sleep_time', validator=int, default=10,
+VALIDATOR.option('graphdb', 'sleep_time', validator=int, default=1,
                  info='how long to wait for the graph database to come up (this can take a while, '
                       'depending on the system)')
 # Builder

{hpc_beeflow-0.1.9.dev3 → hpc_beeflow-0.1.9.dev4}/beeflow/common/db/wfm_db.py

@@ -10,7 +10,7 @@ class WorkflowInfo:
 
     def __init__(self, db_file):
         """Initialize Info and db file."""
-        self.Info = namedtuple("Info", "id wfm_port tm_port sched_port num_workflows") # noqa Snake Case
+        self.Info = namedtuple("Info", "id wfm_port tm_port sched_port num_workflows bolt_port http_port https_port gdb_pid") # noqa Snake Case
         self.db_file = db_file
 
     def set_port(self, component, new_port):
@@ -22,7 +22,7 @@ class WorkflowInfo:
         """Return port for the specified component."""
         # Need to add code here to make sure we chose a valid component.
         stmt = f"SELECT {component}_port FROM info"
-        result = bdb.getone(self.db_file, stmt)
+        result = bdb.getone(self.db_file, stmt)[0]
         port = result
         return port
 
@@ -45,6 +45,18 @@ class WorkflowInfo:
         info = self.Info(*result)
         return info
 
+    def get_gdb_pid(self):
+        """Return the gdb pid."""
+        stmt = "SELECT gdb_pid FROM info"
+        result = bdb.getone(self.db_file, stmt)[0]
+        gdb_pid = result
+        return gdb_pid
+
+    def update_gdb_pid(self, gdb_pid):
+        """Update the gdb PID."""
+        stmt = "UPDATE info SET gdb_pid=?"
+        bdb.run(self.db_file, stmt, [gdb_pid])
+
 
 class Workflows:
     """Workflow database object."""
@@ -53,7 +65,7 @@ class Workflows:
         """Initialize Task, db_file, and Workflow object."""
         self.Task = namedtuple("Task", "id task_id workflow_id name resource state slurm_id") #noqa
         self.db_file = db_file
-        self.Workflow = namedtuple("Workflow", "id workflow_id name state run_dir bolt_port http_port https_port gdb_pid") #noqa
+        self.Workflow = namedtuple("Workflow", "id workflow_id name state run_dir") #noqa
 
     def get_workflow(self, workflow_id):
         """Return a workflow object."""
@@ -69,13 +81,11 @@ class Workflows:
         workflows = [self.Workflow(*workflow) for workflow in result]
         return workflows
 
-    def init_workflow(self, workflow_id, name, run_dir, bolt_port, http_port, https_port):
+    def init_workflow(self, workflow_id, name, run_dir):
         """Insert a new workflow into the database."""
-        stmt = """INSERT INTO workflows (workflow_id, name, state, run_dir,
-                                         bolt_port, http_port, https_port, gdb_pid)
-                  VALUES(?, ?, ?, ?, ?, ?, ?, ?);"""
-        bdb.run(self.db_file, stmt, [workflow_id, name, 'Initializing', run_dir,
-                                     bolt_port, http_port, https_port, -1])
+        stmt = """INSERT INTO workflows (workflow_id, name, state, run_dir)
+                  VALUES(?, ?, ?, ?);"""
+        bdb.run(self.db_file, stmt, [workflow_id, name, 'Initializing', run_dir])
 
     def delete_workflow(self, workflow_id):
         """Delete a workflow from the database."""
@@ -123,42 +133,9 @@ class Workflows:
         result = bdb.getone(self.db_file, stmt, [task_id, workflow_id])
         return result
 
-    def get_bolt_port(self, workflow_id):
-        """Return the bolt port associated with a workflow."""
-        stmt = "SELECT bolt_port FROM workflows WHERE workflow_id=?"
-        result = bdb.getone(self.db_file, stmt, [workflow_id])[0]
-        bolt_port = result
-        return bolt_port
-
-    def get_http_port(self, workflow_id):
-        """Return the bolt port associated with a workflow."""
-        stmt = "SELECT http_port FROM workflows WHERE workflow_id=?"
-        result = bdb.getone(self.db_file, stmt, [workflow_id])[0]
-        http_port = result
-        return http_port
-
-    def get_https_port(self, workflow_id):
-        """Return the bolt port associated with a workflow."""
-        stmt = "SELECT https_port FROM workflows WHERE workflow_id=?"
-        result = bdb.getone(self.db_file, stmt, [workflow_id])[0]
-        https_port = result
-        return https_port
-
-    def get_gdb_pid(self, workflow_id):
-        """Return the bolt port associated with a workflow."""
-        stmt = "SELECT gdb_pid FROM workflows WHERE workflow_id=?"
-        result = bdb.getone(self.db_file, stmt, [workflow_id])[0]
-        gdb_pid = result
-        return gdb_pid
-
-    def update_gdb_pid(self, workflow_id, gdb_pid):
-        """Update the gdb PID associated with a workflow."""
-        stmt = "UPDATE workflows SET gdb_pid=? WHERE workflow_id=?"
-        bdb.run(self.db_file, stmt, [gdb_pid, workflow_id])
-
     def get_run_dir(self, workflow_id):
-        """Return the bolt port associated with a workflow."""
-        stmt = "SELECT run_dir FROM workflows WHERE workflow_id=?"
+        """Return the run directory."""
+        stmt = "SELECT run_dir FROM info WHERE workflow_id=?"
         result = bdb.getone(self.db_file, stmt, [workflow_id])[0]
         run_dir = result
         return run_dir
@@ -180,11 +157,8 @@ class WorkflowDB:
                                 workflow_id INTEGER UNIQUE,
                                 name TEXT,
                                 state TEST NOT NULL,
-                                run_dir STR,
-                                bolt_port INTEGER,
-                                http_port INTEGER,
-                                https_port INTEGER,
-                                gdb_pid INTEGER);"""
+                                run_dir STR
+                                );"""
 
         tasks_stmt = """CREATE TABLE IF NOT EXISTS tasks (
                         id INTEGER PRIMARY KEY,
@@ -205,7 +179,11 @@ class WorkflowDB:
                            wfm_port INTEGER,
                            tm_port INTEGER,
                            sched_port INTEGER,
-                           num_workflows INTEGER
+                           num_workflows INTEGER,
+                           bolt_port INTEGER,
+                           http_port INTEGER,
+                           https_port INTEGER,
+                           gdb_pid INTEGER
                            );"""
 
         bdb.create_table(self.db_file, workflows_stmt)
@@ -213,9 +191,9 @@ class WorkflowDB:
         if not bdb.table_exists(self.db_file, 'info'):
             bdb.create_table(self.db_file, info_stmt)
             # insert a new workflow into the database
-            stmt = """INSERT INTO info (wfm_port, tm_port, sched_port, num_workflows)
-                                            VALUES(?, ?, ?, ?);"""
-            bdb.run(self.db_file, stmt, [-1, -1, -1, 0])
+            stmt = """INSERT INTO info (wfm_port, tm_port, sched_port, num_workflows,
+                bolt_port, http_port, https_port, gdb_pid) VALUES(?, ?, ?, ?, ?, ?, ?, ?);"""
+            bdb.run(self.db_file, stmt, [-1, -1, -1, 0, -1, -1, -1, -1])
 
     @property
     def workflows(self):

hpc_beeflow-0.1.9.dev4/beeflow/common/deps/container_manager.py

@@ -0,0 +1,88 @@
+#!/usr/bin/env python3
+
+"""Functions for managing the BEE depency container and associated bind mounts."""
+
+import os
+import shutil
+import subprocess
+
+from beeflow.common.config_driver import BeeConfig as bc
+from beeflow.common import paths
+from celery import shared_task #noqa pylama can't find celery
+
+
+class NoContainerRuntime(Exception):
+    """An exception for no container runtime like charliecloud or singularity."""
+
+
+def check_container_runtime():
+    """Check if the container runtime is currently installed."""
+    # Needs to support singuarity as well
+    if shutil.which("ch-convert") is None or shutil.which("ch-run") is None:
+        print("ch-convert or ch-run not found. Charliecloud required"
+              " for neo4j container.")
+        raise NoContainerRuntime('')
+
+
+def make_dep_dir():
+    """Make a new bee dependency container directory."""
+    bee_workdir = paths.workdir()
+    bee_dir = f'{bee_workdir}/deps'
+    bee_dir_exists = os.path.isdir(bee_dir)
+    if not bee_dir_exists:
+        os.makedirs(bee_dir)
+
+
+def get_dep_dir():
+    """Return the dependency directory path."""
+    bee_workdir = paths.workdir()
+    bee_container_dir = f'{bee_workdir}/deps/'
+    return bee_container_dir
+
+
+def get_container_dir(dep_name):
+    """Return the depency container path."""
+    container_name = dep_name + '_container'
+    return get_dep_dir() + container_name
+
+
+def check_container_dir(dep_name):
+    """Return true if the container directory exists."""
+    container_dir = get_container_dir(dep_name)
+    container_dir_exists = os.path.isdir(container_dir)
+    return container_dir_exists
+
+
+def create_image(dep_name):
+    """Create a new BEE dependency container if one does not exist.
+
+    By default, the container is stored in /tmp/<user>/beeflow/deps.
+    """
+    # Can throw an exception that needs to be handled by the caller
+    check_container_runtime()
+
+    image = bc.get('DEFAULT', dep_name + '_image')
+
+    # Check for BEE dependency container directory:
+    container_dir_exists = check_container_dir(dep_name)
+    if container_dir_exists:
+        print(f"Already have {dep_name} container")
+        return
+
+    make_dep_dir()
+    container_dir = get_container_dir(dep_name)
+
+    # Build new dependency container
+    try:
+        subprocess.run(["ch-convert", "-i", "tar", "-o", "dir",
+                        str(image), str(container_dir)], check=True)
+    except subprocess.CalledProcessError as error:
+        print(f"ch-convert failed: {error}")
+        shutil.rmtree(container_dir)
+        print(f"{dep_name} container mount directory {container_dir} removed")
+        return
+
+    # If neo4j, make the certificates directory
+    if dep_name == 'neo4j':
+        container_certs_path = os.path.join(container_dir, 'var/lib/neo4j/certificates')
+        os.makedirs(container_certs_path, exist_ok=True)

hpc_beeflow-0.1.9.dev4/beeflow/common/deps/neo4j_manager.py

@@ -0,0 +1,164 @@
+"""Contains methods for managing neo4j instance."""
+import socket
+import os
+import shutil
+import re
+import subprocess
+import time
+
+from beeflow.wf_manager.resources import wf_utils
+from beeflow.common.db import wfm_db
+
+from beeflow.common import paths
+from beeflow.common.deps import container_manager
+
+from beeflow.common.config_driver import BeeConfig as bc
+from beeflow.common import log as bee_logging
+
+# Define directories within module scope
+bee_workdir = paths.workdir()
+mount_dir = os.path.join(bee_workdir, 'gdb_mount')
+data_dir = mount_dir + '/data'
+logs_dir = mount_dir + '/logs'
+run_dir = mount_dir + '/run'
+certs_dir = mount_dir + '/certificates'
+confs_dir = mount_dir + "/conf"
+container_path = container_manager.get_container_dir('neo4j')
+log = bee_logging.setup('neo4j')
+
+
+def get_open_port():
+    """Return an open ephemeral port."""
+    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    s.bind(("", 0))
+    s.listen(1)
+    port = s.getsockname()[1]
+    s.close()
+    return port
+
+
+def setup_ports():
+    """Return three open ports for bolt, http, https."""
+    # Get ports for neo4j to run
+    bolt_port = get_open_port()
+    http_port = get_open_port()
+    https_port = get_open_port()
+
+    db = wf_utils.connect_db(wfm_db, wf_utils.get_db_path())
+
+    db.info.set_port('bolt', bolt_port)
+    db.info.set_port('http', http_port)
+    db.info.set_port('https', https_port)
+    return bolt_port, http_port, https_port
+
+
+def setup_mounts():
+    """Set up mount directories for the graph database."""
+    os.makedirs(data_dir, exist_ok=True)
+    os.makedirs(logs_dir, exist_ok=True)
+    os.makedirs(mount_dir, exist_ok=True)
+    os.makedirs(certs_dir, exist_ok=True)
+    os.makedirs(run_dir, exist_ok=True)
+
+
+def setup_configs(bolt_port, http_port, https_port):
+    """Set up GDB configuration.
+
+    This function needs to be run before each new invocation since the
+    config file could have changed.
+    """
+    os.makedirs(confs_dir, exist_ok=True)
+    gdb_configfile = shutil.copyfile(container_path + "/var/lib/neo4j/conf/neo4j.conf",
+                                     confs_dir + "/neo4j.conf")
+    log.info(gdb_configfile)
+
+    with open(gdb_configfile, "rt", encoding="utf8") as cfile:
+        data = cfile.read()
+
+    log.info(f"new bolt: {bolt_port}")
+    bolt_config = r'#(server.bolt.listen_address=):[0-9]*'
+    data = re.sub(bolt_config, rf'\1:{bolt_port}', data)
+    http_config = r'#(server.http.listen_address=):[0-9]*'
+    data = re.sub(http_config, rf'\1:{http_port}', data)
+    https_config = r'#(server.https.listen_address=):[0-9]*'
+    data = re.sub(https_config, rf'\1:{https_port}', data)
+
+    with open(gdb_configfile, "wt", encoding="utf8") as cfile:
+        cfile.write(data)
+
+
+def create_credentials():
+    """Create the password and set the logfiles in environment."""
+    db_password = bc.get('graphdb', 'dbpass')
+    try:
+        command = ['neo4j-admin', 'dbms', 'set-initial-password', str(db_password)]
+        subprocess.run([
+            "ch-run",
+            "--set-env=" + container_path + "/ch/environment",
+            "-b", confs_dir + ":/var/lib/neo4j/conf",
+            "-b", data_dir + ":/data",
+            "-b", logs_dir + ":/logs",
+            "-b", run_dir + ":/var/lib/neo4j/run", container_path,
+            "--", *command
+        ], check=True)
+    except subprocess.CalledProcessError:
+        log.error("neo4j-admin set-initial-password failed")
+
+
+def create_database():
+    """Create the neo4j database and return the process."""
+    try:
+        command = ['neo4j', 'console']
+        proc = subprocess.Popen([ #noqa can't use with because returning
+            "ch-run",
+            "--set-env=" + container_path + "/ch/environment",
+            "-b", confs_dir + ":/var/lib/neo4j/conf",
+            "-b", data_dir + ":/data",
+            "-b", logs_dir + ":/logs",
+            "-b", run_dir + ":/var/lib/neo4j/run",
+            "-b", certs_dir + ":/var/lib/neo4j/certificates",
+            container_path, "--", *command
+        ], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        wait_gdb()
+        return proc
+
+    except FileNotFoundError:
+        log.error("Neo4j failed to start.")
+        return -1
+
+
+def start():
+    """Start the graph database."""
+    log.info('Starting Neo4j Database')
+
+    bolt_port, http_port, https_port = setup_ports()
+
+    setup_configs(bolt_port, http_port, https_port)
+
+    setup_mounts()
+
+    create_credentials()
+
+    return create_database()
+
+
+def wait_gdb():
+    """Need to wait for the GDB. Currently, we're using the sleep time paramater.
+
+    We'd like to remove that in the future.
+    """
+    gdb_sleep_time = bc.get('graphdb', 'sleep_time')
+    print(f'waiting {gdb_sleep_time}s for GDB to come up')
+    time.sleep(gdb_sleep_time)
+
+
+def remove_gdb():
+    """Remove the current GDB bind mount directory."""
+    gdb_workdir = os.path.join(bee_workdir, 'current_gdb')
+    old_gdb_workdir = os.path.join(bee_workdir, 'old_gdb')
+    if os.path.isdir(gdb_workdir):
+        # Rename the directory to guard against NFS errors
+        shutil.move(gdb_workdir, old_gdb_workdir)
+        time.sleep(2)
+        shutil.rmtree(old_gdb_workdir)
+        time.sleep(2)

hpc_beeflow-0.1.9.dev4/beeflow/common/deps/redis_manager.py

@@ -0,0 +1,37 @@
+"""Module contains the code for launching redis subprocess."""
+import os
+import subprocess
+
+from beeflow.common import paths
+
+
+def start(log):
+    """Start redis."""
+    data_dir = 'data'
+    os.makedirs(os.path.join(paths.redis_root(), data_dir), exist_ok=True)
+    conf_name = 'redis.conf'
+    container_path = paths.redis_container()
+    # Dump the config
+    conf_path = os.path.join(paths.redis_root(), conf_name)
+    if not os.path.exists(conf_path):
+        with open(conf_path, 'w', encoding='utf-8') as fp:
+            # Don't listen on TCP
+            print('port 0', file=fp)
+            print('dir', os.path.join('/mnt', data_dir), file=fp)
+            print('maxmemory 2mb', file=fp)
+            print('unixsocket', os.path.join('/mnt', paths.redis_sock_fname()), file=fp)
+            print('unixsocketperm 700', file=fp)
+    cmd = [
+        'ch-run',
+        f'--bind={paths.redis_root()}:/mnt',
+        container_path,
+        'redis-server',
+        '/mnt/redis.conf',
+    ]
+    # Ran into a strange "Failed to configure LOCALE for invalid locale name."
+    # from Redis, so setting LANG=C. This could have consequences for UTF-8
+    # strings.
+    env = dict(os.environ)
+    env['LANG'] = 'C'
+    env['LC_ALL'] = 'C'
+    return subprocess.Popen(cmd, env=env, stdout=log, stderr=log)

hpc_beeflow-0.1.9.dev4/beeflow/common/gdb/DESIGN.md

@@ -0,0 +1,74 @@
+# Graph Database Design
+
+## Basics
+
+A Neo4j graph consists of nodes and relationships. Nodes can have labels (e.g. `:Label`) and properties (name value pairs like `name = "Pat"`). Relationships have a type (e.g. `:DEPENDS_ON`) and properties. Property values can be `number`, `string`, `boolean`, spatial (only `Point`), temporal (e.g. `Time`, `Date`, `Duration`), or composite types of the previous types (e.g. lists and maps).
+
+## BEE Workflow Graph Entities
+
+A BEE workflow is a DAG (currently implemented in Neo4j) that represents a workflow originally specified in a CWL (Common Workflow Language) file. The fundamental structure of the workflow is based on `Task` nodes and `DEPENDS_ON` relationships. There are other nodes and relationships that are used to represent other properties of the workflow such as `Metadata`, `Requirements`, `Hints`, `Inputs`, `Outputs`,  etc. BEE's nodes, relationships, and properties are documented below.
+
+## List of Node Types
+* Listed edges are **outgoing**. Nodes are english cased, edges are all caps.
+
+### `:BEE`
+* This is the head node to the Graph Database. All `:Workflow` nodes point to this head. Only one such node exists in the graph database.
+* `name`: Is always named "Head"
+
+### `:Workflow`
+* Node representing a workflow
+* `id`: unique ID for a workflow in this database
+* `name`: name of workflow as a string
+* `state`: state of entire workflow
+    - See `states,py` for more information
+* `:WORKFLOW_OF`: Edge between from `:Workflow` to `:BEE` node to denote workflows of a BEE instance
+
+### `:Task`
+* Node representing a single task in a workflow
+* `id`: unique ID for every task in workflow
+* `workflow_id`: `id` of the workflow the task belongs to
+* `name`: name of task as a string
+* `base_command`: command (and parameters) to be executed
+* `stdout`: command output location for task
+* `stdin`: error output location for task
+* `:BEGINS`: Edge from `:Task` to `:Workflow` node to indicate using an input parameter from the workflow
+* `:DEPENDS_ON`: Edge from a `:Task`(1) to a `:Task`(2). (2) must be completed before (1) can be submitted to the task manager. `:DEPENDS_ON` relationships are deduced by parsing CWL.
+
+### `:Input`
+* Node that contains information for an input either to a workflow or task
+* `id`: Moniker for input
+* `type` Type of input (e.g File, string)
+* `value` Value of input
+* `:INPUT_OF`: Edge from `:Input` to either `:Task` or `:Workflow` node to indicate being an input to the workflow or task
+#### IF `:Input` node is `:INPUT_OF` a `:Task` node:
+* `position`: Serial number for multiple inputs
+* `source`: 
+
+### `:Output`
+* Node that contains information for an output either from a workflow or task
+* `id`: Moniker for output
+* `type` Type of output (e.g File, string)
+* `value` Value of output
+* `:OUTPUT_OF`: Edge from `:Output` to either `:Task` or `:Workflow` node to indicate being an output of the task or workflow.
+#### If `:Output` node is `:OUTPUT_OF` a `:Workflow` node:
+* `source`:
+#### IF `:Output` node is `:OUTPUT_OF` a `:Task` node:
+* `glob`: 
+
+### `:Hint`
+* Node specifying hints (optional parameters) for a task or workflow
+* `params`: Array of hint parameters for workflow or task
+* `:HINT_OF`: Edge from `:Hint` to either `:Task` or `:Workflow` node to indicate being a hint of task or workflow
+
+### `:Requirement`
+* Node specifying requirements (mandatory parameters) for a task or workflow
+* `params`: Array of requirement parameters for workflow or task
+* `:REQUIREMENT_OF`: Edge from `:Requirement` to either`:Task` or `:Workflow` node to indicate being a requirement of task or workflow
+
+### `Metadata`
+* Contains state of task in workflow
+* `state`: State of the task being described (defaults to `WAITING`)
+* :DESCRIBES`: edge to the `:Task` node whose metadata is being described
+
+
+

{hpc_beeflow-0.1.9.dev3 → hpc_beeflow-0.1.9.dev4}/beeflow/common/gdb/gdb_driver.py

@@ -284,17 +284,6 @@ class GraphDatabaseDriver(ABC):
         :rtype: bool
         """
 
-    @abstractmethod
-    def empty(self):
-        """Determine if the database is empty.
-
-        :rtype: bool
-        """
-
-    @abstractmethod
-    def cleanup(self):
-        """Clean up all the data stored in the graph database."""
-
     @abstractmethod
     def close(self):
         """Close the connection to the graph database."""