hpc-beeflow 0.1.10.dev0__tar.gz → 0.1.10.dev1__tar.gz

{hpc_beeflow-0.1.10.dev0 → hpc_beeflow-0.1.10.dev1}/PKG-INFO

@@ -1,12 +1,12 @@
 Metadata-Version: 2.1
 Name: hpc-beeflow
-Version: 0.1.10.dev0
+Version: 0.1.10.dev1
 Summary: A software package for containerizing HPC applications and managing job workflows
 Home-page: https://github.com/lanl/BEE
 Keywords: bee,hpc,workflow,cluster,computing
 Author: BEE-LANL Dev Team
 Author-email: bee-dev@lanl.gov
-Requires-Python: >=3.8.3,<=3.12.2
+Requires-Python: >=3.8.3,<=3.13.0
 Classifier: Environment :: Console
 Classifier: Environment :: OpenStack
 Classifier: Environment :: Other Environment
@@ -19,7 +19,8 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Topic :: Scientific/Engineering
 Classifier: Topic :: System :: Clustering
@@ -110,6 +111,7 @@ Contributors:
 * Patricia Grubel - `pagrubel <https://github.com/pagrubel>`_
 * Qiang Guan - `guanxyz <https://github.com/guanxyz>`_
 * Ragini Gupta - `raginigupta6 <https://github.com/raginigupta6>`_
+* Aaron R Hall - `arhall0 <https://github.com/arhall0>`_
 * Leah Howell - `Leahh02 <https://github.com/Leahh02>`_
 * Andres Quan - `aquan9 <https://github.com/aquan9>`_
 * Quincy Wofford - `qwofford <https://github.com/qwofford>`_

{hpc_beeflow-0.1.10.dev0 → hpc_beeflow-0.1.10.dev1}/README.rst

@@ -56,6 +56,7 @@ Contributors:
 * Patricia Grubel - `pagrubel <https://github.com/pagrubel>`_
 * Qiang Guan - `guanxyz <https://github.com/guanxyz>`_
 * Ragini Gupta - `raginigupta6 <https://github.com/raginigupta6>`_
+* Aaron R Hall - `arhall0 <https://github.com/arhall0>`_
 * Leah Howell - `Leahh02 <https://github.com/Leahh02>`_
 * Andres Quan - `aquan9 <https://github.com/aquan9>`_
 * Quincy Wofford - `qwofford <https://github.com/qwofford>`_

{hpc_beeflow-0.1.10.dev0 → hpc_beeflow-0.1.10.dev1}/beeflow/client/bee_client.py

@@ -11,6 +11,7 @@ import inspect
 import pathlib
 import shutil
 import subprocess
+import getpass
 import tarfile
 import tempfile
 import textwrap
@@ -79,15 +80,78 @@ def get_hostname():
     return curr_hn
 
 
+def set_backend_status(new_status):
+    """Set backend flag to true in database."""
+    db = bdb.connect_db(client_db, db_path())
+    db.info.set_backend_status(new_status)
+
+
+def check_backend_status():
+    """Check if backend flag has been set."""
+    db = bdb.connect_db(client_db, db_path())
+    status = db.info.get_backend_status()
+    return status
+
+
+def reset_client_db():
+    """Reset client db when beeflow is stopped."""
+    setup_hostname("")
+    set_backend_status("")
+
+
+def check_backend_jobs(start_hn, command=False):
+    """Check if there is an instance of beeflow running on a backend node."""
+    user_name = getpass.getuser()
+    cmd = ['squeue', '-u', f'{user_name}', '-o', '%N', '-h']
+    resp = subprocess.run(cmd, text=True, check=True, stdout=subprocess.PIPE)
+
+    # iterate through available nodes
+    data = resp.stdout.splitlines()
+    cur_alloc = False
+    if get_hostname() in data:
+        cur_alloc = True
+
+    if cur_alloc:
+        if command:
+            warn(f'beeflow was started on "{get_hostname()}" and you are trying to '
+                 f'run a command on "{start_hn}".')
+            sys.exit(1)
+        else:
+            warn(f'beeflow was started on compute node "{get_hostname()}" '
+                 'and it is still running. ')
+            sys.exit(1)
+    else:  # beeflow was started on compute node but user no longer owns node
+        if command:
+            warn('beeflow has not been started!')
+            sys.exit(1)
+        else:
+            warn('beeflow was started on a compute node (no longer owned by user) and '
+                 'not stopped correctly. ')
+            warn("Resetting client database.")
+            reset_client_db()
+            setup_hostname(start_hn)  # add to client db
+
+
+def check_db_flags(start_hn):
+    """Check that beeflow was stopped correctly during the last run."""
+    if get_hostname() and get_hostname() != start_hn and check_backend_status() == "":
+        warn(f'Error: beeflow is already running on "{get_hostname()}".')
+        sys.exit(1)
+    if get_hostname() and get_hostname() != start_hn and check_backend_status() == "true":
+        check_backend_jobs(start_hn)
+
+
 def check_hostname(curr_hn):
     """Check current front end name matches the one beeflow was started on."""
-    db = bdb.connect_db(client_db, db_path())
-    start_hn = db.info.get_hostname()
-    if start_hn and curr_hn != start_hn:  # noqa: don't use set instead
-        warn(f'beeflow was started on "{start_hn}" and you are trying to '
+    if get_hostname() and curr_hn != get_hostname() and check_backend_status() == "":
+        warn(f'beeflow was started on "{get_hostname()}" and you are trying to '
              f'run a command on "{curr_hn}".')
-    if start_hn == "":
+        sys.exit(1)
+    elif get_hostname() and curr_hn != get_hostname() and check_backend_status() == "true":
+        check_backend_jobs(curr_hn, command=True)
+    if get_hostname() == "" and check_backend_status() == "":
         warn('beeflow has not been started!')
+        sys.exit(1)
 
 
 def error_exit(msg, include_caller=True):
@@ -515,25 +579,6 @@ def query(wf_id: str = typer.Argument(..., callback=match_short_id)):
     return wf_status, tasks_status
 
 
-@app.command()
-def metadata(wf_id: str = typer.Argument(..., callback=match_short_id)):
-    """Get metadata about a given workflow."""
-    try:
-        conn = _wfm_conn()
-        resp = conn.get(_resource(wf_id) + '/metadata', timeout=60)
-    except requests.exceptions.ConnectionError:
-        error_exit('Could not reach WF Manager.')
-
-    if resp.status_code != requests.codes.okay:  # noqa (pylama doesn't know about the okay member)
-        error_exit('Could not successfully query workflow manager')
-
-    # Print and or return the metadata
-    data = resp.json()
-    for key, value in data.items():
-        typer.echo(f'{key} = {value}')
-    return data
-
-
 @app.command()
 def pause(wf_id: str = typer.Argument(..., callback=match_short_id)):
     """Pause a workflow (Running tasks will finish)."""
@@ -723,5 +768,4 @@ if __name__ == "__main__":
 
 # Ignore W0511: This allows us to have TODOs in the code
 # Ignore R1732: Significant code restructuring required to fix
-# Ignore R1714: Not using a set instead
 # pylama:ignore=W0511,R1732

{hpc_beeflow-0.1.10.dev0 → hpc_beeflow-0.1.10.dev1}/beeflow/client/core.py

@@ -9,7 +9,6 @@ services is specified using the appropriate flag(s) then ONLY those services
 will be started.
 """
 import os
-import re
 import signal
 import subprocess
 import socket
@@ -19,7 +18,6 @@ import datetime
 import time
 import importlib.metadata
 from pathlib import Path
-import packaging
 
 import daemon
 import typer
@@ -31,6 +29,7 @@ from beeflow.common.config_driver import AlterConfig
 from beeflow.common import cli_connection
 from beeflow.common import paths
 from beeflow.wf_manager.resources import wf_utils
+import beeflow.common.worker.utils as worker_utils
 
 from beeflow.common.deps import container_manager
 from beeflow.common.deps import neo4j_manager
@@ -168,22 +167,6 @@ def need_slurmrestd():
             and not bc.get('slurm', 'use_commands'))
 
 
-def get_slurmrestd_version():
-    """Get the newest slurmrestd version."""
-    resp = subprocess.run(["slurmrestd", "-s", "list"], check=True, stderr=subprocess.PIPE,
-                          text=True).stderr
-    resp = resp.split("\n")
-    # Confirm slurmrestd format is the same
-    # If the slurmrestd list outputs has changed potentially something else has broken
-    if "Possible OpenAPI plugins" not in resp[0]:
-        print("Slurmrestd OpenAPI format has changed and things may break")
-    api_versions = [line.split('/')[1] for line in resp[1:] if re.search(r"openapi/v\d+\.\d+\.\d+",
-                                                                         line)]
-    # Sort the versions and grab the newest one
-    newest_api = sorted(api_versions, key=packaging.version.Version, reverse=True)[0]
-    return newest_api
-
-
 def init_components():
     """Initialize the components and component manager."""
     mgr = ComponentManager()
@@ -265,10 +248,8 @@ def init_components():
             """Start BEESlurmRestD. Returns a Popen process object."""
             bee_workdir = bc.get('DEFAULT', 'bee_workdir')
             slurmrestd_log = '/'.join([bee_workdir, 'logs', 'restd.log'])
-            openapi_version = bc.get('slurm', 'openapi_version')
-            if not openapi_version:
-                # Detect the newest version of the slurmrestd API
-                openapi_version = get_slurmrestd_version()
+            openapi_version = worker_utils.get_slurmrestd_version()
+            print(f"Inferred slurmrestd version: {openapi_version}")
             slurm_args = f'-s openapi/{openapi_version}'
             # The following adds the db plugin we opted not to use for now
             # slurm_args = f'-s openapi/{openapi_version},openapi/db{openapi_version}'
@@ -325,11 +306,22 @@ def load_check_charliecloud():
 
 def check_dependencies(backend=False):
     """Check for various dependencies in the environment."""
-    # Check if running on compute node under Slurm scheduler
+    # Note: Code block of lines 309-320 is for SLURM scheduler!
+    # Check if running on compute node under Slurm scheduler and --backend not specified
     if not backend and os.environ.get('SLURM_JOB_NODELIST') is not None:
-        warn('Slurm job node detected! Beeflow should not be run on a compute node.')
+        warn('Slurm job node detected! Beeflow should not be run on a compute node. ')
         warn(f'SLURM_JOB_NODELIST = {os.environ.get("SLURM_JOB_NODELIST")}')
+        bee_client.reset_client_db()
         sys.exit(1)
+    # Check if running on front end with --backend flag specified
+    if backend and os.environ.get('SLURM_JOB_NODELIST') is None:
+        warn('Slurm node was not detected! Are you sure you are on a compute node? ')
+        warn("Please run 'beeflow core start' again without the --backend flag.")
+        bee_client.reset_client_db()
+        sys.exit(1)
+
+    # TO DO: Add checks when using Flux scheduler
+
     print('Checking dependencies...')
     # Check for Charliecloud and its version
     load_check_charliecloud()
@@ -416,13 +408,15 @@ def start(foreground: bool = typer.Option(False, '--foreground', '-F',
           '-B', help='allow to run on a backend node')):
     """Start all BEE components."""
     start_hn = socket.gethostname()  # hostname when beeflow starts
-    if bee_client.get_hostname() == "":
+    if bee_client.get_hostname() == "" and bee_client.check_backend_status() == "":
         bee_client.setup_hostname(start_hn)  # add to client db
-    elif bee_client.get_hostname() != start_hn:
-        warn(f'Error: beeflow is already running on "{bee_client.get_hostname()}."')
-        sys.exit(1)
+    else:
+        time.sleep(10)  # giving Slurm time to relinquish compute node, if necessary
+        bee_client.check_db_flags(start_hn)
+
     if backend:  # allow beeflow to run on backend node
         check_dependencies(backend=True)
+        bee_client.set_backend_status("true")  # add flag to db
     else:
         check_dependencies()
     mgr = init_components()
@@ -470,9 +464,10 @@ def start(foreground: bool = typer.Option(False, '--foreground', '-F',
 def status():
     """Check the status of beeflow and the components."""
     status_hn = socket.gethostname()  # hostname when beeflow core status returned
+    bee_client.check_hostname(status_hn)
     resp = cli_connection.send(paths.beeflow_socket(), {'type': 'status'})
     if resp is None:
-        bee_client.check_hostname(status_hn)
+        # bee_client.check_hostname(status_hn)  # need to remove since checked in line 465?
         sys.exit(1)
     print('beeflow components:')
     for comp, stat in resp['components'].items():
@@ -492,6 +487,7 @@ def info():
 def stop(query='yes'):
     """Stop the current running beeflow daemon."""
     stop_hn = socket.gethostname()  # hostname when beeflow core stop returned
+    bee_client.check_hostname(stop_hn)
     # Check workflow states; warn if there are active states, pause running workflows
     workflow_list = bee_client.get_wf_list()
     concern_states = {'Running', 'Initializing', 'Waiting'}
@@ -514,13 +510,13 @@ def stop(query='yes'):
             bee_client.pause(wf_id)
     resp = cli_connection.send(paths.beeflow_socket(), {'type': 'quit'})
     if resp is None:
-        bee_client.check_hostname(stop_hn)
+        # bee_client.check_hostname(stop_hn)  # need to remove since checked in line 487?
         sys.exit(1)
     # As long as it returned something, we should be good
     beeflow_log = paths.log_fname('beeflow')
     if query == "yes":
         print(f'Beeflow has stopped. Check the log at "{beeflow_log}".')
-        bee_client.setup_hostname("")
+        bee_client.reset_client_db()
 
 
 def archive_dir(dir_to_archive):
@@ -579,6 +575,7 @@ def reset(archive: bool = typer.Option(False, '--archive', '-a',
         Shutdown beeflow and all BEE components.
         Delete the bee_workdir directory which results in:
             Removing the archive of all workflows.
+                (unless archives is configured elsewhere).
             Removing the archive of workflow containers
                 (unless container_archive is configured elsewhere).
             Reset all databases associated with the beeflow app.

{hpc_beeflow-0.1.10.dev0 → hpc_beeflow-0.1.10.dev1}/beeflow/common/config_driver.py

@@ -105,7 +105,7 @@ class BeeConfig:
                 break
             except FileNotFoundError:
                 print("Configuration file is missing! Generating new config file.")
-                new(USERCONFIG_FILE)
+                new(USERCONFIG_FILE, interactive=False)
         # remove default keys from the other sections
         cls.CONFIG = config_utils.filter_and_validate(config, VALIDATOR)
 
@@ -222,6 +222,7 @@ DEFAULT_NEO4J_IMAGE = join_path('/usr/projects/BEE/neo4j.tar.gz')
 DEFAULT_REDIS_IMAGE = join_path('/usr/projects/BEE/redis.tar.gz')
 
 DEFAULT_BEE_WORKDIR = join_path(HOME_DIR, '.beeflow')
+DEFAULT_BEE_ARCHIVE_DIR = join_path(DEFAULT_BEE_WORKDIR, 'archives')
 DEFAULT_BEE_DROPPOINT = join_path(HOME_DIR, '.beeflow/droppoint')
 USER = getpass.getuser()
 
@@ -240,33 +241,39 @@ VALIDATOR = ConfigValidator('BEE configuration file and validation information.'
 VALIDATOR.section('DEFAULT', info='Default bee.conf configuration section.')
 
 VALIDATOR.option('DEFAULT', 'bee_workdir', info='main BEE workdir',
-                 default=DEFAULT_BEE_WORKDIR, validator=validation.make_dir)
+                 default=DEFAULT_BEE_WORKDIR, validator=validation.make_dir,
+                 prompt=True)
+
+VALIDATOR.option('DEFAULT', 'bee_archive_dir', info='directory to store workflow archives',
+                 default=DEFAULT_BEE_ARCHIVE_DIR, validator=validation.make_dir,
+                 prompt=True)
 
 VALIDATOR.option('DEFAULT', 'bee_droppoint', info='BEE remote workflow drop point',
-                 default=DEFAULT_BEE_DROPPOINT, validator=validation.make_dir)
+                 default=DEFAULT_BEE_DROPPOINT, validator=validation.make_dir,
+                 prompt=False)
 
 VALIDATOR.option('DEFAULT', 'remote_api', info='BEE remote REST API activation',
-                 default=False, validator=validation.bool_)
+                 default=False, validator=validation.bool_, prompt=False)
 
 VALIDATOR.option('DEFAULT', 'remote_api_port', info='BEE remote REST API port',
-                 default=7777, validator=int)
+                 default=7777, validator=int, prompt=False)
 
 VALIDATOR.option('DEFAULT', 'workload_scheduler', choices=('Slurm', 'LSF', 'Flux', 'Simple'),
-                 info='backend workload scheduler to interact with ')
+                 default='Slurm', info='backend workload scheduler to interact with ',
+                 prompt=True)
 
 VALIDATOR.option('DEFAULT', 'delete_completed_workflow_dirs', validator=validation.bool_,
-                 default=True, info='delete workflow directory for completed jobs')
+                 default=True, info='delete workflow directory for completed jobs', prompt=False)
 
 VALIDATOR.option('DEFAULT', 'neo4j_image', validator=validation.file_,
                  default=NEO4J_IMAGE, info='neo4j container image',
-                 input_fn=filepath_completion_input)
+                 input_fn=filepath_completion_input, prompt=True)
 
 VALIDATOR.option('DEFAULT', 'redis_image', validator=validation.file_,
                  default=REDIS_IMAGE, info='redis container image',
-                 input_fn=filepath_completion_input)
+                 input_fn=filepath_completion_input, prompt=True)
 
-VALIDATOR.option('DEFAULT', 'max_restarts', validator=int,
-                 default=3,
+VALIDATOR.option('DEFAULT', 'max_restarts', validator=int, default=3, prompt=False,
                  info='max number of times beeflow will restart a component on failure')
 
 # Workflow Manager
@@ -275,27 +282,31 @@ VALIDATOR.section('workflow_manager', info='Workflow manager section.')
 VALIDATOR.section('task_manager',
                   info='Task manager configuration and config of container to use.')
 VALIDATOR.option('task_manager', 'container_runtime', default='Charliecloud',
-                 choices=('Charliecloud', 'Singularity'),
+                 choices=('Charliecloud', 'Singularity'), prompt=False,
                  info='container runtime to use for configuration')
-VALIDATOR.option('task_manager', 'runner_opts', default='',
+VALIDATOR.option('task_manager', 'runner_opts', default='', prompt=False,
                  info='special runner options to pass to the runner opts')
 VALIDATOR.option('task_manager', 'background_interval', default=5,
-                 validator=int,
+                 validator=int, prompt=False,
                  info='interval at which the task manager processes queues and updates states')
 
 # Charliecloud (depends on task_manager::container_runtime == Charliecloud)
 VALIDATOR.section('charliecloud', info='Charliecloud configuration section.',
                   depends_on=('task_manager', 'container_runtime', 'Charliecloud'))
-VALIDATOR.option('charliecloud', 'image_mntdir', default=join_path('/tmp', USER),
+VALIDATOR.option('charliecloud', 'image_mntdir', default=join_path('/tmp', USER), prompt=False,
                  info='Charliecloud mount directory', validator=validation.make_dir)
 # General job requirements
 VALIDATOR.section('job', info='General job requirements.')
-VALIDATOR.option('job', 'default_account', validator=lambda val: val.strip(),
-                 info='default account to launch jobs with (leave blank if none)')
-VALIDATOR.option('job', 'default_time_limit', validator=validation.time_limit,
-                 info='default account time limit (leave blank if none)')
-VALIDATOR.option('job', 'default_partition', validator=lambda val: val.strip(),
-                 info='default partition to run jobs on (leave blank if none)')
+VALIDATOR.option('job', 'default_account', validator=lambda val: val.strip(), prompt=True,
+                 default='', info='default account to launch jobs with (leave blank if none)')
+VALIDATOR.option('job', 'default_time_limit', validator=validation.time_limit, prompt=True,
+                 default='', info='default account time limit (leave blank if none)')
+VALIDATOR.option('job', 'default_partition', validator=lambda val: val.strip(), prompt=True,
+                 default='', info='default partition to run jobs on (leave blank if none)')
+VALIDATOR.option('job', 'default_qos', validator=lambda val: val.strip(), prompt=True,
+                 default='', info='default qos to run jobs on (leave blank if none)')
+VALIDATOR.option('job', 'default_reservation', validator=lambda val: val.strip(), prompt=True,
+                 default='', info='default reservation to run jobs on (leave blank if none)')
 
 
 def validate_chrun_opts(opts):
@@ -309,49 +320,49 @@ def validate_chrun_opts(opts):
 
 
 VALIDATOR.option('charliecloud', 'chrun_opts', default='--home',
-                 validator=validate_chrun_opts,
+                 validator=validate_chrun_opts, prompt=False,
                  info='extra options to pass to ch-run')
-VALIDATOR.option('charliecloud', 'setup', default='',
+VALIDATOR.option('charliecloud', 'setup', default='', prompt=False,
                  info='extra Charliecloud setup to put in a job script')
 # Graph Database
 VALIDATOR.section('graphdb', info='Main graph database configuration section.')
-VALIDATOR.option('graphdb', 'hostname', default='localhost',
+VALIDATOR.option('graphdb', 'hostname', default='localhost', prompt=False,
                  info='hostname of database')
 
-VALIDATOR.option('graphdb', 'dbpass', default='password', info='password for database')
+VALIDATOR.option('graphdb', 'dbpass', default='password', info='password for database',
+                 prompt=False)
 
-VALIDATOR.option('graphdb', 'gdb_image_mntdir', default=join_path('/tmp', USER),
+VALIDATOR.option('graphdb', 'gdb_image_mntdir', default=join_path('/tmp', USER), prompt=False,
                  info='graph database image mount directory', validator=validation.make_dir)
-VALIDATOR.option('graphdb', 'sleep_time', validator=int, default=1,
+VALIDATOR.option('graphdb', 'sleep_time', validator=int, default=1, prompt=False,
                  info='how long to wait for the graph database to come up (this can take a while, '
                       'depending on the system)')
 # Builder
 VALIDATOR.section('builder', info='General builder configuration section.')
-VALIDATOR.option('builder', 'deployed_image_root', default='/tmp',
+VALIDATOR.option('builder', 'deployed_image_root', default='/tmp', prompt=False,
                  info='where to deploy container images', validator=validation.make_dir)
-VALIDATOR.option('builder', 'container_output_path', default='/tmp',
+VALIDATOR.option('builder', 'container_output_path', default='/tmp', prompt=False,
                  info='container output path', validator=validation.make_dir)
-VALIDATOR.option('builder', 'container_archive',
+VALIDATOR.option('builder', 'container_archive', prompt=True,
                  default=join_path(DEFAULT_BEE_WORKDIR, 'container_archive'),
                  info='container archive location')
 VALIDATOR.option('builder', 'container_type', default='charliecloud',
-                 info='container type to use')
+                 info='container type to use', prompt=False)
 # Slurmrestd (depends on DEFAULT:workload_scheduler == Slurm)
 VALIDATOR.section('slurm', info='Configuration section for Slurm.',
                   depends_on=('DEFAULT', 'workload_scheduler', 'Slurm'))
 VALIDATOR.option('slurm', 'use_commands', validator=validation.bool_,
-                 default=(shutil.which('slurmrestd') is None),
+                 default=(shutil.which('slurmrestd') is None), prompt=False,
                  info='if set, use slurm cli commands instead of slurmrestd')
 DEFAULT_SLURMRESTD_SOCK = join_path('/tmp', f'slurm_{USER}_{random.randint(1, 10000)}.sock')
-VALIDATOR.option('slurm', 'openapi_version', default='v0.0.39',
-                 info='openapi version to use for slurmrestd')
+
 # Scheduler
 VALIDATOR.section('scheduler', info='Scheduler configuration section.')
 SCHEDULER_ALGORITHMS = ('fcfs', 'backfill', 'sjf')
 VALIDATOR.option('scheduler', 'algorithm', default='fcfs', choices=SCHEDULER_ALGORITHMS,
-                 info='scheduling algorithm to use')
+                 info='scheduling algorithm to use', prompt=False)
 VALIDATOR.option('scheduler', 'default_algorithm', default='fcfs',
-                 choices=SCHEDULER_ALGORITHMS,
+                 choices=SCHEDULER_ALGORITHMS, prompt=False,
                  info=('default algorithm to use'))
 
 
@@ -370,19 +381,20 @@ class ConfigGenerator:
         self.validator = validator
         self.sections = {}
 
-    def choose_values(self):
+    def choose_values(self, interactive=False, flux=False):
         """Choose configuration values based on user input."""
         dirname = os.path.dirname(self.fname)
         if dirname:
             os.makedirs(dirname, exist_ok=True)
         print(f'Creating a new config: "{self.fname}".')
         print()
-        print_wrap('This will walk you through creating a new configuration for BEE. '
-                   'Note that you will only be required to enter values for options '
-                   'without defaults. Please take a look at the other options and '
-                   'their defaults before running BEE.')
-        print()
-        print('Please enter values for the following sections and options:')
+        if interactive:
+            print_wrap('This will walk you through creating a new configuration for BEE. '
+                       'Note that you will only be required to enter values for some '
+                       'options. Please take a look at the other options and their '
+                       'values before running BEE.')
+            print()
+            print('Please enter values for the following sections and options:')
         # Let the user choose values for each required attribute
         for sec_name, section in self.validator.sections:
             # Determine if this section is valid under the current configuration
@@ -398,10 +410,13 @@ class ConfigGenerator:
                     print()
                     printed = True
 
+                this_default = option.default
+                if flux is True and opt_name == 'workload_scheduler':
+                    this_default = "Flux"
+                    option.prompt = False
                 # Check for a default value
-                if option.default is not None:
-                    default = option.default
-                    value = option.validate(default)
+                if (not interactive or option.prompt is False) and this_default is not None:
+                    value = option.validate(this_default)
                     print(f'Setting option "{opt_name}" to default value "{value}".')
                     print()
                     self.sections[sec_name][opt_name] = value
@@ -424,7 +439,8 @@ class ConfigGenerator:
             print_wrap(f'{opt_name} - {option.info}')
             if option.choices is not None:
                 print(f'(allowed values: {",".join(option.choices)})')
-            value = input_fn(f'{opt_name}: ')
+            value = input_fn(f'Enter selection for {opt_name} or\n'
+                             + f'leave blank for default ({option.default}): ') or option.default
             # Validate the input
             try:
                 option.validate(value)
@@ -433,7 +449,7 @@ class ConfigGenerator:
                 value = None
         return value
 
-    def save(self):
+    def save(self, interactive=False):
         """Save the config to a file."""
         print()
         print('The following configuration options were chosen:')
@@ -443,10 +459,11 @@ class ConfigGenerator:
             for opt_name in section:
                 print(f'{opt_name} = {section[opt_name]}')
         print()
-        ans = input('Would you like to save this config? [y/n] ')
-        if ans.lower() != 'y':
-            print('Quitting without saving')
-            return
+        if interactive:
+            ans = input('Would you like to save this config? [y/n] ')
+            if ans.lower() != 'y':
+                print('Quitting without saving')
+                return
         config_utils.write_config(self.fname, self.sections)
         print(70 * '#')
         print('Before running BEE, check defaults in the configuration file:',
@@ -565,12 +582,21 @@ def info():
 
 @app.command()
 def new(path: str = typer.Argument(default=USERCONFIG_FILE,
-                                   help='Path to new config file')):
+                                   help='Path to new config file'),
+        interactive: bool = typer.Option(False, '--interactive', '-i',
+                                         help='Whether or not to be prompted'
+                                         + ' during config generation'),
+        flux: bool = typer.Option(False, '--flux', '-f',
+                                  help='Changes default scheduler to Flux')):
     """Create a new config file."""
     if os.path.exists(path):
-        if check_yes(f'Path "{path}" already exists.\nWould you like to save a copy of it?'):
+        if not interactive or check_yes(f'Path "{path}" already exists.\n'
+                                        + 'Would you like to save a copy of it?'):
             config_utils.backup(path)
-    ConfigGenerator(path, VALIDATOR).choose_values().save()
+    ConfigGenerator(path, VALIDATOR).choose_values(
+        flux=flux,
+        interactive=interactive
+    ).save(interactive=interactive)
 
 
 @app.command()

{hpc_beeflow-0.1.10.dev0 → hpc_beeflow-0.1.10.dev1}/beeflow/common/config_validator.py

@@ -160,7 +160,8 @@ class ConfigSection:
 class ConfigOption:
     """Config option/validation class."""
 
-    def __init__(self, info, validator=str, choices=None, default=None, input_fn=None):
+    def __init__(self, info, validator=str, choices=None, default=None, input_fn=None,
+                 prompt=False):
         """Construct the config option class."""
         self.info = info
         self.choices = choices
@@ -169,6 +170,7 @@ class ConfigOption:
         # Function used to set the option from user input at config generation time
         self.input_fn = input_fn
         self._validator = validator
+        self.prompt = prompt
 
     def validate(self, value):
         """Validate the value and return it."""