cmd-queue 0.1.12__py3-none-any.whl → 0.1.14__py3-none-any.whl

cmd_queue/__init__.py

@@ -1,8 +1,16 @@
 r"""
-Simplify chaining of multiple shell commands
+The cmd_queue module is a tool that lets users define a DAG of bash commands.
+This DAG can be executed in a lightweight tmux backend, or a heavyweight slurm
+backend, or in simple serial mode that runs in the foreground thread. Rich
+provides monitoring / live control. For more information see the gitlab
+`README <https://gitlab.kitware.com/computer-vision/cmd_queue>`_.
+There is also a
+`Google slides presentation <https://docs.google.com/presentation/d/1BjJkjMx6bxu1uek-hAGpwj760u9rraVn7st8J5OsZME>`_
+that gives a high level overview.
+
+The following examples show how to use the cmd_queue API in Python. For
+examples of the Bash API see: :mod:`cmd_queue.__main__`.
 
-Serves as a frontend for several DAG backends, including our own custom tmux
-queue. We also support slurm and will soon support airflow.
 
 Example:
     >>> # The available backends classmethod lets you know which backends
@@ -295,7 +303,7 @@ Example:
 __mkinit__ = """
 mkinit -m cmd_queue
 """
-__version__ = '0.1.12'
+__version__ = '0.1.14'
 
 
 __submodules__ = {

cmd_queue/__main__.py

@@ -1,5 +1,15 @@
 #!/usr/bin/env python3
 # PYTHON_ARGCOMPLETE_OK
+"""
+This is the main script for the cmd_queue CLI. The :class:`CmdQueueConfig`
+defines the available options and its docstring provides a quick tutorial.
+For help run:
+
+.. code:: bash
+
+    cmd_queue --help
+
+"""
 import scriptconfig as scfg
 import ubelt as ub
 
@@ -12,13 +22,12 @@ class CmdQueueConfig(scfg.DataConfig):
     Most behaviors are related to creating and submitting custom queues.
 
     The ``cleanup`` action is for helping to manage the tmux backend, maingly
-    killing session names that start with "cmdq_".
+    killing session names that start with ``"cmdq_"``.
 
-    Example:
+    Step 1:  Initialize a new queue
+    ###############################
 
-        #################################
-        # Step 1:  Initialize a new queue
-        #################################
+    .. code:: bash
 
         cmd_queue new "my_cli_queue"
 
@@ -32,9 +41,10 @@ class CmdQueueConfig(scfg.DataConfig):
         # e.g. for pyenv
         cmd_queue new "my_cli_queue" --header="pyenv shell 3.11.0 && source $PYENV_ROOT/versions/3.11.0/envs/pyenv3.11.0/bin/activate"
 
-        #################################
-        # Step 2:  Initialize a new queue
-        #################################
+    Step 2:  Initialize a new queue
+    ###############################
+
+    .. code:: bash
 
         cmd_queue submit "my_cli_queue" --  echo hello world
         cmd_queue submit "my_cli_queue" --  echo "hello world"
@@ -45,14 +55,17 @@ class CmdQueueConfig(scfg.DataConfig):
         cmd_queue submit "my_cli_queue" -- 'cowsay MOOOOOO && sleep 3'
         cmd_queue submit "my_cli_queue" -- 'cowsay MOOOOOOOO && sleep 4'
 
-        #################################
-        # Step 3:  Inspect your commands before you run
-        #################################
+    Step 3:  Inspect your commands before you run
+    #############################################
+
+    .. code:: bash
+
         cmd_queue show "my_cli_queue"
 
-        #################################
-        # Step 4:  Run your commands
-        #################################
+    Step 4:  Run your commands
+    ##########################
+
+    .. code:: bash
 
         # Run using the serial backend
         cmd_queue run "my_cli_queue" --backend=serial
@@ -60,9 +73,10 @@ class CmdQueueConfig(scfg.DataConfig):
         # Run using the tmux backend
         cmd_queue run "my_cli_queue" --backend=tmux --workers=2
 
-        #################################
-        # Extra: other features
-        #################################
+    Extra: other features
+    #####################
+
+    .. code:: bash
 
         # List all the known queues you've created
         cmd_queue list
@@ -99,7 +113,7 @@ class CmdQueueConfig(scfg.DataConfig):
         '''
     ))
 
-    def normalize(config):
+    def __post_init__(config):
         if config['dpath'] == 'auto':
             config['dpath'] = str(ub.Path.appdir('cmd_queue/cli'))
 
@@ -109,8 +123,7 @@ def main(cmdline=1, **kwargs):
     Example:
         >>> # xdoctest: +SKIP
         >>> cmdline = 0
-        >>> kwargs = dict(
-        >>> )
+        >>> kwargs = dict()
         >>> main(cmdline=cmdline, **kwargs)
     """
     import json

cmd_queue/cli_boilerplate.py

@@ -131,7 +131,7 @@ class CMDQueueConfig(scfg.DataConfig):
         Command to start the appropriate virtual environment if your bashrc
         does not start it by default.'''), group='cmd-queue')
 
-    tmux_workers = scfg.Value(4, help='number of tmux workers in the queue for the tmux backend', group='cmd-queue')
+    tmux_workers = scfg.Value(8, help='number of tmux workers in the queue for the tmux backend', group='cmd-queue')
 
     slurm_options = scfg.Value(None, help='if the backend is slurm, provide a YAML dictionary for things like partition / etc...', group='cmd-queue')
 

cmd_queue/tmux_queue.py

@@ -54,7 +54,6 @@ import uuid
 
 from cmd_queue import base_queue
 from cmd_queue import serial_queue
-from cmd_queue.util import util_tags
 from cmd_queue.util.util_tmux import tmux
 
 

{cmd_queue-0.1.12.dist-info → cmd_queue-0.1.14.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cmd-queue
-Version: 0.1.12
+Version: 0.1.14
 Summary: The cmd_queue module for a DAG of bash commands
 Home-page: https://gitlab.kitware.com/computer-vision/cmd_queue
 Author: Kitware Inc., Jon Crall
@@ -49,7 +49,7 @@ Requires-Dist: xdoctest ; extra == 'all'
 Provides-Extra: all-strict
 Requires-Dist: ubelt (==1.2.3) ; extra == 'all-strict'
 Requires-Dist: rich (==12.5.1) ; extra == 'all-strict'
-Requires-Dist: scriptconfig (==0.7.7) ; extra == 'all-strict'
+Requires-Dist: scriptconfig (==0.7.8) ; extra == 'all-strict'
 Requires-Dist: psutil (==5.9.1) ; extra == 'all-strict'
 Requires-Dist: ruamel.yaml (==0.17.21) ; extra == 'all-strict'
 Requires-Dist: xdoctest (==1.0.1) ; extra == 'all-strict'
@@ -106,7 +106,7 @@ Requires-Dist: textual ; (python_version >= "3.7") and extra == 'optional'
 Provides-Extra: runtime-strict
 Requires-Dist: ubelt (==1.2.3) ; extra == 'runtime-strict'
 Requires-Dist: rich (==12.5.1) ; extra == 'runtime-strict'
-Requires-Dist: scriptconfig (==0.7.7) ; extra == 'runtime-strict'
+Requires-Dist: scriptconfig (==0.7.8) ; extra == 'runtime-strict'
 Requires-Dist: psutil (==5.9.1) ; extra == 'runtime-strict'
 Requires-Dist: ruamel.yaml (==0.17.21) ; extra == 'runtime-strict'
 Requires-Dist: numpy (==1.19.3) ; (python_version < "3.10" and python_version >= "3.6.0") and extra == 'runtime-strict'
@@ -147,17 +147,15 @@ Requires-Dist: pytest ; (python_version >= "3.7") and extra == 'tests'
 Command Queue - cmd_queue
 =========================
 
-.. .. |GitlabCIPipeline| |GitlabCICoverage| |Appveyor| |Codecov| 
+.. .. |GitlabCIPipeline| |GitlabCICoverage| |Appveyor| |Codecov|
 
 |Pypi| |Downloads| |ReadTheDocs|
 
 
-.. The ``cmd_queue`` module.
-
 +------------------+-------------------------------------------------------------------------------------+
 | Read the docs    | https://cmd_queue.readthedocs.io                                                    |
 +------------------+-------------------------------------------------------------------------------------+
-| Github           | https://github.com/Erotemic/cmd_queue                                               |
+| Gitlab           | https://gitlab.kitware.com/computer-vision/cmd_queue                                |
 +------------------+-------------------------------------------------------------------------------------+
 | Pypi             | https://pypi.org/project/cmd_queue                                                  |
 +------------------+-------------------------------------------------------------------------------------+
@@ -170,13 +168,13 @@ jobs (in parallel if possible) on a single machine. There are 3 backends with
 increasing levels of complexity: serial, tmux, and slurm.
 
 In serial mode, a single bash script gets written that executes your jobs in
-sequence. There are no external dependencies 
+sequence. There are no external dependencies
 
 In tmux mode, multiple tmux sessions get opened and each of them executes your
 independent parts of your jobs. Dependencies are handled.
 
 In slurm mode, a real heavy-weight scheduling algorithm is used. In this mode
-we simply convert your jobs to slurm commands and execute them. 
+we simply convert your jobs to slurm commands and execute them.
 
 Under the hood we build a DAG based on your specified dependencies and use this
 to appropriately order jobs.
@@ -185,15 +183,54 @@ By default, bash scripts that would execute your jobs print to the console.
 This gives the user fine-grained control if they only want to run a subset of a
 pipeline manually. But if asked to run, cmd_queue will execute the bash jobs.
 
+Features
+~~~~~~~~
+
+* Bash command scheduling
+
+* Execution is optional, can just print commands instead
+
+* No-parallelism always-available serial backend
+
+* Tmux based lightweight backend
+
+* Slurm based heavyweight backend
+
+* Python and Bash interface
+
+* Rich monitoring / live-control
+
+
+Installation
+============
+
+The cmd_queue package is available on pypi.
+
+.. code:: bash
+
+    pip install cmd_queue
+
+The serial queue backend will always work. To gain access other backends you
+must install their associated dependencies. The tmux backend is the easiest and
+simply requires that tmux is installed (e.g. ``sudo apt install tmux`` on
+Debian systems).
+
+Other backends require more complex setups. The slurm backend will require that
+`slurm is installed <https://slurm.schedmd.com/quickstart_admin.html>`_ and the
+daemon is running. The slurm backend is functional and tested, but improvements
+can still be made (help wanted). The airflow backend similarly requires a
+configured airflow server, but is not fully functional or tested (contributions
+to make airflow work / easier are wanted!).
+
 
 Tmux Queue Demo
 ===============
 
 After installing, the following command runs a demo of the tmux queue:
 
-.. code:: bash 
+.. code:: bash
 
-   # Reproduce the 
+   # Reproduce the
    INTERACTIVE_TEST=1 xdoctest -m cmd_queue.tmux_queue TMUXMultiQueue.monitor:1
 
 
@@ -221,7 +258,7 @@ submits several bash jobs with non-trivial dependencies.
     self.print_graph()
     if self.is_available():
         self.run(block=True, other_session_handler='kill')
-   
+
 
 When running the ``print_commands`` command will first display all of the submitted
 commands that will be distributed across multiple new tmux sessions. These are
@@ -235,8 +272,8 @@ command templating is correct before the queue is executed with ``run``.
    :align: left
 
 
-The ``print_graph`` command will render the DAG to be executed using `network
-text <https://github.com/networkx/networkx/pull/5602>`_.
+The ``print_graph`` command will render the DAG to be executed using
+`network text <https://networkx.org/documentation/stable/reference/readwrite/generated/networkx.readwrite.text.write_network_text.html#networkx.readwrite.text.write_network_text>`_.
 And finally ``run`` is called with ``block=True``, which starts executing the
 DAG and displays progress and job status in rich or textual monitor.
 
@@ -255,7 +292,7 @@ While this is running it is possible to simply attach to a tmux sessions (e.g.
 using ``<ctrl-b>s`` inside of a tmux session to view and navigate through the
 tmux sessions). Unlike the slurm backend, the entire execution of the DAG is
 entirely transparent to the developer! The following screenshot shows the tmux
-sessions spawned while running this demo. 
+sessions spawned while running this demo.
 
 .. .. Screenshot of the tmux sessions
 .. image:: https://i.imgur.com/46LRK8M.png
@@ -327,7 +364,7 @@ use cmd_queue to "transpile" these sequences of commands to pure bash.
    self.print_commands()
 
    # Display the real bash that gets executed under the hood
-   # that is independencly executable, tracks the success / failure of each job, 
+   # that is independencly executable, tracks the success / failure of each job,
    # and manages dependencies.
    self.print_commands(1, 1)
 
@@ -336,7 +373,7 @@ use cmd_queue to "transpile" these sequences of commands to pure bash.
    self.run(block=True)
 
 
-This prints the bash commands in an appropriate order to resolve dependencies. 
+This prints the bash commands in an appropriate order to resolve dependencies.
 
 
 .. code:: bash
@@ -376,7 +413,7 @@ This prints the bash commands in an appropriate order to resolve dependencies.
     #
     ### Command 10 / 10 - demo_queue-job-9
     echo bazbiz && sleep 0.5
-       
+
 
 
 .. code:: python
@@ -399,7 +436,7 @@ This prints the bash commands in an appropriate order to resolve dependencies.
    self.print_commands()
 
    # Display the real bash that gets executed under the hood
-   # that is independencly executable, tracks the success / failure of each job, 
+   # that is independencly executable, tracks the success / failure of each job,
    # and manages dependencies.
    self.print_commands(1, 1)
 
@@ -408,8 +445,8 @@ This prints the bash commands in an appropriate order to resolve dependencies.
    self.run(block=True)
 
 
-This prints the sequence of bash commands that will be executed in each tmux session. 
- 
+This prints the sequence of bash commands that will be executed in each tmux session.
+
 .. code:: bash
 
     # --- /home/joncrall/.cache/base_queue/demo_queue_2022-04-08_a1ef7600/queue_demo_queue_0_2022-04-08_a1ef7600.sh
@@ -503,7 +540,7 @@ options here
    self.print_commands()
 
    # Display the real bash that gets executed under the hood
-   # that is independencly executable, tracks the success / failure of each job, 
+   # that is independencly executable, tracks the success / failure of each job,
    # and manages dependencies.
    self.print_commands(1, 1)
 
@@ -513,7 +550,7 @@ options here
 
 
 This prints the very simple slurm submission script:
- 
+
 .. code:: bash
 
     # --- /home/joncrall/.cache/slurm_queue/demo_queue-20220408T170615-a9e238b5/demo_queue-20220408T170615-a9e238b5.sh
@@ -532,36 +569,14 @@ This prints the very simple slurm submission script:
 
 
 
-Installation
-============
-
-The cmd_queue package is available on pypi. 
- 
-.. code:: bash
-
-    pip install cmd_queue
-
-The serial queue backend will always work. To gain access other backends you
-must install their associated dependencies. The tmux backend is the easiest and
-simply requires that tmux is installed (e.g. ``sudo apt install tmux`` on
-Debian systems). 
-
-Other backends require more complex setups. The slurm backend will require that
-slurm is installed and the daemon is running. The slurm backend is functional
-and tested, but improvements can still be made (help wanted). The airflow
-backend similarly requires a configured airflow server, but is not fully
-functional or tested (contributions to make airflow work / easier are wanted!).
-   
-
-
 .. |Pypi| image:: https://img.shields.io/pypi/v/cmd_queue.svg
    :target: https://pypi.python.org/pypi/cmd_queue
 
 .. |Downloads| image:: https://img.shields.io/pypi/dm/cmd_queue.svg
    :target: https://pypistats.org/packages/cmd_queue
 
-.. |ReadTheDocs| image:: https://readthedocs.org/projects/cmd_queue/badge/?version=release
-    :target: https://cmd_queue.readthedocs.io/en/release/
+.. |ReadTheDocs| image:: https://readthedocs.org/projects/cmd-queue/badge/?version=release
+    :target: https://cmd-queue.readthedocs.io/en/release/
 
 .. # See: https://ci.appveyor.com/project/jon.crall/cmd_queue/settings/badges
 .. |Appveyor| image:: https://ci.appveyor.com/api/projects/status/py3s2d6tyfjc8lm3/branch/master?svg=true

{cmd_queue-0.1.12.dist-info → cmd_queue-0.1.14.dist-info}/RECORD

@@ -1,12 +1,12 @@
-cmd_queue/__init__.py,sha256=GXwocyEx1v1UOQIYLyzMHre_jG1IUsGHeArJTQBbZUM,14386
-cmd_queue/__main__.py,sha256=yNgCJADmln3LQxFxBEUcI_EuiqV2NsxQ0F2rYICTYMQ,7170
+cmd_queue/__init__.py,sha256=f2dntM8omHeqviSQ7lmSbsKk-rSam1fHwROXoZC8oNI,14870
+cmd_queue/__main__.py,sha256=zgHtvXbQcpw3R2BXa5EF0BREeBdWJ4INPfSd0kYJ7WU,7192
 cmd_queue/airflow_queue.py,sha256=zJ86GckW5yXVcy9gCK7dmRlezs7H5ABpUWK7FeVtJSk,10414
 cmd_queue/base_queue.py,sha256=HQQAqioNn97UCh2qTSumww2zewdip6KEaSkg-FDcucg,13824
-cmd_queue/cli_boilerplate.py,sha256=jRL4PQkgCQShzmUj4xZniv5yROKqNjnvWuue-bzV3yY,8430
+cmd_queue/cli_boilerplate.py,sha256=kGeStl6F5uY_sGQoYiYgHfpLN9Di9YymPJdEkS_9Kpc,8430
 cmd_queue/monitor_app.py,sha256=m0cROUkBvTfpArjdZBfDvBMYnVJKBU_gDiW4HqENm0o,4641
 cmd_queue/serial_queue.py,sha256=IugMzDoNAT7DaJGTtSlSLqG_T5CApkOybM9SbmOHM2U,26336
 cmd_queue/slurm_queue.py,sha256=ZnKc4GCVaaTRY6Uj2-9gUzwdpddRt9yFzRuXgbaCAMA,23641
-cmd_queue/tmux_queue.py,sha256=sbKooYZO-o8uXMqPteqOE3n98OVg25VwDuiul6ZvwFU,40464
+cmd_queue/tmux_queue.py,sha256=GcY_mnRXADAvAmZLtMIauvUe7fXGgLpr1e4noVIJx3c,40427
 cmd_queue/util/__init__.py,sha256=QzlId47F1lzuLyeIIJoPGZR3b9ljJl50sFEcJzYWrB8,1450
 cmd_queue/util/richer.py,sha256=OPXpoXYgdhv_49sskF9FMwj6KCSiNcBo1kNhbuj6hoU,3373
 cmd_queue/util/texter.py,sha256=I6Fcrk0Yp3QTgbxF8cn8vSZWIkkytGBk3ZD4c1diMM0,2264
@@ -16,9 +16,9 @@ cmd_queue/util/util_networkx.py,sha256=5CV8MCAokAnHneC-_Sg2sQ4MOxkj3KYpRNn_t8kya
 cmd_queue/util/util_tags.py,sha256=Hlxpel759AduUy3Wdq3fCe-XKjxqBUFCLOr3Mw2bzsw,719
 cmd_queue/util/util_tmux.py,sha256=vC4xeYZCV8uVAp363zD24ROyKqUAdCynIULNJ8UgLQE,1078
 cmd_queue/util/util_yaml.py,sha256=uWyENCbjBt-tx7LtkI_JHWnBSe3jbx8d4H7JT1giH0k,9281
-cmd_queue-0.1.12.dist-info/LICENSE,sha256=o6jcFk_bwjiPUz6vHK0Ju7RwbFp9eXMwAS2BDnwER-4,11343
-cmd_queue-0.1.12.dist-info/METADATA,sha256=oEx0YcGBRyhytsC4QNw4tcKSnkNskaX3IWCgV2WbfVg,28733
-cmd_queue-0.1.12.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
-cmd_queue-0.1.12.dist-info/entry_points.txt,sha256=HDxa1dTf0Dne-a-QeDu7cWZVYRyEtCvqaau0GCCMEyw,54
-cmd_queue-0.1.12.dist-info/top_level.txt,sha256=C2JVEsOZsjnMx3jIAWhIQGWAXjGs-hyBzzjkOIm7qW8,10
-cmd_queue-0.1.12.dist-info/RECORD,,
+cmd_queue-0.1.14.dist-info/LICENSE,sha256=o6jcFk_bwjiPUz6vHK0Ju7RwbFp9eXMwAS2BDnwER-4,11343
+cmd_queue-0.1.14.dist-info/METADATA,sha256=v8Ji1ff1gQkXkRrahEB5C185oy9Q7aURFXTkhzq3gQw,29128
+cmd_queue-0.1.14.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
+cmd_queue-0.1.14.dist-info/entry_points.txt,sha256=HDxa1dTf0Dne-a-QeDu7cWZVYRyEtCvqaau0GCCMEyw,54
+cmd_queue-0.1.14.dist-info/top_level.txt,sha256=C2JVEsOZsjnMx3jIAWhIQGWAXjGs-hyBzzjkOIm7qW8,10
+cmd_queue-0.1.14.dist-info/RECORD,,