cmd-queue 0.1.19__py3-none-any.whl → 0.2.0__py3-none-any.whl

cmd_queue/__init__.py

@@ -306,7 +306,7 @@ Example:
 __mkinit__ = """
 mkinit -m cmd_queue
 """
-__version__ = '0.1.19'
+__version__ = '0.2.0'
 
 
 __submodules__ = {

cmd_queue/__main__.pyi

@@ -0,0 +1,3 @@
+from _typeshed import Incomplete
+
+main: Incomplete

cmd_queue/airflow_queue.pyi

@@ -0,0 +1,84 @@
+from _typeshed import Incomplete
+from cmd_queue import base_queue
+
+
+class AirflowJob(base_queue.Job):
+    unused_kwargs: Incomplete
+    command: Incomplete
+    name: Incomplete
+    output_fpath: Incomplete
+    depends: Incomplete
+    cpus: Incomplete
+    gpus: Incomplete
+    mem: Incomplete
+    begin: Incomplete
+    shell: Incomplete
+
+    def __init__(self,
+                 command,
+                 name: Incomplete | None = ...,
+                 output_fpath: Incomplete | None = ...,
+                 depends: Incomplete | None = ...,
+                 partition: Incomplete | None = ...,
+                 cpus: Incomplete | None = ...,
+                 gpus: Incomplete | None = ...,
+                 mem: Incomplete | None = ...,
+                 begin: Incomplete | None = ...,
+                 shell: Incomplete | None = ...,
+                 **kwargs) -> None:
+        ...
+
+    def __nice__(self):
+        ...
+
+    def finalize_text(self):
+        ...
+
+
+class AirflowQueue(base_queue.Queue):
+    jobs: Incomplete
+    name: Incomplete
+    unused_kwargs: Incomplete
+    queue_id: Incomplete
+    dpath: Incomplete
+    log_dpath: Incomplete
+    fpath: Incomplete
+    shell: Incomplete
+    header_commands: Incomplete
+    all_depends: Incomplete
+
+    def __init__(self,
+                 name: Incomplete | None = ...,
+                 shell: Incomplete | None = ...,
+                 **kwargs) -> None:
+        ...
+
+    @classmethod
+    def is_available(cls):
+        ...
+
+    def run(self, block: bool = ..., system: bool = ...) -> None:
+        ...
+
+    def finalize_text(self):
+        ...
+
+    def submit(self, command, **kwargs):
+        ...
+
+    def print_commands(self,
+                       with_status: bool = ...,
+                       with_gaurds: bool = ...,
+                       with_locks: int = ...,
+                       exclude_tags: Incomplete | None = ...,
+                       style: str = ...,
+                       with_rich: Incomplete | None = ...,
+                       colors: int = ...,
+                       **kwargs) -> None:
+        ...
+
+    rprint = print_commands
+
+
+def demo() -> None:
+    ...

cmd_queue/base_queue.py

@@ -135,6 +135,11 @@ class Queue(ub.NiceRepr):
             name = kwargs.get('name', None)
             if name is None:
                 name = kwargs['name'] = self.name + '-job-{}'.format(self.num_real_jobs)
+
+            # TODO: make sure name is path safe.
+            if ':' in name:
+                raise ValueError('Name must be path-safe')
+
             if self.all_depends:
                 depends = kwargs.get('depends', None)
                 if depends is None:

cmd_queue/base_queue.pyi

@@ -0,0 +1,86 @@
+from typing import List
+import ubelt as ub
+from _typeshed import Incomplete
+
+
+class DuplicateJobError(KeyError):
+    ...
+
+
+class UnknownBackendError(KeyError):
+    ...
+
+
+class Job(ub.NiceRepr):
+    name: Incomplete
+    command: Incomplete
+    depends: Incomplete
+    kwargs: Incomplete
+
+    def __init__(self,
+                 command: Incomplete | None = ...,
+                 name: Incomplete | None = ...,
+                 depends: Incomplete | None = ...,
+                 **kwargs) -> None:
+        ...
+
+    def __nice__(self):
+        ...
+
+
+class Queue(ub.NiceRepr):
+    num_real_jobs: int
+    all_depends: Incomplete
+    named_jobs: Incomplete
+
+    def __init__(self) -> None:
+        ...
+
+    def change_backend(self, backend, **kwargs):
+        ...
+
+    def __len__(self):
+        ...
+
+    def sync(self) -> Queue:
+        ...
+
+    def write(self):
+        ...
+
+    def submit(self, command, **kwargs):
+        ...
+
+    @classmethod
+    def available_backends(cls):
+        ...
+
+    @classmethod
+    def create(cls, backend: str = ..., **kwargs):
+        ...
+
+    def write_network_text(self,
+                           reduced: bool = ...,
+                           rich: str = ...,
+                           vertical_chains: bool = ...) -> None:
+        ...
+
+    def print_commands(self,
+                       with_status: bool = False,
+                       with_gaurds: bool = False,
+                       with_locks: bool | int = 1,
+                       exclude_tags: List[str] | None = None,
+                       style: str = 'colors',
+                       **kwargs) -> None:
+        ...
+
+    def rprint(self, **kwargs) -> None:
+        ...
+
+    def print_graph(self,
+                    reduced: bool = True,
+                    vertical_chains: bool = ...) -> None:
+        ...
+
+    def monitor(self) -> None:
+        ...

cmd_queue/cli_boilerplate.pyi

@@ -0,0 +1,32 @@
+from typing import Dict
+import scriptconfig as scfg
+from _typeshed import Incomplete
+
+import cmd_queue
+
+__docstubs__: str
+
+
+class CMDQueueConfig(scfg.DataConfig):
+    run: Incomplete
+    backend: Incomplete
+    queue_name: Incomplete
+    print_commands: Incomplete
+    print_queue: Incomplete
+    with_textual: Incomplete
+    other_session_handler: Incomplete
+    virtualenv_cmd: Incomplete
+    tmux_workers: Incomplete
+    slurm_options: Incomplete
+
+    def __post_init__(self) -> None:
+        ...
+
+    def create_queue(config, **kwargs) -> cmd_queue.Queue:
+        ...
+
+    def run_queue(config,
+                  queue: cmd_queue.Queue,
+                  print_kwargs: None | Dict = None,
+                  **kwargs) -> None:
+        ...

cmd_queue/main.py

@@ -94,12 +94,15 @@ class CommonShowRun(CommonConfig):
 
     backend = scfg.Value('tmux', help='the execution backend to use', choices=['tmux', 'slurm', 'serial', 'airflow'])
 
+    gpus = scfg.Value(None, help='a comma separated list of the gpu numbers to spread across. tmux backend only.')
+
     def _build_queue(config):
         import cmd_queue
         import json
         queue = cmd_queue.Queue.create(size=max(1, config['workers']),
                                        backend=config['backend'],
-                                       name=config['qname'])
+                                       name=config['qname'],
+                                       gpus=config['gpus'])
         # Run a new CLI queue
         data = json.loads(config.cli_queue_fpath.read_text())
         print('data = {}'.format(ub.urepr(data, nl=1)))

cmd_queue/main.pyi

@@ -0,0 +1,66 @@
+import scriptconfig as scfg
+from _typeshed import Incomplete
+
+
+class CommonConfig(scfg.DataConfig):
+    qname: Incomplete
+    dpath: Incomplete
+
+    def __post_init__(config) -> None:
+        ...
+
+    @classmethod
+    def main(cls, cmdline: int = ..., **kwargs) -> None:
+        ...
+
+
+class CommonShowRun(CommonConfig):
+    workers: Incomplete
+    backend: Incomplete
+
+
+class CmdQueueCLI(scfg.ModalCLI):
+
+    class cleanup(CommonConfig):
+        yes: Incomplete
+        __command__: str
+
+        def run(config) -> None:
+            ...
+
+    class run(CommonShowRun):
+        __command__: str
+
+        def run(config) -> None:
+            ...
+
+    class show(CommonShowRun):
+        __command__: str
+
+        def run(config) -> None:
+            ...
+
+    class submit(CommonConfig):
+        jobname: Incomplete
+        depends: Incomplete
+        command: Incomplete
+        __command__: str
+
+        def run(config) -> None:
+            ...
+
+    class new(CommonConfig):
+        __command__: str
+        header: Incomplete
+
+        def run(config) -> None:
+            ...
+
+    class list(CommonConfig):
+        __command__: str
+
+        def run(config) -> None:
+            ...
+
+
+main: Incomplete

cmd_queue/monitor_app.pyi

@@ -0,0 +1,41 @@
+from _typeshed import Incomplete
+from cmd_queue.util.textual_extensions import InstanceRunnableApp
+from textual.widget import Widget
+
+
+class JobTable(Widget):
+    table_fn: Incomplete
+
+    def __init__(self, table_fn: Incomplete | None = ..., **kwargs) -> None:
+        ...
+
+    def on_mount(self) -> None:
+        ...
+
+    def render(self):
+        ...
+
+
+class CmdQueueMonitorApp(InstanceRunnableApp):
+    job_table: Incomplete
+    kill_fn: Incomplete
+    graceful_exit: bool
+
+    def __init__(self,
+                 table_fn,
+                 kill_fn: Incomplete | None = ...,
+                 **kwargs) -> None:
+        ...
+
+    @classmethod
+    def demo(CmdQueueMonitorApp):
+        ...
+
+    async def on_load(self, event) -> None:
+        ...
+
+    async def action_quit(self) -> None:
+        ...
+
+    async def on_mount(self, event) -> None:
+        ...

cmd_queue/serial_queue.pyi

@@ -0,0 +1,124 @@
+from typing import List
+from os import PathLike
+from _typeshed import Incomplete
+from cmd_queue import base_queue
+
+
+def indent(text: str, prefix: str = '    '):
+    ...
+
+
+class BashJob(base_queue.Job):
+    name: str
+    pathid: str
+    command: str
+    depends: List[BashJob] | None
+    bookkeeper: bool
+    info_dpath: PathLike | None
+    log: bool
+    tags: List[str] | str | None
+    allow_indent: bool
+    kwargs: Incomplete
+    pass_fpath: Incomplete
+    fail_fpath: Incomplete
+    stat_fpath: Incomplete
+    log_fpath: Incomplete
+
+    def __init__(self,
+                 command,
+                 name: Incomplete | None = ...,
+                 depends: Incomplete | None = ...,
+                 gpus: Incomplete | None = ...,
+                 cpus: Incomplete | None = ...,
+                 mem: Incomplete | None = ...,
+                 bookkeeper: int = ...,
+                 info_dpath: Incomplete | None = ...,
+                 log: bool = ...,
+                 tags: Incomplete | None = ...,
+                 allow_indent: bool = ...,
+                 **kwargs) -> None:
+        ...
+
+    def finalize_text(self,
+                      with_status: bool = ...,
+                      with_gaurds: bool = ...,
+                      conditionals: Incomplete | None = ...,
+                      **kwargs):
+        ...
+
+    def print_commands(self,
+                       with_status: bool = False,
+                       with_gaurds: bool = False,
+                       with_rich: Incomplete | None = ...,
+                       style: str = 'colors',
+                       **kwargs) -> None:
+        ...
+
+
+class SerialQueue(base_queue.Queue):
+    name: Incomplete
+    rootid: Incomplete
+    dpath: Incomplete
+    unused_kwargs: Incomplete
+    fpath: Incomplete
+    state_fpath: Incomplete
+    environ: Incomplete
+    header: str
+    header_commands: Incomplete
+    jobs: Incomplete
+    cwd: Incomplete
+    job_info_dpath: Incomplete
+
+    def __init__(self,
+                 name: str = ...,
+                 dpath: Incomplete | None = ...,
+                 rootid: Incomplete | None = ...,
+                 environ: Incomplete | None = ...,
+                 cwd: Incomplete | None = ...,
+                 **kwargs) -> None:
+        ...
+
+    @property
+    def pathid(self):
+        ...
+
+    def __nice__(self):
+        ...
+
+    @classmethod
+    def is_available(cls):
+        ...
+
+    def order_jobs(self) -> None:
+        ...
+
+    def finalize_text(self,
+                      with_status: bool = ...,
+                      with_gaurds: bool = ...,
+                      with_locks: bool = ...,
+                      exclude_tags: Incomplete | None = ...):
+        ...
+
+    def add_header_command(self, command) -> None:
+        ...
+
+    def print_commands(self, *args, **kwargs):
+        ...
+
+    rprint = print_commands
+
+    def run(self,
+            block: bool = ...,
+            system: bool = ...,
+            shell: int = ...,
+            capture: bool = ...,
+            mode: str = ...,
+            verbose: int = ...,
+            **kw) -> None:
+        ...
+
+    def job_details(self) -> None:
+        ...
+
+    def read_state(self):
+        ...

cmd_queue/slurm_queue.py

@@ -41,23 +41,51 @@ from cmd_queue import base_queue  # NOQA
 from cmd_queue.util import util_tags
 
 
-def _coerce_mem(mem):
+try:
+    from functools import cache  # Python 3.9+ only
+except ImportError:
+    from ubelt import memoize as cache
+
+
+@cache
+def _unit_registery():
+    import sys
+    if sys.version_info[0:2] == (3, 9):
+        # backwards compatability support for numpy 2.0 and pint on cp39
+        try:
+            import numpy as np
+        except ImportError:
+            ...
+        else:
+            if not np.__version__.startswith('1.'):
+                np.cumproduct = np.cumprod
+    import pint
+    reg = pint.UnitRegistry()
+    return reg
+
+
+def _coerce_mem_megabytes(mem):
     """
+    Transform input into an integer representing amount of megabytes.
+
     Args:
         mem (int | str): integer number of megabytes or a parseable string
 
+    Returns:
+        int: number of megabytes
+
     Example:
+        >>> # xdoctest: +REQUIRES(module:pint)
         >>> from cmd_queue.slurm_queue import *  # NOQA
-        >>> print(_coerce_mem(30602))
-        >>> print(_coerce_mem('4GB'))
-        >>> print(_coerce_mem('32GB'))
-        >>> print(_coerce_mem('300000000 bytes'))
+        >>> print(_coerce_mem_megabytes(30602))
+        >>> print(_coerce_mem_megabytes('4GB'))
+        >>> print(_coerce_mem_megabytes('32GB'))
+        >>> print(_coerce_mem_megabytes('300000000 bytes'))
     """
     if isinstance(mem, int):
         assert mem > 0
     elif isinstance(mem, str):
-        import pint
-        reg = pint.UnitRegistry()
+        reg = _unit_registery()
         mem = reg.parse_expression(mem)
         mem = int(mem.to('megabytes').m)
     else:
@@ -190,6 +218,7 @@ class SlurmJob(base_queue.Job):
     Represents a slurm job that hasn't been submitted yet
 
     Example:
+        >>> # xdoctest: +REQUIRES(module:pint)
         >>> from cmd_queue.slurm_queue import *  # NOQA
         >>> self = SlurmJob('python -c print("hello world")', 'hi', cpus=5, gpus=1, mem='10GB')
         >>> command = self._build_sbatch_args()
@@ -245,7 +274,7 @@ class SlurmJob(base_queue.Job):
         if self.cpus:
             sbatch_args.append(f'--cpus-per-task={self.cpus}')
         if self.mem:
-            mem = _coerce_mem(self.mem)
+            mem = _coerce_mem_megabytes(self.mem)
             sbatch_args.append(f'--mem={mem}')
         if self.gpus and 'gres' not in self._sbatch_kvargs:
             ub.schedule_deprecation(

cmd_queue/slurm_queue.pyi

@@ -0,0 +1,97 @@
+from _typeshed import Incomplete
+from cmd_queue import base_queue
+
+__dev__: str
+SLURM_SBATCH_KVARGS: Incomplete
+SLURM_SBATCH_FLAGS: Incomplete
+
+
+class SlurmJob(base_queue.Job):
+    unused_kwargs: Incomplete
+    command: Incomplete
+    name: Incomplete
+    output_fpath: Incomplete
+    depends: Incomplete
+    cpus: Incomplete
+    gpus: Incomplete
+    mem: Incomplete
+    begin: Incomplete
+    shell: Incomplete
+    tags: Incomplete
+    jobid: Incomplete
+
+    def __init__(self,
+                 command,
+                 name: Incomplete | None = ...,
+                 output_fpath: Incomplete | None = ...,
+                 depends: Incomplete | None = ...,
+                 cpus: Incomplete | None = ...,
+                 gpus: Incomplete | None = ...,
+                 mem: Incomplete | None = ...,
+                 begin: Incomplete | None = ...,
+                 shell: Incomplete | None = ...,
+                 tags: Incomplete | None = ...,
+                 **kwargs) -> None:
+        ...
+
+    def __nice__(self):
+        ...
+
+
+class SlurmQueue(base_queue.Queue):
+    jobs: Incomplete
+    unused_kwargs: Incomplete
+    queue_id: Incomplete
+    dpath: Incomplete
+    log_dpath: Incomplete
+    fpath: Incomplete
+    shell: Incomplete
+    header_commands: Incomplete
+    all_depends: Incomplete
+
+    def __init__(self,
+                 name: Incomplete | None = ...,
+                 shell: Incomplete | None = ...,
+                 **kwargs) -> None:
+        ...
+
+    def __nice__(self):
+        ...
+
+    @classmethod
+    def is_available(cls):
+        ...
+
+    def submit(self, command, **kwargs):
+        ...
+
+    def add_header_command(self, command) -> None:
+        ...
+
+    def order_jobs(self):
+        ...
+
+    jobname_to_varname: Incomplete
+
+    def finalize_text(self, exclude_tags: Incomplete | None = ..., **kwargs):
+        ...
+
+    def run(self, block: bool = ..., system: bool = ..., **kw):
+        ...
+
+    def monitor(self, refresh_rate: float = ...):
+        ...
+
+    def kill(self) -> None:
+        ...
+
+    def read_state(self):
+        ...
+
+    def print_commands(self, *args, **kwargs):
+        ...
+
+    rprint = print_commands
+
+
+SLURM_NOTES: str

cmd_queue/tmux_queue.py

@@ -1055,4 +1055,24 @@ if 0:
     tmux kill-session -t my_session_id
 
     tmux new-session -d -s my_session_id -e "MYVAR1" -- "bash"
+
+
+
+    #### to start a tmux session with 4 panes
+    tmux new-session -d -s my_session_id1 "bash"
+    tmux send -t my_session_id1 "tmux split-window -h -t 0" Enter
+    tmux send -t my_session_id1 "tmux split-window -v -t 0" Enter
+    tmux send -t my_session_id1 "tmux split-window -v -t 2" Enter
+
+    # Now send a command to each pane
+    tmux send -t my_session_id1 "tmux select-pane -t 0" Enter
+    tmux send -t my_session_id1 "echo pane0" Enter
+    tmux send -t my_session_id1 "tmux select-pane -t 1" Enter
+    tmux send -t my_session_id1 "echo pane1" Enter
+    tmux send -t my_session_id1 "tmux select-pane -t 2" Enter
+    tmux send -t my_session_id1 "echo pane2" Enter
+    tmux send -t my_session_id1 "tmux select-pane -t 3" Enter
+    tmux send -t my_session_id1 "echo pane3" Enter
+
+
     """