cmd-queue 0.1.20__py3-none-any.whl → 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cmd-queue might be problematic. Click here for more details.
- cmd_queue/__init__.py +1 -1
- cmd_queue/base_queue.py +5 -0
- cmd_queue/main.py +4 -1
- cmd_queue/slurm_queue.py +37 -8
- cmd_queue/tmux_queue.py +20 -0
- {cmd_queue-0.1.20.dist-info → cmd_queue-0.2.0.dist-info}/METADATA +90 -23
- {cmd_queue-0.1.20.dist-info → cmd_queue-0.2.0.dist-info}/RECORD +11 -11
- {cmd_queue-0.1.20.dist-info → cmd_queue-0.2.0.dist-info}/WHEEL +1 -1
- {cmd_queue-0.1.20.dist-info → cmd_queue-0.2.0.dist-info}/LICENSE +0 -0
- {cmd_queue-0.1.20.dist-info → cmd_queue-0.2.0.dist-info}/entry_points.txt +0 -0
- {cmd_queue-0.1.20.dist-info → cmd_queue-0.2.0.dist-info}/top_level.txt +0 -0
cmd_queue/__init__.py
CHANGED
cmd_queue/base_queue.py
CHANGED
|
@@ -135,6 +135,11 @@ class Queue(ub.NiceRepr):
|
|
|
135
135
|
name = kwargs.get('name', None)
|
|
136
136
|
if name is None:
|
|
137
137
|
name = kwargs['name'] = self.name + '-job-{}'.format(self.num_real_jobs)
|
|
138
|
+
|
|
139
|
+
# TODO: make sure name is path safe.
|
|
140
|
+
if ':' in name:
|
|
141
|
+
raise ValueError('Name must be path-safe')
|
|
142
|
+
|
|
138
143
|
if self.all_depends:
|
|
139
144
|
depends = kwargs.get('depends', None)
|
|
140
145
|
if depends is None:
|
cmd_queue/main.py
CHANGED
|
@@ -94,12 +94,15 @@ class CommonShowRun(CommonConfig):
|
|
|
94
94
|
|
|
95
95
|
backend = scfg.Value('tmux', help='the execution backend to use', choices=['tmux', 'slurm', 'serial', 'airflow'])
|
|
96
96
|
|
|
97
|
+
gpus = scfg.Value(None, help='a comma separated list of the gpu numbers to spread across. tmux backend only.')
|
|
98
|
+
|
|
97
99
|
def _build_queue(config):
|
|
98
100
|
import cmd_queue
|
|
99
101
|
import json
|
|
100
102
|
queue = cmd_queue.Queue.create(size=max(1, config['workers']),
|
|
101
103
|
backend=config['backend'],
|
|
102
|
-
name=config['qname']
|
|
104
|
+
name=config['qname'],
|
|
105
|
+
gpus=config['gpus'])
|
|
103
106
|
# Run a new CLI queue
|
|
104
107
|
data = json.loads(config.cli_queue_fpath.read_text())
|
|
105
108
|
print('data = {}'.format(ub.urepr(data, nl=1)))
|
cmd_queue/slurm_queue.py
CHANGED
|
@@ -41,23 +41,51 @@ from cmd_queue import base_queue # NOQA
|
|
|
41
41
|
from cmd_queue.util import util_tags
|
|
42
42
|
|
|
43
43
|
|
|
44
|
-
|
|
44
|
+
try:
|
|
45
|
+
from functools import cache # Python 3.9+ only
|
|
46
|
+
except ImportError:
|
|
47
|
+
from ubelt import memoize as cache
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
@cache
|
|
51
|
+
def _unit_registery():
|
|
52
|
+
import sys
|
|
53
|
+
if sys.version_info[0:2] == (3, 9):
|
|
54
|
+
# backwards compatability support for numpy 2.0 and pint on cp39
|
|
55
|
+
try:
|
|
56
|
+
import numpy as np
|
|
57
|
+
except ImportError:
|
|
58
|
+
...
|
|
59
|
+
else:
|
|
60
|
+
if not np.__version__.startswith('1.'):
|
|
61
|
+
np.cumproduct = np.cumprod
|
|
62
|
+
import pint
|
|
63
|
+
reg = pint.UnitRegistry()
|
|
64
|
+
return reg
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _coerce_mem_megabytes(mem):
|
|
45
68
|
"""
|
|
69
|
+
Transform input into an integer representing amount of megabytes.
|
|
70
|
+
|
|
46
71
|
Args:
|
|
47
72
|
mem (int | str): integer number of megabytes or a parseable string
|
|
48
73
|
|
|
74
|
+
Returns:
|
|
75
|
+
int: number of megabytes
|
|
76
|
+
|
|
49
77
|
Example:
|
|
78
|
+
>>> # xdoctest: +REQUIRES(module:pint)
|
|
50
79
|
>>> from cmd_queue.slurm_queue import * # NOQA
|
|
51
|
-
>>> print(
|
|
52
|
-
>>> print(
|
|
53
|
-
>>> print(
|
|
54
|
-
>>> print(
|
|
80
|
+
>>> print(_coerce_mem_megabytes(30602))
|
|
81
|
+
>>> print(_coerce_mem_megabytes('4GB'))
|
|
82
|
+
>>> print(_coerce_mem_megabytes('32GB'))
|
|
83
|
+
>>> print(_coerce_mem_megabytes('300000000 bytes'))
|
|
55
84
|
"""
|
|
56
85
|
if isinstance(mem, int):
|
|
57
86
|
assert mem > 0
|
|
58
87
|
elif isinstance(mem, str):
|
|
59
|
-
|
|
60
|
-
reg = pint.UnitRegistry()
|
|
88
|
+
reg = _unit_registery()
|
|
61
89
|
mem = reg.parse_expression(mem)
|
|
62
90
|
mem = int(mem.to('megabytes').m)
|
|
63
91
|
else:
|
|
@@ -190,6 +218,7 @@ class SlurmJob(base_queue.Job):
|
|
|
190
218
|
Represents a slurm job that hasn't been submitted yet
|
|
191
219
|
|
|
192
220
|
Example:
|
|
221
|
+
>>> # xdoctest: +REQUIRES(module:pint)
|
|
193
222
|
>>> from cmd_queue.slurm_queue import * # NOQA
|
|
194
223
|
>>> self = SlurmJob('python -c print("hello world")', 'hi', cpus=5, gpus=1, mem='10GB')
|
|
195
224
|
>>> command = self._build_sbatch_args()
|
|
@@ -245,7 +274,7 @@ class SlurmJob(base_queue.Job):
|
|
|
245
274
|
if self.cpus:
|
|
246
275
|
sbatch_args.append(f'--cpus-per-task={self.cpus}')
|
|
247
276
|
if self.mem:
|
|
248
|
-
mem =
|
|
277
|
+
mem = _coerce_mem_megabytes(self.mem)
|
|
249
278
|
sbatch_args.append(f'--mem={mem}')
|
|
250
279
|
if self.gpus and 'gres' not in self._sbatch_kvargs:
|
|
251
280
|
ub.schedule_deprecation(
|
cmd_queue/tmux_queue.py
CHANGED
|
@@ -1055,4 +1055,24 @@ if 0:
|
|
|
1055
1055
|
tmux kill-session -t my_session_id
|
|
1056
1056
|
|
|
1057
1057
|
tmux new-session -d -s my_session_id -e "MYVAR1" -- "bash"
|
|
1058
|
+
|
|
1059
|
+
|
|
1060
|
+
|
|
1061
|
+
#### to start a tmux session with 4 panes
|
|
1062
|
+
tmux new-session -d -s my_session_id1 "bash"
|
|
1063
|
+
tmux send -t my_session_id1 "tmux split-window -h -t 0" Enter
|
|
1064
|
+
tmux send -t my_session_id1 "tmux split-window -v -t 0" Enter
|
|
1065
|
+
tmux send -t my_session_id1 "tmux split-window -v -t 2" Enter
|
|
1066
|
+
|
|
1067
|
+
# Now send a command to each pane
|
|
1068
|
+
tmux send -t my_session_id1 "tmux select-pane -t 0" Enter
|
|
1069
|
+
tmux send -t my_session_id1 "echo pane0" Enter
|
|
1070
|
+
tmux send -t my_session_id1 "tmux select-pane -t 1" Enter
|
|
1071
|
+
tmux send -t my_session_id1 "echo pane1" Enter
|
|
1072
|
+
tmux send -t my_session_id1 "tmux select-pane -t 2" Enter
|
|
1073
|
+
tmux send -t my_session_id1 "echo pane2" Enter
|
|
1074
|
+
tmux send -t my_session_id1 "tmux select-pane -t 3" Enter
|
|
1075
|
+
tmux send -t my_session_id1 "echo pane3" Enter
|
|
1076
|
+
|
|
1077
|
+
|
|
1058
1078
|
"""
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: cmd_queue
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: The cmd_queue module for a DAG of bash commands
|
|
5
5
|
Home-page: https://gitlab.kitware.com/computer-vision/cmd_queue
|
|
6
6
|
Author: Kitware Inc., Jon Crall
|
|
@@ -11,14 +11,12 @@ Classifier: Intended Audience :: Developers
|
|
|
11
11
|
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
12
12
|
Classifier: Topic :: Utilities
|
|
13
13
|
Classifier: License :: OSI Approved :: Apache Software License
|
|
14
|
-
Classifier: Programming Language :: Python :: 3.6
|
|
15
|
-
Classifier: Programming Language :: Python :: 3.7
|
|
16
14
|
Classifier: Programming Language :: Python :: 3.8
|
|
17
15
|
Classifier: Programming Language :: Python :: 3.9
|
|
18
16
|
Classifier: Programming Language :: Python :: 3.10
|
|
19
17
|
Classifier: Programming Language :: Python :: 3.11
|
|
20
18
|
Classifier: Programming Language :: Python :: 3.12
|
|
21
|
-
Requires-Python: >=3.
|
|
19
|
+
Requires-Python: >=3.8
|
|
22
20
|
Description-Content-Type: text/x-rst
|
|
23
21
|
License-File: LICENSE
|
|
24
22
|
Requires-Dist: ubelt >=1.3.0
|
|
@@ -34,7 +32,6 @@ Requires-Dist: networkx >=2.7 ; python_version < "3.11" and python_version >= "3
|
|
|
34
32
|
Requires-Dist: numpy >=1.23.2 ; python_version < "3.12" and python_version >= "3.11"
|
|
35
33
|
Requires-Dist: pandas >=1.5.0 ; python_version < "3.12" and python_version >= "3.11"
|
|
36
34
|
Requires-Dist: pandas >=1.1.5 ; python_version < "3.7" and python_version >= "3.6"
|
|
37
|
-
Requires-Dist: pint >=0.10 ; python_version < "3.7" and python_version >= "3.6"
|
|
38
35
|
Requires-Dist: networkx <=2.5.1,>=2.5.1 ; python_version < "3.7.0" and python_version >= "3.6.0"
|
|
39
36
|
Requires-Dist: networkx >=2.6.2 ; python_version < "3.8" and python_version >= "3.7"
|
|
40
37
|
Requires-Dist: pandas >=1.3.5 ; python_version < "3.8" and python_version >= "3.7"
|
|
@@ -42,7 +39,6 @@ Requires-Dist: pandas >=1.4.0 ; python_version < "3.9" and python_version >= "3.
|
|
|
42
39
|
Requires-Dist: networkx >=2.8 ; python_version < "4.0" and python_version >= "3.11"
|
|
43
40
|
Requires-Dist: numpy >=1.26.0 ; python_version < "4.0" and python_version >= "3.12"
|
|
44
41
|
Requires-Dist: pandas >=2.1.1 ; python_version < "4.0" and python_version >= "3.12"
|
|
45
|
-
Requires-Dist: pint >=0.18 ; python_version >= "3.7"
|
|
46
42
|
Provides-Extra: all
|
|
47
43
|
Requires-Dist: ubelt >=1.3.0 ; extra == 'all'
|
|
48
44
|
Requires-Dist: rich >=12.5.1 ; extra == 'all'
|
|
@@ -66,9 +62,9 @@ Requires-Dist: networkx ==2.7 ; (python_version < "3.11" and python_version >= "
|
|
|
66
62
|
Requires-Dist: numpy ==1.23.2 ; (python_version < "3.12" and python_version >= "3.11") and extra == 'all-strict'
|
|
67
63
|
Requires-Dist: pandas ==1.5.0 ; (python_version < "3.12" and python_version >= "3.11") and extra == 'all-strict'
|
|
68
64
|
Requires-Dist: pandas ==1.1.5 ; (python_version < "3.7" and python_version >= "3.6") and extra == 'all-strict'
|
|
69
|
-
Requires-Dist: pint ==0.10 ; (python_version < "3.7" and python_version >= "3.6") and extra == 'all-strict'
|
|
70
65
|
Requires-Dist: pytest ==6.2.0 ; (python_version < "3.7" and python_version >= "3.6") and extra == 'all-strict'
|
|
71
66
|
Requires-Dist: coverage ==6.1.1 ; (python_version < "3.7" and python_version >= "3.6") and extra == 'all-strict'
|
|
67
|
+
Requires-Dist: pint ==0.10 ; (python_version < "3.7" and python_version >= "3.6") and extra == 'all-strict'
|
|
72
68
|
Requires-Dist: networkx <=2.5.1,==2.5.1 ; (python_version < "3.7.0" and python_version >= "3.6.0") and extra == 'all-strict'
|
|
73
69
|
Requires-Dist: networkx ==2.6.2 ; (python_version < "3.8" and python_version >= "3.7") and extra == 'all-strict'
|
|
74
70
|
Requires-Dist: pandas ==1.3.5 ; (python_version < "3.8" and python_version >= "3.7") and extra == 'all-strict'
|
|
@@ -80,9 +76,9 @@ Requires-Dist: numpy ==1.26.0 ; (python_version < "4.0" and python_version >= "3
|
|
|
80
76
|
Requires-Dist: pandas ==2.1.1 ; (python_version < "4.0" and python_version >= "3.12") and extra == 'all-strict'
|
|
81
77
|
Requires-Dist: coverage ==6.1.1 ; (python_version >= "3.10") and extra == 'all-strict'
|
|
82
78
|
Requires-Dist: pytest-cov ==3.0.0 ; (python_version >= "3.6.0") and extra == 'all-strict'
|
|
83
|
-
Requires-Dist: pint ==0.18 ; (python_version >= "3.7") and extra == 'all-strict'
|
|
84
79
|
Requires-Dist: pytest ==7.1.0 ; (python_version >= "3.7") and extra == 'all-strict'
|
|
85
80
|
Requires-Dist: textual ==0.1.18 ; (python_version >= "3.7") and extra == 'all-strict'
|
|
81
|
+
Requires-Dist: pint ==0.18 ; (python_version >= "3.7") and extra == 'all-strict'
|
|
86
82
|
Requires-Dist: numpy >=1.19.3 ; (python_version < "3.10" and python_version >= "3.6.0") and extra == 'all'
|
|
87
83
|
Requires-Dist: pandas >=1.4.0 ; (python_version < "3.10" and python_version >= "3.9") and extra == 'all'
|
|
88
84
|
Requires-Dist: coverage >=5.3.1 ; (python_version < "3.10" and python_version >= "3.9") and extra == 'all'
|
|
@@ -92,9 +88,9 @@ Requires-Dist: networkx >=2.7 ; (python_version < "3.11" and python_version >= "
|
|
|
92
88
|
Requires-Dist: numpy >=1.23.2 ; (python_version < "3.12" and python_version >= "3.11") and extra == 'all'
|
|
93
89
|
Requires-Dist: pandas >=1.5.0 ; (python_version < "3.12" and python_version >= "3.11") and extra == 'all'
|
|
94
90
|
Requires-Dist: pandas >=1.1.5 ; (python_version < "3.7" and python_version >= "3.6") and extra == 'all'
|
|
95
|
-
Requires-Dist: pint >=0.10 ; (python_version < "3.7" and python_version >= "3.6") and extra == 'all'
|
|
96
91
|
Requires-Dist: pytest >=6.2.0 ; (python_version < "3.7" and python_version >= "3.6") and extra == 'all'
|
|
97
92
|
Requires-Dist: coverage >=6.1.1 ; (python_version < "3.7" and python_version >= "3.6") and extra == 'all'
|
|
93
|
+
Requires-Dist: pint >=0.10 ; (python_version < "3.7" and python_version >= "3.6") and extra == 'all'
|
|
98
94
|
Requires-Dist: networkx <=2.5.1,>=2.5.1 ; (python_version < "3.7.0" and python_version >= "3.6.0") and extra == 'all'
|
|
99
95
|
Requires-Dist: networkx >=2.6.2 ; (python_version < "3.8" and python_version >= "3.7") and extra == 'all'
|
|
100
96
|
Requires-Dist: pandas >=1.3.5 ; (python_version < "3.8" and python_version >= "3.7") and extra == 'all'
|
|
@@ -106,9 +102,9 @@ Requires-Dist: numpy >=1.26.0 ; (python_version < "4.0" and python_version >= "3
|
|
|
106
102
|
Requires-Dist: pandas >=2.1.1 ; (python_version < "4.0" and python_version >= "3.12") and extra == 'all'
|
|
107
103
|
Requires-Dist: coverage >=6.1.1 ; (python_version >= "3.10") and extra == 'all'
|
|
108
104
|
Requires-Dist: pytest-cov >=3.0.0 ; (python_version >= "3.6.0") and extra == 'all'
|
|
109
|
-
Requires-Dist: pint >=0.18 ; (python_version >= "3.7") and extra == 'all'
|
|
110
105
|
Requires-Dist: pytest >=7.1.0 ; (python_version >= "3.7") and extra == 'all'
|
|
111
106
|
Requires-Dist: textual >=0.1.18 ; (python_version >= "3.7") and extra == 'all'
|
|
107
|
+
Requires-Dist: pint >=0.18 ; (python_version >= "3.7") and extra == 'all'
|
|
112
108
|
Provides-Extra: docs
|
|
113
109
|
Requires-Dist: sphinx >=5.0.1 ; extra == 'docs'
|
|
114
110
|
Requires-Dist: sphinx-autobuild >=2021.3.14 ; extra == 'docs'
|
|
@@ -127,10 +123,18 @@ Requires-Dist: sphinx-autoapi ==1.8.4 ; extra == 'docs-strict'
|
|
|
127
123
|
Requires-Dist: Pygments ==2.9.0 ; extra == 'docs-strict'
|
|
128
124
|
Requires-Dist: myst-parser ==0.18.0 ; extra == 'docs-strict'
|
|
129
125
|
Requires-Dist: sphinx-reredirects ==0.0.1 ; extra == 'docs-strict'
|
|
126
|
+
Provides-Extra: linting
|
|
127
|
+
Requires-Dist: flake8 >=5.0.0 ; extra == 'linting'
|
|
128
|
+
Provides-Extra: linting-strict
|
|
129
|
+
Requires-Dist: flake8 ==5.0.0 ; extra == 'linting-strict'
|
|
130
130
|
Provides-Extra: optional
|
|
131
131
|
Provides-Extra: optional-strict
|
|
132
|
+
Requires-Dist: pint ==0.10 ; (python_version < "3.7" and python_version >= "3.6") and extra == 'optional-strict'
|
|
132
133
|
Requires-Dist: textual ==0.1.18 ; (python_version >= "3.7") and extra == 'optional-strict'
|
|
134
|
+
Requires-Dist: pint ==0.18 ; (python_version >= "3.7") and extra == 'optional-strict'
|
|
135
|
+
Requires-Dist: pint >=0.10 ; (python_version < "3.7" and python_version >= "3.6") and extra == 'optional'
|
|
133
136
|
Requires-Dist: textual >=0.1.18 ; (python_version >= "3.7") and extra == 'optional'
|
|
137
|
+
Requires-Dist: pint >=0.18 ; (python_version >= "3.7") and extra == 'optional'
|
|
134
138
|
Provides-Extra: runtime
|
|
135
139
|
Requires-Dist: ubelt >=1.3.0 ; extra == 'runtime'
|
|
136
140
|
Requires-Dist: rich >=12.5.1 ; extra == 'runtime'
|
|
@@ -151,7 +155,6 @@ Requires-Dist: networkx ==2.7 ; (python_version < "3.11" and python_version >= "
|
|
|
151
155
|
Requires-Dist: numpy ==1.23.2 ; (python_version < "3.12" and python_version >= "3.11") and extra == 'runtime-strict'
|
|
152
156
|
Requires-Dist: pandas ==1.5.0 ; (python_version < "3.12" and python_version >= "3.11") and extra == 'runtime-strict'
|
|
153
157
|
Requires-Dist: pandas ==1.1.5 ; (python_version < "3.7" and python_version >= "3.6") and extra == 'runtime-strict'
|
|
154
|
-
Requires-Dist: pint ==0.10 ; (python_version < "3.7" and python_version >= "3.6") and extra == 'runtime-strict'
|
|
155
158
|
Requires-Dist: networkx <=2.5.1,==2.5.1 ; (python_version < "3.7.0" and python_version >= "3.6.0") and extra == 'runtime-strict'
|
|
156
159
|
Requires-Dist: networkx ==2.6.2 ; (python_version < "3.8" and python_version >= "3.7") and extra == 'runtime-strict'
|
|
157
160
|
Requires-Dist: pandas ==1.3.5 ; (python_version < "3.8" and python_version >= "3.7") and extra == 'runtime-strict'
|
|
@@ -159,7 +162,6 @@ Requires-Dist: pandas ==1.4.0 ; (python_version < "3.9" and python_version >= "3
|
|
|
159
162
|
Requires-Dist: networkx ==2.8 ; (python_version < "4.0" and python_version >= "3.11") and extra == 'runtime-strict'
|
|
160
163
|
Requires-Dist: numpy ==1.26.0 ; (python_version < "4.0" and python_version >= "3.12") and extra == 'runtime-strict'
|
|
161
164
|
Requires-Dist: pandas ==2.1.1 ; (python_version < "4.0" and python_version >= "3.12") and extra == 'runtime-strict'
|
|
162
|
-
Requires-Dist: pint ==0.18 ; (python_version >= "3.7") and extra == 'runtime-strict'
|
|
163
165
|
Requires-Dist: numpy >=1.19.3 ; (python_version < "3.10" and python_version >= "3.6.0") and extra == 'runtime'
|
|
164
166
|
Requires-Dist: pandas >=1.4.0 ; (python_version < "3.10" and python_version >= "3.9") and extra == 'runtime'
|
|
165
167
|
Requires-Dist: numpy >=1.21.6 ; (python_version < "3.11" and python_version >= "3.10") and extra == 'runtime'
|
|
@@ -168,7 +170,6 @@ Requires-Dist: networkx >=2.7 ; (python_version < "3.11" and python_version >= "
|
|
|
168
170
|
Requires-Dist: numpy >=1.23.2 ; (python_version < "3.12" and python_version >= "3.11") and extra == 'runtime'
|
|
169
171
|
Requires-Dist: pandas >=1.5.0 ; (python_version < "3.12" and python_version >= "3.11") and extra == 'runtime'
|
|
170
172
|
Requires-Dist: pandas >=1.1.5 ; (python_version < "3.7" and python_version >= "3.6") and extra == 'runtime'
|
|
171
|
-
Requires-Dist: pint >=0.10 ; (python_version < "3.7" and python_version >= "3.6") and extra == 'runtime'
|
|
172
173
|
Requires-Dist: networkx <=2.5.1,>=2.5.1 ; (python_version < "3.7.0" and python_version >= "3.6.0") and extra == 'runtime'
|
|
173
174
|
Requires-Dist: networkx >=2.6.2 ; (python_version < "3.8" and python_version >= "3.7") and extra == 'runtime'
|
|
174
175
|
Requires-Dist: pandas >=1.3.5 ; (python_version < "3.8" and python_version >= "3.7") and extra == 'runtime'
|
|
@@ -176,7 +177,6 @@ Requires-Dist: pandas >=1.4.0 ; (python_version < "3.9" and python_version >= "3
|
|
|
176
177
|
Requires-Dist: networkx >=2.8 ; (python_version < "4.0" and python_version >= "3.11") and extra == 'runtime'
|
|
177
178
|
Requires-Dist: numpy >=1.26.0 ; (python_version < "4.0" and python_version >= "3.12") and extra == 'runtime'
|
|
178
179
|
Requires-Dist: pandas >=2.1.1 ; (python_version < "4.0" and python_version >= "3.12") and extra == 'runtime'
|
|
179
|
-
Requires-Dist: pint >=0.18 ; (python_version >= "3.7") and extra == 'runtime'
|
|
180
180
|
Provides-Extra: tests
|
|
181
181
|
Requires-Dist: xdoctest >=1.0.1 ; extra == 'tests'
|
|
182
182
|
Provides-Extra: tests-strict
|
|
@@ -390,6 +390,73 @@ that you define in Python.
|
|
|
390
390
|
We plan on adding an airflow backend.
|
|
391
391
|
|
|
392
392
|
|
|
393
|
+
Usage
|
|
394
|
+
=====
|
|
395
|
+
|
|
396
|
+
|
|
397
|
+
There are two ways to use ``cmd_queue``:
|
|
398
|
+
|
|
399
|
+
1. In Python create a Queue object, and then call the .submit method to pass it
|
|
400
|
+
a shell invocation. It returns an object that you can use to specify
|
|
401
|
+
dependencies of any further calls to .submit. This simply organizes all of
|
|
402
|
+
your CLI invocations into a bash script, which can be inspected and then
|
|
403
|
+
run. There are different backends that enable parallel execution of jobs
|
|
404
|
+
when dependencies allow.
|
|
405
|
+
|
|
406
|
+
2. There is a way to use it via the CLI, with details shown in cmd_queue
|
|
407
|
+
--help. Usage is basically the same. You create a queue, submit jobs to it,
|
|
408
|
+
you can inspect it, and you can run it.
|
|
409
|
+
|
|
410
|
+
|
|
411
|
+
Example usage in Python:
|
|
412
|
+
|
|
413
|
+
.. code:: python
|
|
414
|
+
|
|
415
|
+
import cmd_queue
|
|
416
|
+
|
|
417
|
+
# Create a Queue object
|
|
418
|
+
self = cmd_queue.Queue.create(name='demo_queue', backend='serial')
|
|
419
|
+
|
|
420
|
+
# Submit bash invocations that you want to run, and mark dependencies.
|
|
421
|
+
job1 = self.submit('echo hello')
|
|
422
|
+
job2 = self.submit('echo world', depends=[job1])
|
|
423
|
+
job3 = self.submit('echo foo')
|
|
424
|
+
job4 = self.submit('echo bar', depends=[job2, job3])
|
|
425
|
+
job5 = self.submit('echo spam', depends=[job1])
|
|
426
|
+
|
|
427
|
+
# Print a graph of job dependencies
|
|
428
|
+
self.print_graph()
|
|
429
|
+
|
|
430
|
+
# Display the simplified bash script to be executed.
|
|
431
|
+
self.print_commands()
|
|
432
|
+
|
|
433
|
+
# Execute the jobs
|
|
434
|
+
self.run()
|
|
435
|
+
|
|
436
|
+
|
|
437
|
+
Example usage in the CLI:
|
|
438
|
+
|
|
439
|
+
.. code:: bash
|
|
440
|
+
|
|
441
|
+
# Create a Queue
|
|
442
|
+
cmd_queue new "demo_cli_queue"
|
|
443
|
+
|
|
444
|
+
# Submit bash invocations that you want to run, and mark dependencies.
|
|
445
|
+
cmd_queue submit --jobname job1 "demo_cli_queue" -- echo hello
|
|
446
|
+
cmd_queue submit --jobname job2 --depends job1 "demo_cli_queue" -- echo world
|
|
447
|
+
cmd_queue submit --jobname job3 "demo_cli_queue" -- echo foo
|
|
448
|
+
cmd_queue submit --jobname job4 --depends job1,job2 "demo_cli_queue" -- echo bar
|
|
449
|
+
cmd_queue submit --jobname job5 --depends job1 "demo_cli_queue" -- echo spam
|
|
450
|
+
|
|
451
|
+
# Display the simplified bash script to be executed.
|
|
452
|
+
cmd_queue show "demo_cli_queue" --backend=serial
|
|
453
|
+
|
|
454
|
+
# Execute the jobs
|
|
455
|
+
cmd_queue run "demo_cli_queue" --backend=serial
|
|
456
|
+
|
|
457
|
+
|
|
458
|
+
|
|
459
|
+
|
|
393
460
|
Examples
|
|
394
461
|
========
|
|
395
462
|
|
|
@@ -402,7 +469,11 @@ use cmd_queue to "transpile" these sequences of commands to pure bash.
|
|
|
402
469
|
.. code:: python
|
|
403
470
|
|
|
404
471
|
import cmd_queue
|
|
472
|
+
|
|
473
|
+
# Create a Queue object
|
|
405
474
|
self = cmd_queue.Queue.create(name='demo_queue', backend='serial')
|
|
475
|
+
|
|
476
|
+
# Submit bash invocations that you want to run, and mark dependencies.
|
|
406
477
|
job1 = self.submit('echo hello && sleep 0.5')
|
|
407
478
|
job2 = self.submit('echo world && sleep 0.5', depends=[job1])
|
|
408
479
|
job3 = self.submit('echo foo && sleep 0.5')
|
|
@@ -414,17 +485,11 @@ use cmd_queue to "transpile" these sequences of commands to pure bash.
|
|
|
414
485
|
job9 = self.submit('echo eggs && sleep 0.5', depends=[job8])
|
|
415
486
|
job10 = self.submit('echo bazbiz && sleep 0.5', depends=[job9])
|
|
416
487
|
|
|
417
|
-
# Display the
|
|
488
|
+
# Display the simplified bash script to be executed.
|
|
418
489
|
self.print_commands()
|
|
419
490
|
|
|
420
|
-
#
|
|
421
|
-
|
|
422
|
-
# and manages dependencies.
|
|
423
|
-
self.print_commands(1, 1)
|
|
424
|
-
|
|
425
|
-
# Blocking will display a job monitor while it waits for everything to
|
|
426
|
-
# complete
|
|
427
|
-
self.run(block=True)
|
|
491
|
+
# Execute the jobs
|
|
492
|
+
self.run()
|
|
428
493
|
|
|
429
494
|
|
|
430
495
|
This prints the bash commands in an appropriate order to resolve dependencies.
|
|
@@ -469,6 +534,8 @@ This prints the bash commands in an appropriate order to resolve dependencies.
|
|
|
469
534
|
echo bazbiz && sleep 0.5
|
|
470
535
|
|
|
471
536
|
|
|
537
|
+
The same code can be run in parallel by chosing a more powerful backend.
|
|
538
|
+
The tmux backend is the lightest weight parallel backend.
|
|
472
539
|
|
|
473
540
|
.. code:: python
|
|
474
541
|
|
|
@@ -1,22 +1,22 @@
|
|
|
1
|
-
cmd_queue/__init__.py,sha256=
|
|
1
|
+
cmd_queue/__init__.py,sha256=_aGdqmuT1Evxgec-s9ANpdQI3_wKiBhPBSsH3Umvcqo,14898
|
|
2
2
|
cmd_queue/__main__.py,sha256=11Af1e3gd3CeDOj6x-OuHMttQWRvMxiB9xWdgKJeJa0,179
|
|
3
3
|
cmd_queue/__main__.pyi,sha256=UJCsqOQ7pnRJOHYNksDffyazWYrXgu6nCFZkLq_PgxA,51
|
|
4
4
|
cmd_queue/airflow_queue.py,sha256=aQE6t8im2nzJ1w_U5swBDbDEoJWrC8tUr8oEoJcqFLE,10503
|
|
5
5
|
cmd_queue/airflow_queue.pyi,sha256=2aFkIimTZ47lMBWIlnxlFhoKqNtEu7fiqARgkxqmQ5w,2141
|
|
6
|
-
cmd_queue/base_queue.py,sha256=
|
|
6
|
+
cmd_queue/base_queue.py,sha256=2Wtj_Hfvz96cxeW-7SDH5ffiPvYlh8cqNLKV8D_INhg,14529
|
|
7
7
|
cmd_queue/base_queue.pyi,sha256=WQyMlhX5PD-erM6YyOPwHvOcq1kRj7zVmgH7rdI3B2g,1850
|
|
8
8
|
cmd_queue/cli_boilerplate.py,sha256=fieFDUkzabrEYNkFzfta1LdKzIaCRjRqnA9sEpmS9zk,10117
|
|
9
9
|
cmd_queue/cli_boilerplate.pyi,sha256=7aLV5F7s7esFa_pnXFmNmWQdeJfNWdN1lTE4kAMnj4M,742
|
|
10
|
-
cmd_queue/main.py,sha256=
|
|
10
|
+
cmd_queue/main.py,sha256=CTv43Dc_19wAWdqmZ3qesHm492jg4Se25XJajpzg2HE,14309
|
|
11
11
|
cmd_queue/main.pyi,sha256=QK2dH-nvCsPSSEDNKlO4Uld6rBHsAIswXrkMd6JK2JU,1199
|
|
12
12
|
cmd_queue/monitor_app.py,sha256=m0cROUkBvTfpArjdZBfDvBMYnVJKBU_gDiW4HqENm0o,4641
|
|
13
13
|
cmd_queue/monitor_app.pyi,sha256=iMjOCX4djY-3F5GMlMcYBavm4aKYHz6RZJbsUrh43Po,860
|
|
14
14
|
cmd_queue/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
15
15
|
cmd_queue/serial_queue.py,sha256=IEqoUNdye9jL9QLkBTYiDrgTA_kM86vhZao3zrqUFdw,27101
|
|
16
16
|
cmd_queue/serial_queue.pyi,sha256=puYmRndz0-z8TzGKYrDS3fHs6OMuZT7wbHDv3NKKngU,3163
|
|
17
|
-
cmd_queue/slurm_queue.py,sha256=
|
|
17
|
+
cmd_queue/slurm_queue.py,sha256=R-i9Pf_wyPvesJi3U8yJF2coV8bWkEq-hoprp-ab94E,24634
|
|
18
18
|
cmd_queue/slurm_queue.pyi,sha256=z67qF4_IncO3YyuPIutk8-xzhQ2zOY_-IvYix0OWqJs,2174
|
|
19
|
-
cmd_queue/tmux_queue.py,sha256=
|
|
19
|
+
cmd_queue/tmux_queue.py,sha256=NfWTqSH9mFHtx0bmmFoWhvPyFrJ8Gw4Z-O3gNPF3lm4,43046
|
|
20
20
|
cmd_queue/tmux_queue.pyi,sha256=bBxxQKKVIDGXLmb2O9eHX37lXrJuo4BSePbTDmaEiNc,1992
|
|
21
21
|
cmd_queue/util/__init__.py,sha256=QzlId47F1lzuLyeIIJoPGZR3b9ljJl50sFEcJzYWrB8,1450
|
|
22
22
|
cmd_queue/util/richer.py,sha256=OPXpoXYgdhv_49sskF9FMwj6KCSiNcBo1kNhbuj6hoU,3373
|
|
@@ -37,9 +37,9 @@ cmd_queue/util/util_tmux.py,sha256=vC4xeYZCV8uVAp363zD24ROyKqUAdCynIULNJ8UgLQE,1
|
|
|
37
37
|
cmd_queue/util/util_tmux.pyi,sha256=a4XMkzhPntU4MTpeHteZIvOpP0FQt7Qym-tueHojSnQ,246
|
|
38
38
|
cmd_queue/util/util_yaml.py,sha256=5BAuP_fyucB0LrWmXLJLNyXGW7kqbyjJHLlH0zhRiKk,14460
|
|
39
39
|
cmd_queue/util/util_yaml.pyi,sha256=FmbRydo62JgVWtq6oH7ICqVeij8GEpqjEetPiCeO0sE,775
|
|
40
|
-
cmd_queue-0.
|
|
41
|
-
cmd_queue-0.
|
|
42
|
-
cmd_queue-0.
|
|
43
|
-
cmd_queue-0.
|
|
44
|
-
cmd_queue-0.
|
|
45
|
-
cmd_queue-0.
|
|
40
|
+
cmd_queue-0.2.0.dist-info/LICENSE,sha256=o6jcFk_bwjiPUz6vHK0Ju7RwbFp9eXMwAS2BDnwER-4,11343
|
|
41
|
+
cmd_queue-0.2.0.dist-info/METADATA,sha256=sq4dpnCsiiajWf82k-7RIdGtGWWWARiapmD_7eaQ4hE,36034
|
|
42
|
+
cmd_queue-0.2.0.dist-info/WHEEL,sha256=mguMlWGMX-VHnMpKOjjQidIo1ssRlCFu4a4mBpz1s2M,91
|
|
43
|
+
cmd_queue-0.2.0.dist-info/entry_points.txt,sha256=HDxa1dTf0Dne-a-QeDu7cWZVYRyEtCvqaau0GCCMEyw,54
|
|
44
|
+
cmd_queue-0.2.0.dist-info/top_level.txt,sha256=C2JVEsOZsjnMx3jIAWhIQGWAXjGs-hyBzzjkOIm7qW8,10
|
|
45
|
+
cmd_queue-0.2.0.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|