oh-my-batch 0.3.2__py3-none-any.whl → 0.4.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- oh_my_batch/combo.py +65 -39
- {oh_my_batch-0.3.2.dist-info → oh_my_batch-0.4.1.dist-info}/METADATA +4 -2
- {oh_my_batch-0.3.2.dist-info → oh_my_batch-0.4.1.dist-info}/RECORD +6 -6
- {oh_my_batch-0.3.2.dist-info → oh_my_batch-0.4.1.dist-info}/WHEEL +1 -1
- {oh_my_batch-0.3.2.dist-info → oh_my_batch-0.4.1.dist-info}/LICENSE +0 -0
- {oh_my_batch-0.3.2.dist-info → oh_my_batch-0.4.1.dist-info}/entry_points.txt +0 -0
oh_my_batch/combo.py
CHANGED
@@ -4,7 +4,7 @@ import random
|
|
4
4
|
import json
|
5
5
|
import os
|
6
6
|
|
7
|
-
from .util import expand_globs, mode_translate, ensure_dir
|
7
|
+
from .util import expand_globs, mode_translate, ensure_dir, shell_run
|
8
8
|
|
9
9
|
class ComboMaker:
|
10
10
|
|
@@ -14,13 +14,13 @@ class ComboMaker:
|
|
14
14
|
|
15
15
|
:param seed: Seed for random number generator
|
16
16
|
"""
|
17
|
-
self.
|
18
|
-
self.
|
17
|
+
self._vars = {}
|
18
|
+
self._broadcast_keys = []
|
19
19
|
if seed is not None:
|
20
20
|
random.seed(seed)
|
21
21
|
self._combos = []
|
22
22
|
|
23
|
-
def add_seq(self, key: str, start: int, stop: int, step: int=1
|
23
|
+
def add_seq(self, key: str, start: int, stop: int, step: int=1):
|
24
24
|
"""
|
25
25
|
Add a variable with sequence of integer values
|
26
26
|
|
@@ -28,13 +28,12 @@ class ComboMaker:
|
|
28
28
|
:param start: Start value
|
29
29
|
:param stop: Stop value
|
30
30
|
:param step: Step
|
31
|
-
:param broadcast: If True, values are broadcasted, otherwise they are producted when making combos
|
32
31
|
"""
|
33
32
|
args = list(range(start, stop, step))
|
34
|
-
self.add_var(key, *args
|
33
|
+
self.add_var(key, *args)
|
35
34
|
return self
|
36
35
|
|
37
|
-
def add_randint(self, key: str, n: int, a: int, b: int,
|
36
|
+
def add_randint(self, key: str, n: int, a: int, b: int, seed=None):
|
38
37
|
"""
|
39
38
|
Add a variable with random integer values
|
40
39
|
|
@@ -42,16 +41,15 @@ class ComboMaker:
|
|
42
41
|
:param n: Number of values
|
43
42
|
:param a: Lower bound
|
44
43
|
:param b: Upper bound
|
45
|
-
:param broadcast: If True, values are broadcasted, otherwise they are producted when making combos
|
46
44
|
:param seed: Seed for random number generator
|
47
45
|
"""
|
48
46
|
if seed is not None:
|
49
47
|
random.seed(seed)
|
50
48
|
args = [random.randint(a, b) for _ in range(n)]
|
51
|
-
self.add_var(key, *args
|
49
|
+
self.add_var(key, *args)
|
52
50
|
return self
|
53
51
|
|
54
|
-
def add_rand(self, key: str, n: int, a: float, b: float,
|
52
|
+
def add_rand(self, key: str, n: int, a: float, b: float, seed=None):
|
55
53
|
"""
|
56
54
|
Add a variable with random float values
|
57
55
|
|
@@ -59,16 +57,15 @@ class ComboMaker:
|
|
59
57
|
:param n: Number of values
|
60
58
|
:param a: Lower bound
|
61
59
|
:param b: Upper bound
|
62
|
-
:param broadcast: If True, values are broadcasted, otherwise they are producted when making combos
|
63
60
|
:param seed: Seed for random number generator
|
64
61
|
"""
|
65
62
|
if seed is not None:
|
66
63
|
random.seed(seed)
|
67
64
|
args = [random.uniform(a, b) for _ in range(n)]
|
68
|
-
self.add_var(key, *args
|
65
|
+
self.add_var(key, *args)
|
69
66
|
return self
|
70
67
|
|
71
|
-
def add_files(self, key: str, *path: str,
|
68
|
+
def add_files(self, key: str, *path: str, abs=False, raise_invalid=False):
|
72
69
|
"""
|
73
70
|
Add a variable with files by glob pattern
|
74
71
|
For example, suppose there are 3 files named 1.txt, 2.txt, 3.txt in data directory,
|
@@ -77,7 +74,6 @@ class ComboMaker:
|
|
77
74
|
|
78
75
|
:param key: Variable name
|
79
76
|
:param path: Path to files, can include glob pattern
|
80
|
-
:param broadcast: If True, values are broadcasted, otherwise they are producted when making combos
|
81
77
|
:param abs: If True, path will be turned into absolute path
|
82
78
|
:param raise_invalid: If True, will raise error if no file found for a glob pattern
|
83
79
|
"""
|
@@ -86,10 +82,10 @@ class ComboMaker:
|
|
86
82
|
raise ValueError(f"No files found for {path}")
|
87
83
|
if abs:
|
88
84
|
args = [os.path.abspath(p) for p in args]
|
89
|
-
self.add_var(key, *args
|
85
|
+
self.add_var(key, *args)
|
90
86
|
return self
|
91
87
|
|
92
|
-
def add_files_as_one(self, key: str, *path: str,
|
88
|
+
def add_files_as_one(self, key: str, *path: str, format=None,
|
93
89
|
sep=' ', abs=False, raise_invalid=False):
|
94
90
|
"""
|
95
91
|
Add a variable with files by glob pattern as one string
|
@@ -100,7 +96,6 @@ class ComboMaker:
|
|
100
96
|
|
101
97
|
:param key: Variable name
|
102
98
|
:param path: Path to files, can include glob pattern
|
103
|
-
:param broadcast: If True, values are broadcasted, otherwise they are producted when making combos
|
104
99
|
:param format: the way to format the files, can be None, 'json-list','json-item'
|
105
100
|
:param sep: Separator to join files
|
106
101
|
:param abs: If True, path will be turned into absolute path
|
@@ -119,28 +114,19 @@ class ComboMaker:
|
|
119
114
|
value = json.dumps(args).strip('[]')
|
120
115
|
else:
|
121
116
|
raise ValueError(f"Invalid format: {format}")
|
122
|
-
self.add_var(key, value
|
117
|
+
self.add_var(key, value)
|
123
118
|
return self
|
124
119
|
|
125
|
-
def add_var(self, key: str, *args
|
120
|
+
def add_var(self, key: str, *args):
|
126
121
|
"""
|
127
122
|
Add a variable with values
|
128
123
|
|
129
124
|
:param key: Variable name
|
130
125
|
:param args: Values
|
131
|
-
:param broadcast: If True, values are broadcasted, otherwise they are producted when making combos
|
132
126
|
"""
|
133
127
|
if key == 'i':
|
134
128
|
raise ValueError("Variable name 'i' is reserved")
|
135
|
-
|
136
|
-
if broadcast:
|
137
|
-
if key in self._product_vars:
|
138
|
-
raise ValueError(f"Variable {key} already defined as product variable")
|
139
|
-
self._broadcast_vars.setdefault(key, []).extend(args)
|
140
|
-
else:
|
141
|
-
if key in self._broadcast_vars:
|
142
|
-
raise ValueError(f"Variable {key} already defined as broadcast variable")
|
143
|
-
self._product_vars.setdefault(key, []).extend(args)
|
129
|
+
self._vars.setdefault(key, []).extend(args)
|
144
130
|
return self
|
145
131
|
|
146
132
|
def shuffle(self, *keys: str, seed=None):
|
@@ -153,13 +139,23 @@ class ComboMaker:
|
|
153
139
|
random.seed(seed)
|
154
140
|
|
155
141
|
for key in keys:
|
156
|
-
if key in self.
|
157
|
-
random.shuffle(self.
|
158
|
-
elif key in self._broadcast_vars:
|
159
|
-
random.shuffle(self._broadcast_vars[key])
|
142
|
+
if key in self._vars:
|
143
|
+
random.shuffle(self._vars[key])
|
160
144
|
else:
|
161
145
|
raise ValueError(f"Variable {key} not found")
|
162
146
|
return self
|
147
|
+
|
148
|
+
def set_broadcast(self, *keys: str):
|
149
|
+
"""
|
150
|
+
Specify variables use broadcast strategy instead of cartesian product
|
151
|
+
|
152
|
+
:param keys: Variable names to broadcast
|
153
|
+
"""
|
154
|
+
for key in keys:
|
155
|
+
if key not in self._vars:
|
156
|
+
raise ValueError(f"Variable {key} not found")
|
157
|
+
self._broadcast_keys.append(key)
|
158
|
+
return self
|
163
159
|
|
164
160
|
def make_files(self, file: str, template: str, delimiter='@', mode=None, encoding='utf-8'):
|
165
161
|
"""
|
@@ -218,6 +214,28 @@ class ComboMaker:
|
|
218
214
|
else:
|
219
215
|
print(out)
|
220
216
|
return self
|
217
|
+
|
218
|
+
def run_cmd(self, cmd: str):
|
219
|
+
"""
|
220
|
+
Run command against each combo
|
221
|
+
|
222
|
+
For example,
|
223
|
+
|
224
|
+
run_cmd "cp {DATA_FIEL} ./path/to/workdir/{i}/data.txt"
|
225
|
+
|
226
|
+
will copy each file in DATA_FILE to ./path/to/workdir/{i}/data.txt
|
227
|
+
|
228
|
+
:param cmd: Command to run, can include format style variables, e.g. {i}, {i:03d}, {TEMP}
|
229
|
+
"""
|
230
|
+
combos = self._make_combos()
|
231
|
+
for i, combo in enumerate(combos):
|
232
|
+
_cmd = cmd.format(i=i, **combo)
|
233
|
+
cp = shell_run(_cmd)
|
234
|
+
if cp.returncode != 0:
|
235
|
+
print(cp.stdout.decode('utf-8'))
|
236
|
+
print(cp.stderr.decode('utf-8'))
|
237
|
+
raise RuntimeError(f"Failed to run command: {_cmd}")
|
238
|
+
return self
|
221
239
|
|
222
240
|
def done(self):
|
223
241
|
"""
|
@@ -226,15 +244,23 @@ class ComboMaker:
|
|
226
244
|
pass
|
227
245
|
|
228
246
|
def _make_combos(self):
|
229
|
-
if not self.
|
247
|
+
if not self._vars:
|
230
248
|
return self._combos
|
231
|
-
|
232
|
-
|
249
|
+
|
250
|
+
broadcast_vars = {}
|
251
|
+
|
252
|
+
for k in self._broadcast_keys:
|
253
|
+
broadcast_vars[k] = self._vars[k]
|
254
|
+
del self._vars[k]
|
255
|
+
|
256
|
+
keys = self._vars.keys()
|
257
|
+
values_list = product(*self._vars.values())
|
258
|
+
|
233
259
|
combos = [ dict(zip(keys, values)) for values in values_list ]
|
234
260
|
for i, combo in enumerate(combos):
|
235
|
-
for k, v in
|
261
|
+
for k, v in broadcast_vars.items():
|
236
262
|
combo[k] = v[i % len(v)]
|
237
263
|
self._combos.extend(combos)
|
238
|
-
self.
|
239
|
-
self.
|
264
|
+
self._vars = {}
|
265
|
+
self._broadcast_keys = []
|
240
266
|
return self._combos
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: oh-my-batch
|
3
|
-
Version: 0.
|
3
|
+
Version: 0.4.1
|
4
4
|
Summary:
|
5
5
|
License: GPL
|
6
6
|
Author: weihong.xu
|
@@ -13,6 +13,7 @@ Classifier: Programming Language :: Python :: 3.9
|
|
13
13
|
Classifier: Programming Language :: Python :: 3.10
|
14
14
|
Classifier: Programming Language :: Python :: 3.11
|
15
15
|
Classifier: Programming Language :: Python :: 3.12
|
16
|
+
Classifier: Programming Language :: Python :: 3.13
|
16
17
|
Requires-Dist: fire (>=0.7.0,<0.8.0)
|
17
18
|
Description-Content-Type: text/markdown
|
18
19
|
|
@@ -84,7 +85,8 @@ EOF
|
|
84
85
|
omb combo \
|
85
86
|
add_files DATA_FILE tmp/*.data - \
|
86
87
|
add_var TEMP 300 400 500 - \
|
87
|
-
add_randint RANDOM -n 3 -a 1 -b 1000
|
88
|
+
add_randint RANDOM -n 3 -a 1 -b 1000 - \
|
89
|
+
set_broadcast RANDOM - \
|
88
90
|
make_files tmp/tasks/{i}-T-{TEMP}/in.lmp --template tmp/in.lmp.tmp - \
|
89
91
|
make_files tmp/tasks/{i}-T-{TEMP}/run.sh --template tmp/run.sh.tmp --mode 755 - \
|
90
92
|
done
|
@@ -4,12 +4,12 @@ oh_my_batch/assets/__init__.py,sha256=Exub46UbQaz2V2eXpQeiVfnThQpXaNeuyjlGY6gBSZ
|
|
4
4
|
oh_my_batch/assets/functions.sh,sha256=LaiavZBu84D7C1r-dWhf91vPTuHXCMV1DQZUIPVQnjE,1001
|
5
5
|
oh_my_batch/batch.py,sha256=6qnaXEVyA493heGzzbCrdZXCcnYk8zgl7WP0rmo7KlU,3690
|
6
6
|
oh_my_batch/cli.py,sha256=Jyz8q2pUYke3mfJS6F_G9S9hApddgXxQw1BsN6Kfkjc,553
|
7
|
-
oh_my_batch/combo.py,sha256=
|
7
|
+
oh_my_batch/combo.py,sha256=K1pkmdDbpe-FCqUHe8fnuvWHz9aVHptGTvPds-jt_XA,9368
|
8
8
|
oh_my_batch/job.py,sha256=Uk0E-WxnexBu9wuUV3uc1aAwVF_5rWdNdLEOB8Y9B-U,6252
|
9
9
|
oh_my_batch/misc.py,sha256=G_iOovRCrShBJJCc82QLN0CvMqW4adOefEoY1GedEiw,452
|
10
10
|
oh_my_batch/util.py,sha256=5ve2QcviuF0UHFLrsXmjMTj0ogXJ4g05q1y-yWCFuOk,2409
|
11
|
-
oh_my_batch-0.
|
12
|
-
oh_my_batch-0.
|
13
|
-
oh_my_batch-0.
|
14
|
-
oh_my_batch-0.
|
15
|
-
oh_my_batch-0.
|
11
|
+
oh_my_batch-0.4.1.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
|
12
|
+
oh_my_batch-0.4.1.dist-info/METADATA,sha256=lOMdg0K_zSzMR-tlhcET2jowvBicqBTAHLBnm4Y6H6M,5560
|
13
|
+
oh_my_batch-0.4.1.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
|
14
|
+
oh_my_batch-0.4.1.dist-info/entry_points.txt,sha256=ZY2GutSoNjjSyJ4qO2pTeseKUFgoTYdvmgkuZZkwi68,77
|
15
|
+
oh_my_batch-0.4.1.dist-info/RECORD,,
|
File without changes
|
File without changes
|