oh-my-batch 0.3.1__py3-none-any.whl → 0.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- oh_my_batch/combo.py +47 -42
- {oh_my_batch-0.3.1.dist-info → oh_my_batch-0.4.0.dist-info}/METADATA +4 -2
- {oh_my_batch-0.3.1.dist-info → oh_my_batch-0.4.0.dist-info}/RECORD +6 -6
- {oh_my_batch-0.3.1.dist-info → oh_my_batch-0.4.0.dist-info}/WHEEL +1 -1
- {oh_my_batch-0.3.1.dist-info → oh_my_batch-0.4.0.dist-info}/LICENSE +0 -0
- {oh_my_batch-0.3.1.dist-info → oh_my_batch-0.4.0.dist-info}/entry_points.txt +0 -0
oh_my_batch/combo.py
CHANGED
@@ -14,13 +14,13 @@ class ComboMaker:
|
|
14
14
|
|
15
15
|
:param seed: Seed for random number generator
|
16
16
|
"""
|
17
|
-
self.
|
18
|
-
self.
|
17
|
+
self._vars = {}
|
18
|
+
self._broadcast_keys = []
|
19
19
|
if seed is not None:
|
20
20
|
random.seed(seed)
|
21
21
|
self._combos = []
|
22
22
|
|
23
|
-
def add_seq(self, key: str, start: int, stop: int, step: int=1
|
23
|
+
def add_seq(self, key: str, start: int, stop: int, step: int=1):
|
24
24
|
"""
|
25
25
|
Add a variable with sequence of integer values
|
26
26
|
|
@@ -28,13 +28,12 @@ class ComboMaker:
|
|
28
28
|
:param start: Start value
|
29
29
|
:param stop: Stop value
|
30
30
|
:param step: Step
|
31
|
-
:param broadcast: If True, values are broadcasted, otherwise they are producted when making combos
|
32
31
|
"""
|
33
32
|
args = list(range(start, stop, step))
|
34
|
-
self.add_var(key, *args
|
33
|
+
self.add_var(key, *args)
|
35
34
|
return self
|
36
35
|
|
37
|
-
def add_randint(self, key: str, n: int, a: int, b: int,
|
36
|
+
def add_randint(self, key: str, n: int, a: int, b: int, seed=None):
|
38
37
|
"""
|
39
38
|
Add a variable with random integer values
|
40
39
|
|
@@ -42,16 +41,15 @@ class ComboMaker:
|
|
42
41
|
:param n: Number of values
|
43
42
|
:param a: Lower bound
|
44
43
|
:param b: Upper bound
|
45
|
-
:param broadcast: If True, values are broadcasted, otherwise they are producted when making combos
|
46
44
|
:param seed: Seed for random number generator
|
47
45
|
"""
|
48
46
|
if seed is not None:
|
49
47
|
random.seed(seed)
|
50
48
|
args = [random.randint(a, b) for _ in range(n)]
|
51
|
-
self.add_var(key, *args
|
49
|
+
self.add_var(key, *args)
|
52
50
|
return self
|
53
51
|
|
54
|
-
def add_rand(self, key: str, n: int, a: float, b: float,
|
52
|
+
def add_rand(self, key: str, n: int, a: float, b: float, seed=None):
|
55
53
|
"""
|
56
54
|
Add a variable with random float values
|
57
55
|
|
@@ -59,16 +57,15 @@ class ComboMaker:
|
|
59
57
|
:param n: Number of values
|
60
58
|
:param a: Lower bound
|
61
59
|
:param b: Upper bound
|
62
|
-
:param broadcast: If True, values are broadcasted, otherwise they are producted when making combos
|
63
60
|
:param seed: Seed for random number generator
|
64
61
|
"""
|
65
62
|
if seed is not None:
|
66
63
|
random.seed(seed)
|
67
64
|
args = [random.uniform(a, b) for _ in range(n)]
|
68
|
-
self.add_var(key, *args
|
65
|
+
self.add_var(key, *args)
|
69
66
|
return self
|
70
67
|
|
71
|
-
def add_files(self, key: str, *path: str,
|
68
|
+
def add_files(self, key: str, *path: str, abs=False, raise_invalid=False):
|
72
69
|
"""
|
73
70
|
Add a variable with files by glob pattern
|
74
71
|
For example, suppose there are 3 files named 1.txt, 2.txt, 3.txt in data directory,
|
@@ -77,7 +74,6 @@ class ComboMaker:
|
|
77
74
|
|
78
75
|
:param key: Variable name
|
79
76
|
:param path: Path to files, can include glob pattern
|
80
|
-
:param broadcast: If True, values are broadcasted, otherwise they are producted when making combos
|
81
77
|
:param abs: If True, path will be turned into absolute path
|
82
78
|
:param raise_invalid: If True, will raise error if no file found for a glob pattern
|
83
79
|
"""
|
@@ -86,10 +82,10 @@ class ComboMaker:
|
|
86
82
|
raise ValueError(f"No files found for {path}")
|
87
83
|
if abs:
|
88
84
|
args = [os.path.abspath(p) for p in args]
|
89
|
-
self.add_var(key, *args
|
85
|
+
self.add_var(key, *args)
|
90
86
|
return self
|
91
87
|
|
92
|
-
def add_files_as_one(self, key: str, *path: str,
|
88
|
+
def add_files_as_one(self, key: str, *path: str, format=None,
|
93
89
|
sep=' ', abs=False, raise_invalid=False):
|
94
90
|
"""
|
95
91
|
Add a variable with files by glob pattern as one string
|
@@ -100,7 +96,6 @@ class ComboMaker:
|
|
100
96
|
|
101
97
|
:param key: Variable name
|
102
98
|
:param path: Path to files, can include glob pattern
|
103
|
-
:param broadcast: If True, values are broadcasted, otherwise they are producted when making combos
|
104
99
|
:param format: the way to format the files, can be None, 'json-list','json-item'
|
105
100
|
:param sep: Separator to join files
|
106
101
|
:param abs: If True, path will be turned into absolute path
|
@@ -119,28 +114,19 @@ class ComboMaker:
|
|
119
114
|
value = json.dumps(args).strip('[]')
|
120
115
|
else:
|
121
116
|
raise ValueError(f"Invalid format: {format}")
|
122
|
-
self.add_var(key, value
|
117
|
+
self.add_var(key, value)
|
123
118
|
return self
|
124
119
|
|
125
|
-
def add_var(self, key: str, *args
|
120
|
+
def add_var(self, key: str, *args):
|
126
121
|
"""
|
127
122
|
Add a variable with values
|
128
123
|
|
129
124
|
:param key: Variable name
|
130
125
|
:param args: Values
|
131
|
-
:param broadcast: If True, values are broadcasted, otherwise they are producted when making combos
|
132
126
|
"""
|
133
127
|
if key == 'i':
|
134
128
|
raise ValueError("Variable name 'i' is reserved")
|
135
|
-
|
136
|
-
if broadcast:
|
137
|
-
if key in self._product_vars:
|
138
|
-
raise ValueError(f"Variable {key} already defined as product variable")
|
139
|
-
self._broadcast_vars.setdefault(key, []).extend(args)
|
140
|
-
else:
|
141
|
-
if key in self._broadcast_vars:
|
142
|
-
raise ValueError(f"Variable {key} already defined as broadcast variable")
|
143
|
-
self._product_vars.setdefault(key, []).extend(args)
|
129
|
+
self._vars.setdefault(key, []).extend(args)
|
144
130
|
return self
|
145
131
|
|
146
132
|
def shuffle(self, *keys: str, seed=None):
|
@@ -153,13 +139,23 @@ class ComboMaker:
|
|
153
139
|
random.seed(seed)
|
154
140
|
|
155
141
|
for key in keys:
|
156
|
-
if key in self.
|
157
|
-
random.shuffle(self.
|
158
|
-
elif key in self._broadcast_vars:
|
159
|
-
random.shuffle(self._broadcast_vars[key])
|
142
|
+
if key in self._vars:
|
143
|
+
random.shuffle(self._vars[key])
|
160
144
|
else:
|
161
145
|
raise ValueError(f"Variable {key} not found")
|
162
146
|
return self
|
147
|
+
|
148
|
+
def set_broadcast(self, *keys: str):
|
149
|
+
"""
|
150
|
+
Specify variables use broadcast strategy instead of cartesian product
|
151
|
+
|
152
|
+
:param keys: Variable names to broadcast
|
153
|
+
"""
|
154
|
+
for key in keys:
|
155
|
+
if key not in self._vars:
|
156
|
+
raise ValueError(f"Variable {key} not found")
|
157
|
+
self._broadcast_keys.append(key)
|
158
|
+
return self
|
163
159
|
|
164
160
|
def make_files(self, file: str, template: str, delimiter='@', mode=None, encoding='utf-8'):
|
165
161
|
"""
|
@@ -168,11 +164,11 @@ class ComboMaker:
|
|
168
164
|
For example, if delimiter is '@', then the template file can include @var1, @var2, ...
|
169
165
|
|
170
166
|
The destination can also include variables in string format style.
|
171
|
-
For example, if dest is 'output/{i}-{TEMP}.txt',
|
167
|
+
For example, if dest is 'output/{i}-{TEMP}.txt',
|
172
168
|
then files are saved as output/0-300K.txt, output/1-400K.txt, ...
|
173
169
|
|
174
170
|
:param file: Path pattern to destination file
|
175
|
-
:param template: Path to template file
|
171
|
+
:param template: Path to template file, the path can include variables in string format style
|
176
172
|
:param delimiter: Delimiter for variables in template, default is '@', as '$' is popular in shell scripts
|
177
173
|
can be changed to other character, e.g $, $$, ...
|
178
174
|
:param mode: File mode, e.g. 755, 644, ...
|
@@ -185,7 +181,8 @@ class ComboMaker:
|
|
185
181
|
|
186
182
|
combos = self._make_combos()
|
187
183
|
for i, combo in enumerate(combos):
|
188
|
-
|
184
|
+
_template = template.format(i=i, **combo)
|
185
|
+
with open(_template, 'r') as f:
|
189
186
|
template_text = f.read()
|
190
187
|
text = _Template(template_text).safe_substitute(combo)
|
191
188
|
_file = file.format(i=i, **combo)
|
@@ -195,7 +192,7 @@ class ComboMaker:
|
|
195
192
|
if mode is not None:
|
196
193
|
os.chmod(_file, mode_translate(str(mode)))
|
197
194
|
return self
|
198
|
-
|
195
|
+
|
199
196
|
def print(self, *line: str, file: str = '', mode=None, encoding='utf-8'):
|
200
197
|
"""
|
201
198
|
Print lines to a file against each combo
|
@@ -225,15 +222,23 @@ class ComboMaker:
|
|
225
222
|
pass
|
226
223
|
|
227
224
|
def _make_combos(self):
|
228
|
-
if not self.
|
225
|
+
if not self._vars:
|
229
226
|
return self._combos
|
230
|
-
|
231
|
-
|
227
|
+
|
228
|
+
broadcast_vars = {}
|
229
|
+
|
230
|
+
for k in self._broadcast_keys:
|
231
|
+
broadcast_vars[k] = self._vars[k]
|
232
|
+
del self._vars[k]
|
233
|
+
|
234
|
+
keys = self._vars.keys()
|
235
|
+
values_list = product(*self._vars.values())
|
236
|
+
|
232
237
|
combos = [ dict(zip(keys, values)) for values in values_list ]
|
233
238
|
for i, combo in enumerate(combos):
|
234
|
-
for k, v in
|
239
|
+
for k, v in broadcast_vars.items():
|
235
240
|
combo[k] = v[i % len(v)]
|
236
241
|
self._combos.extend(combos)
|
237
|
-
self.
|
238
|
-
self.
|
242
|
+
self._vars = {}
|
243
|
+
self._broadcast_keys = []
|
239
244
|
return self._combos
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: oh-my-batch
|
3
|
-
Version: 0.
|
3
|
+
Version: 0.4.0
|
4
4
|
Summary:
|
5
5
|
License: GPL
|
6
6
|
Author: weihong.xu
|
@@ -13,6 +13,7 @@ Classifier: Programming Language :: Python :: 3.9
|
|
13
13
|
Classifier: Programming Language :: Python :: 3.10
|
14
14
|
Classifier: Programming Language :: Python :: 3.11
|
15
15
|
Classifier: Programming Language :: Python :: 3.12
|
16
|
+
Classifier: Programming Language :: Python :: 3.13
|
16
17
|
Requires-Dist: fire (>=0.7.0,<0.8.0)
|
17
18
|
Description-Content-Type: text/markdown
|
18
19
|
|
@@ -84,9 +85,10 @@ EOF
|
|
84
85
|
omb combo \
|
85
86
|
add_files DATA_FILE tmp/*.data - \
|
86
87
|
add_var TEMP 300 400 500 - \
|
87
|
-
add_randint RANDOM -n 3 -a 1 -b 1000
|
88
|
+
add_randint RANDOM -n 3 -a 1 -b 1000 - \
|
88
89
|
make_files tmp/tasks/{i}-T-{TEMP}/in.lmp --template tmp/in.lmp.tmp - \
|
89
90
|
make_files tmp/tasks/{i}-T-{TEMP}/run.sh --template tmp/run.sh.tmp --mode 755 - \
|
91
|
+
set_broadcast RANDOM - \
|
90
92
|
done
|
91
93
|
```
|
92
94
|
|
@@ -4,12 +4,12 @@ oh_my_batch/assets/__init__.py,sha256=Exub46UbQaz2V2eXpQeiVfnThQpXaNeuyjlGY6gBSZ
|
|
4
4
|
oh_my_batch/assets/functions.sh,sha256=LaiavZBu84D7C1r-dWhf91vPTuHXCMV1DQZUIPVQnjE,1001
|
5
5
|
oh_my_batch/batch.py,sha256=6qnaXEVyA493heGzzbCrdZXCcnYk8zgl7WP0rmo7KlU,3690
|
6
6
|
oh_my_batch/cli.py,sha256=Jyz8q2pUYke3mfJS6F_G9S9hApddgXxQw1BsN6Kfkjc,553
|
7
|
-
oh_my_batch/combo.py,sha256=
|
7
|
+
oh_my_batch/combo.py,sha256=63F6plpGcVjKwdKCQ8b0dQhNCjHwXVirfnHPZDLe49U,8598
|
8
8
|
oh_my_batch/job.py,sha256=Uk0E-WxnexBu9wuUV3uc1aAwVF_5rWdNdLEOB8Y9B-U,6252
|
9
9
|
oh_my_batch/misc.py,sha256=G_iOovRCrShBJJCc82QLN0CvMqW4adOefEoY1GedEiw,452
|
10
10
|
oh_my_batch/util.py,sha256=5ve2QcviuF0UHFLrsXmjMTj0ogXJ4g05q1y-yWCFuOk,2409
|
11
|
-
oh_my_batch-0.
|
12
|
-
oh_my_batch-0.
|
13
|
-
oh_my_batch-0.
|
14
|
-
oh_my_batch-0.
|
15
|
-
oh_my_batch-0.
|
11
|
+
oh_my_batch-0.4.0.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
|
12
|
+
oh_my_batch-0.4.0.dist-info/METADATA,sha256=cOScID0UVLkM7O2Y8D4bBOTUWVqwxHss-DpN1ZH1Ctg,5560
|
13
|
+
oh_my_batch-0.4.0.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
|
14
|
+
oh_my_batch-0.4.0.dist-info/entry_points.txt,sha256=ZY2GutSoNjjSyJ4qO2pTeseKUFgoTYdvmgkuZZkwi68,77
|
15
|
+
oh_my_batch-0.4.0.dist-info/RECORD,,
|
File without changes
|
File without changes
|