oh-my-batch 0.3.2__py3-none-any.whl → 0.4.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
oh_my_batch/combo.py CHANGED
@@ -4,7 +4,7 @@ import random
4
4
  import json
5
5
  import os
6
6
 
7
- from .util import expand_globs, mode_translate, ensure_dir
7
+ from .util import expand_globs, mode_translate, ensure_dir, shell_run
8
8
 
9
9
  class ComboMaker:
10
10
 
@@ -14,13 +14,13 @@ class ComboMaker:
14
14
 
15
15
  :param seed: Seed for random number generator
16
16
  """
17
- self._product_vars = {}
18
- self._broadcast_vars = {}
17
+ self._vars = {}
18
+ self._broadcast_keys = []
19
19
  if seed is not None:
20
20
  random.seed(seed)
21
21
  self._combos = []
22
22
 
23
- def add_seq(self, key: str, start: int, stop: int, step: int=1, broadcast=False):
23
+ def add_seq(self, key: str, start: int, stop: int, step: int=1):
24
24
  """
25
25
  Add a variable with sequence of integer values
26
26
 
@@ -28,13 +28,12 @@ class ComboMaker:
28
28
  :param start: Start value
29
29
  :param stop: Stop value
30
30
  :param step: Step
31
- :param broadcast: If True, values are broadcasted, otherwise they are producted when making combos
32
31
  """
33
32
  args = list(range(start, stop, step))
34
- self.add_var(key, *args, broadcast=broadcast)
33
+ self.add_var(key, *args)
35
34
  return self
36
35
 
37
- def add_randint(self, key: str, n: int, a: int, b: int, broadcast=False, seed=None):
36
+ def add_randint(self, key: str, n: int, a: int, b: int, seed=None):
38
37
  """
39
38
  Add a variable with random integer values
40
39
 
@@ -42,16 +41,15 @@ class ComboMaker:
42
41
  :param n: Number of values
43
42
  :param a: Lower bound
44
43
  :param b: Upper bound
45
- :param broadcast: If True, values are broadcasted, otherwise they are producted when making combos
46
44
  :param seed: Seed for random number generator
47
45
  """
48
46
  if seed is not None:
49
47
  random.seed(seed)
50
48
  args = [random.randint(a, b) for _ in range(n)]
51
- self.add_var(key, *args, broadcast=broadcast)
49
+ self.add_var(key, *args)
52
50
  return self
53
51
 
54
- def add_rand(self, key: str, n: int, a: float, b: float, broadcast=False, seed=None):
52
+ def add_rand(self, key: str, n: int, a: float, b: float, seed=None):
55
53
  """
56
54
  Add a variable with random float values
57
55
 
@@ -59,16 +57,15 @@ class ComboMaker:
59
57
  :param n: Number of values
60
58
  :param a: Lower bound
61
59
  :param b: Upper bound
62
- :param broadcast: If True, values are broadcasted, otherwise they are producted when making combos
63
60
  :param seed: Seed for random number generator
64
61
  """
65
62
  if seed is not None:
66
63
  random.seed(seed)
67
64
  args = [random.uniform(a, b) for _ in range(n)]
68
- self.add_var(key, *args, broadcast=broadcast)
65
+ self.add_var(key, *args)
69
66
  return self
70
67
 
71
- def add_files(self, key: str, *path: str, broadcast=False, abs=False, raise_invalid=False):
68
+ def add_files(self, key: str, *path: str, abs=False, raise_invalid=False):
72
69
  """
73
70
  Add a variable with files by glob pattern
74
71
  For example, suppose there are 3 files named 1.txt, 2.txt, 3.txt in data directory,
@@ -77,7 +74,6 @@ class ComboMaker:
77
74
 
78
75
  :param key: Variable name
79
76
  :param path: Path to files, can include glob pattern
80
- :param broadcast: If True, values are broadcasted, otherwise they are producted when making combos
81
77
  :param abs: If True, path will be turned into absolute path
82
78
  :param raise_invalid: If True, will raise error if no file found for a glob pattern
83
79
  """
@@ -86,10 +82,10 @@ class ComboMaker:
86
82
  raise ValueError(f"No files found for {path}")
87
83
  if abs:
88
84
  args = [os.path.abspath(p) for p in args]
89
- self.add_var(key, *args, broadcast=broadcast)
85
+ self.add_var(key, *args)
90
86
  return self
91
87
 
92
- def add_files_as_one(self, key: str, *path: str, broadcast=False, format=None,
88
+ def add_files_as_one(self, key: str, *path: str, format=None,
93
89
  sep=' ', abs=False, raise_invalid=False):
94
90
  """
95
91
  Add a variable with files by glob pattern as one string
@@ -100,7 +96,6 @@ class ComboMaker:
100
96
 
101
97
  :param key: Variable name
102
98
  :param path: Path to files, can include glob pattern
103
- :param broadcast: If True, values are broadcasted, otherwise they are producted when making combos
104
99
  :param format: the way to format the files, can be None, 'json-list','json-item'
105
100
  :param sep: Separator to join files
106
101
  :param abs: If True, path will be turned into absolute path
@@ -119,28 +114,19 @@ class ComboMaker:
119
114
  value = json.dumps(args).strip('[]')
120
115
  else:
121
116
  raise ValueError(f"Invalid format: {format}")
122
- self.add_var(key, value, broadcast=broadcast)
117
+ self.add_var(key, value)
123
118
  return self
124
119
 
125
- def add_var(self, key: str, *args, broadcast=False):
120
+ def add_var(self, key: str, *args):
126
121
  """
127
122
  Add a variable with values
128
123
 
129
124
  :param key: Variable name
130
125
  :param args: Values
131
- :param broadcast: If True, values are broadcasted, otherwise they are producted when making combos
132
126
  """
133
127
  if key == 'i':
134
128
  raise ValueError("Variable name 'i' is reserved")
135
-
136
- if broadcast:
137
- if key in self._product_vars:
138
- raise ValueError(f"Variable {key} already defined as product variable")
139
- self._broadcast_vars.setdefault(key, []).extend(args)
140
- else:
141
- if key in self._broadcast_vars:
142
- raise ValueError(f"Variable {key} already defined as broadcast variable")
143
- self._product_vars.setdefault(key, []).extend(args)
129
+ self._vars.setdefault(key, []).extend(args)
144
130
  return self
145
131
 
146
132
  def shuffle(self, *keys: str, seed=None):
@@ -153,13 +139,23 @@ class ComboMaker:
153
139
  random.seed(seed)
154
140
 
155
141
  for key in keys:
156
- if key in self._product_vars:
157
- random.shuffle(self._product_vars[key])
158
- elif key in self._broadcast_vars:
159
- random.shuffle(self._broadcast_vars[key])
142
+ if key in self._vars:
143
+ random.shuffle(self._vars[key])
160
144
  else:
161
145
  raise ValueError(f"Variable {key} not found")
162
146
  return self
147
+
148
+ def set_broadcast(self, *keys: str):
149
+ """
150
+ Specify variables use broadcast strategy instead of cartesian product
151
+
152
+ :param keys: Variable names to broadcast
153
+ """
154
+ for key in keys:
155
+ if key not in self._vars:
156
+ raise ValueError(f"Variable {key} not found")
157
+ self._broadcast_keys.append(key)
158
+ return self
163
159
 
164
160
  def make_files(self, file: str, template: str, delimiter='@', mode=None, encoding='utf-8'):
165
161
  """
@@ -218,6 +214,28 @@ class ComboMaker:
218
214
  else:
219
215
  print(out)
220
216
  return self
217
+
218
+ def run_cmd(self, cmd: str):
219
+ """
220
+ Run command against each combo
221
+
222
+ For example,
223
+
224
+ run_cmd "cp {DATA_FIEL} ./path/to/workdir/{i}/data.txt"
225
+
226
+ will copy each file in DATA_FILE to ./path/to/workdir/{i}/data.txt
227
+
228
+ :param cmd: Command to run, can include format style variables, e.g. {i}, {i:03d}, {TEMP}
229
+ """
230
+ combos = self._make_combos()
231
+ for i, combo in enumerate(combos):
232
+ _cmd = cmd.format(i=i, **combo)
233
+ cp = shell_run(_cmd)
234
+ if cp.returncode != 0:
235
+ print(cp.stdout.decode('utf-8'))
236
+ print(cp.stderr.decode('utf-8'))
237
+ raise RuntimeError(f"Failed to run command: {_cmd}")
238
+ return self
221
239
 
222
240
  def done(self):
223
241
  """
@@ -226,15 +244,23 @@ class ComboMaker:
226
244
  pass
227
245
 
228
246
  def _make_combos(self):
229
- if not self._product_vars:
247
+ if not self._vars:
230
248
  return self._combos
231
- keys = self._product_vars.keys()
232
- values_list = product(*self._product_vars.values())
249
+
250
+ broadcast_vars = {}
251
+
252
+ for k in self._broadcast_keys:
253
+ broadcast_vars[k] = self._vars[k]
254
+ del self._vars[k]
255
+
256
+ keys = self._vars.keys()
257
+ values_list = product(*self._vars.values())
258
+
233
259
  combos = [ dict(zip(keys, values)) for values in values_list ]
234
260
  for i, combo in enumerate(combos):
235
- for k, v in self._broadcast_vars.items():
261
+ for k, v in broadcast_vars.items():
236
262
  combo[k] = v[i % len(v)]
237
263
  self._combos.extend(combos)
238
- self._product_vars = {}
239
- self._broadcast_vars = {}
264
+ self._vars = {}
265
+ self._broadcast_keys = []
240
266
  return self._combos
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: oh-my-batch
3
- Version: 0.3.2
3
+ Version: 0.4.1
4
4
  Summary:
5
5
  License: GPL
6
6
  Author: weihong.xu
@@ -13,6 +13,7 @@ Classifier: Programming Language :: Python :: 3.9
13
13
  Classifier: Programming Language :: Python :: 3.10
14
14
  Classifier: Programming Language :: Python :: 3.11
15
15
  Classifier: Programming Language :: Python :: 3.12
16
+ Classifier: Programming Language :: Python :: 3.13
16
17
  Requires-Dist: fire (>=0.7.0,<0.8.0)
17
18
  Description-Content-Type: text/markdown
18
19
 
@@ -84,7 +85,8 @@ EOF
84
85
  omb combo \
85
86
  add_files DATA_FILE tmp/*.data - \
86
87
  add_var TEMP 300 400 500 - \
87
- add_randint RANDOM -n 3 -a 1 -b 1000 --broadcast - \
88
+ add_randint RANDOM -n 3 -a 1 -b 1000 - \
89
+ set_broadcast RANDOM - \
88
90
  make_files tmp/tasks/{i}-T-{TEMP}/in.lmp --template tmp/in.lmp.tmp - \
89
91
  make_files tmp/tasks/{i}-T-{TEMP}/run.sh --template tmp/run.sh.tmp --mode 755 - \
90
92
  done
@@ -4,12 +4,12 @@ oh_my_batch/assets/__init__.py,sha256=Exub46UbQaz2V2eXpQeiVfnThQpXaNeuyjlGY6gBSZ
4
4
  oh_my_batch/assets/functions.sh,sha256=LaiavZBu84D7C1r-dWhf91vPTuHXCMV1DQZUIPVQnjE,1001
5
5
  oh_my_batch/batch.py,sha256=6qnaXEVyA493heGzzbCrdZXCcnYk8zgl7WP0rmo7KlU,3690
6
6
  oh_my_batch/cli.py,sha256=Jyz8q2pUYke3mfJS6F_G9S9hApddgXxQw1BsN6Kfkjc,553
7
- oh_my_batch/combo.py,sha256=424EGKgWdgTKRyBHWoE9HdXaQShtV4gRYBd9BRo4Hek,9429
7
+ oh_my_batch/combo.py,sha256=K1pkmdDbpe-FCqUHe8fnuvWHz9aVHptGTvPds-jt_XA,9368
8
8
  oh_my_batch/job.py,sha256=Uk0E-WxnexBu9wuUV3uc1aAwVF_5rWdNdLEOB8Y9B-U,6252
9
9
  oh_my_batch/misc.py,sha256=G_iOovRCrShBJJCc82QLN0CvMqW4adOefEoY1GedEiw,452
10
10
  oh_my_batch/util.py,sha256=5ve2QcviuF0UHFLrsXmjMTj0ogXJ4g05q1y-yWCFuOk,2409
11
- oh_my_batch-0.3.2.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
12
- oh_my_batch-0.3.2.dist-info/METADATA,sha256=z3cE86WXw3q4nvYq_LCvH01ydnMntgWQToZOrAbgjpU,5492
13
- oh_my_batch-0.3.2.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
14
- oh_my_batch-0.3.2.dist-info/entry_points.txt,sha256=ZY2GutSoNjjSyJ4qO2pTeseKUFgoTYdvmgkuZZkwi68,77
15
- oh_my_batch-0.3.2.dist-info/RECORD,,
11
+ oh_my_batch-0.4.1.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
12
+ oh_my_batch-0.4.1.dist-info/METADATA,sha256=lOMdg0K_zSzMR-tlhcET2jowvBicqBTAHLBnm4Y6H6M,5560
13
+ oh_my_batch-0.4.1.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
14
+ oh_my_batch-0.4.1.dist-info/entry_points.txt,sha256=ZY2GutSoNjjSyJ4qO2pTeseKUFgoTYdvmgkuZZkwi68,77
15
+ oh_my_batch-0.4.1.dist-info/RECORD,,
@@ -1,4 +1,4 @@
1
1
  Wheel-Version: 1.0
2
- Generator: poetry-core 1.9.0
2
+ Generator: poetry-core 1.9.1
3
3
  Root-Is-Purelib: true
4
4
  Tag: py3-none-any