oh-my-batch 0.3.2__py3-none-any.whl → 0.4.1__py3-none-any.whl

oh_my_batch/combo.py

@@ -4,7 +4,7 @@ import random
 import json
 import os
 
-from .util import expand_globs, mode_translate, ensure_dir
+from .util import expand_globs, mode_translate, ensure_dir, shell_run
 
 class ComboMaker:
 
@@ -14,13 +14,13 @@ class ComboMaker:
 
         :param seed: Seed for random number generator
         """
-        self._product_vars = {}
-        self._broadcast_vars = {}
+        self._vars = {}
+        self._broadcast_keys = []
         if seed is not None:
             random.seed(seed)
         self._combos = []
 
-    def add_seq(self, key: str, start: int, stop: int, step: int=1, broadcast=False):
+    def add_seq(self, key: str, start: int, stop: int, step: int=1):
         """
         Add a variable with sequence of integer values
 
@@ -28,13 +28,12 @@ class ComboMaker:
         :param start: Start value
         :param stop: Stop value
         :param step: Step
-        :param broadcast: If True, values are broadcasted, otherwise they are producted when making combos
         """
         args = list(range(start, stop, step))
-        self.add_var(key, *args, broadcast=broadcast)
+        self.add_var(key, *args)
         return self
 
-    def add_randint(self, key: str, n: int, a: int, b: int, broadcast=False, seed=None):
+    def add_randint(self, key: str, n: int, a: int, b: int, seed=None):
         """
         Add a variable with random integer values
 
@@ -42,16 +41,15 @@ class ComboMaker:
         :param n: Number of values
         :param a: Lower bound
         :param b: Upper bound
-        :param broadcast: If True, values are broadcasted, otherwise they are producted when making combos
         :param seed: Seed for random number generator
         """
         if seed is not None:
             random.seed(seed)
         args = [random.randint(a, b) for _ in range(n)]
-        self.add_var(key, *args, broadcast=broadcast)
+        self.add_var(key, *args)
         return self
 
-    def add_rand(self, key: str, n: int, a: float, b: float, broadcast=False, seed=None):
+    def add_rand(self, key: str, n: int, a: float, b: float, seed=None):
         """
         Add a variable with random float values
 
@@ -59,16 +57,15 @@ class ComboMaker:
         :param n: Number of values
         :param a: Lower bound
         :param b: Upper bound
-        :param broadcast: If True, values are broadcasted, otherwise they are producted when making combos
         :param seed: Seed for random number generator
         """
         if seed is not None:
             random.seed(seed)
         args = [random.uniform(a, b) for _ in range(n)]
-        self.add_var(key, *args, broadcast=broadcast)
+        self.add_var(key, *args)
         return self
 
-    def add_files(self, key: str, *path: str, broadcast=False, abs=False, raise_invalid=False):
+    def add_files(self, key: str, *path: str, abs=False, raise_invalid=False):
         """
         Add a variable with files by glob pattern
         For example, suppose there are 3 files named 1.txt, 2.txt, 3.txt in data directory,
@@ -77,7 +74,6 @@ class ComboMaker:
 
         :param key: Variable name
         :param path: Path to files, can include glob pattern
-        :param broadcast: If True, values are broadcasted, otherwise they are producted when making combos
         :param abs: If True, path will be turned into absolute path
         :param raise_invalid: If True, will raise error if no file found for a glob pattern
         """
@@ -86,10 +82,10 @@ class ComboMaker:
             raise ValueError(f"No files found for {path}")
         if abs:
             args = [os.path.abspath(p) for p in args]
-        self.add_var(key, *args, broadcast=broadcast)
+        self.add_var(key, *args)
         return self
 
-    def add_files_as_one(self, key: str, *path: str, broadcast=False, format=None,
+    def add_files_as_one(self, key: str, *path: str, format=None,
                          sep=' ', abs=False, raise_invalid=False):
         """
         Add a variable with files by glob pattern as one string
@@ -100,7 +96,6 @@ class ComboMaker:
 
         :param key: Variable name
         :param path: Path to files, can include glob pattern
-        :param broadcast: If True, values are broadcasted, otherwise they are producted when making combos
         :param format: the way to format the files, can be None, 'json-list','json-item'
         :param sep: Separator to join files
         :param abs: If True, path will be turned into absolute path
@@ -119,28 +114,19 @@ class ComboMaker:
             value = json.dumps(args).strip('[]')
         else:
             raise ValueError(f"Invalid format: {format}")
-        self.add_var(key, value, broadcast=broadcast)
+        self.add_var(key, value)
         return self
 
-    def add_var(self, key: str, *args, broadcast=False):
+    def add_var(self, key: str, *args):
         """
         Add a variable with values
 
         :param key: Variable name
         :param args: Values
-        :param broadcast: If True, values are broadcasted, otherwise they are producted when making combos
         """
         if key == 'i':
             raise ValueError("Variable name 'i' is reserved")
-
-        if broadcast:
-            if key in self._product_vars:
-                raise ValueError(f"Variable {key} already defined as product variable")
-            self._broadcast_vars.setdefault(key, []).extend(args)
-        else:
-            if key in self._broadcast_vars:
-                raise ValueError(f"Variable {key} already defined as broadcast variable")
-            self._product_vars.setdefault(key, []).extend(args)
+        self._vars.setdefault(key, []).extend(args)
         return self
 
     def shuffle(self, *keys: str, seed=None):
@@ -153,13 +139,23 @@ class ComboMaker:
             random.seed(seed)
 
         for key in keys:
-            if key in self._product_vars:
-                random.shuffle(self._product_vars[key])
-            elif key in self._broadcast_vars:
-                random.shuffle(self._broadcast_vars[key])
+            if key in self._vars:
+                random.shuffle(self._vars[key])
             else:
                 raise ValueError(f"Variable {key} not found")
         return self
+    
+    def set_broadcast(self, *keys: str):
+        """
+        Specify variables use broadcast strategy instead of cartesian product
+
+        :param keys: Variable names to broadcast
+        """
+        for key in keys:
+            if key not in self._vars:
+                raise ValueError(f"Variable {key} not found")
+            self._broadcast_keys.append(key)
+        return self
 
     def make_files(self, file: str, template: str, delimiter='@', mode=None, encoding='utf-8'):
         """
@@ -218,6 +214,28 @@ class ComboMaker:
         else:
             print(out)
         return self
+    
+    def run_cmd(self, cmd: str):
+        """
+        Run command against each combo
+
+        For example, 
+        
+        run_cmd "cp {DATA_FIEL} ./path/to/workdir/{i}/data.txt"
+
+        will copy each file in DATA_FILE to ./path/to/workdir/{i}/data.txt
+        
+        :param cmd: Command to run, can include format style variables, e.g. {i}, {i:03d}, {TEMP}
+        """
+        combos = self._make_combos()
+        for i, combo in enumerate(combos):
+            _cmd = cmd.format(i=i, **combo)
+            cp = shell_run(_cmd)
+            if cp.returncode != 0:
+                print(cp.stdout.decode('utf-8'))
+                print(cp.stderr.decode('utf-8'))
+                raise RuntimeError(f"Failed to run command: {_cmd}")
+        return self
 
     def done(self):
         """
@@ -226,15 +244,23 @@ class ComboMaker:
         pass
 
     def _make_combos(self):
-        if not self._product_vars:
+        if not self._vars:
             return self._combos
-        keys = self._product_vars.keys()
-        values_list = product(*self._product_vars.values())
+        
+        broadcast_vars = {}
+        
+        for k in self._broadcast_keys:
+            broadcast_vars[k] = self._vars[k]
+            del self._vars[k]
+            
+        keys = self._vars.keys()
+        values_list = product(*self._vars.values())
+
         combos = [ dict(zip(keys, values)) for values in values_list ]
         for i, combo in enumerate(combos):
-            for k, v in self._broadcast_vars.items():
+            for k, v in broadcast_vars.items():
                 combo[k] = v[i % len(v)]
         self._combos.extend(combos)
-        self._product_vars = {}
-        self._broadcast_vars = {}
+        self._vars = {}
+        self._broadcast_keys = []
         return self._combos

{oh_my_batch-0.3.2.dist-info → oh_my_batch-0.4.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: oh-my-batch
-Version: 0.3.2
+Version: 0.4.1
 Summary: 
 License: GPL
 Author: weihong.xu
@@ -13,6 +13,7 @@ Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Requires-Dist: fire (>=0.7.0,<0.8.0)
 Description-Content-Type: text/markdown
 
@@ -84,7 +85,8 @@ EOF
 omb combo \
     add_files DATA_FILE tmp/*.data - \
     add_var TEMP 300 400 500 - \
-    add_randint RANDOM -n 3 -a 1 -b 1000 --broadcast - \
+    add_randint RANDOM -n 3 -a 1 -b 1000 - \
+    set_broadcast RANDOM - \
     make_files tmp/tasks/{i}-T-{TEMP}/in.lmp --template tmp/in.lmp.tmp - \
     make_files tmp/tasks/{i}-T-{TEMP}/run.sh --template tmp/run.sh.tmp --mode 755 - \
     done

{oh_my_batch-0.3.2.dist-info → oh_my_batch-0.4.1.dist-info}/RECORD

@@ -4,12 +4,12 @@ oh_my_batch/assets/__init__.py,sha256=Exub46UbQaz2V2eXpQeiVfnThQpXaNeuyjlGY6gBSZ
 oh_my_batch/assets/functions.sh,sha256=LaiavZBu84D7C1r-dWhf91vPTuHXCMV1DQZUIPVQnjE,1001
 oh_my_batch/batch.py,sha256=6qnaXEVyA493heGzzbCrdZXCcnYk8zgl7WP0rmo7KlU,3690
 oh_my_batch/cli.py,sha256=Jyz8q2pUYke3mfJS6F_G9S9hApddgXxQw1BsN6Kfkjc,553
-oh_my_batch/combo.py,sha256=424EGKgWdgTKRyBHWoE9HdXaQShtV4gRYBd9BRo4Hek,9429
+oh_my_batch/combo.py,sha256=K1pkmdDbpe-FCqUHe8fnuvWHz9aVHptGTvPds-jt_XA,9368
 oh_my_batch/job.py,sha256=Uk0E-WxnexBu9wuUV3uc1aAwVF_5rWdNdLEOB8Y9B-U,6252
 oh_my_batch/misc.py,sha256=G_iOovRCrShBJJCc82QLN0CvMqW4adOefEoY1GedEiw,452
 oh_my_batch/util.py,sha256=5ve2QcviuF0UHFLrsXmjMTj0ogXJ4g05q1y-yWCFuOk,2409
-oh_my_batch-0.3.2.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-oh_my_batch-0.3.2.dist-info/METADATA,sha256=z3cE86WXw3q4nvYq_LCvH01ydnMntgWQToZOrAbgjpU,5492
-oh_my_batch-0.3.2.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-oh_my_batch-0.3.2.dist-info/entry_points.txt,sha256=ZY2GutSoNjjSyJ4qO2pTeseKUFgoTYdvmgkuZZkwi68,77
-oh_my_batch-0.3.2.dist-info/RECORD,,
+oh_my_batch-0.4.1.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+oh_my_batch-0.4.1.dist-info/METADATA,sha256=lOMdg0K_zSzMR-tlhcET2jowvBicqBTAHLBnm4Y6H6M,5560
+oh_my_batch-0.4.1.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+oh_my_batch-0.4.1.dist-info/entry_points.txt,sha256=ZY2GutSoNjjSyJ4qO2pTeseKUFgoTYdvmgkuZZkwi68,77
+oh_my_batch-0.4.1.dist-info/RECORD,,

{oh_my_batch-0.3.2.dist-info → oh_my_batch-0.4.1.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 1.9.0
+Generator: poetry-core 1.9.1
 Root-Is-Purelib: true
 Tag: py3-none-any