oh-my-batch 0.0.1.dev0__py3-none-any.whl → 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,5 @@
1
+ from .cli import main
2
+
3
+
4
+ if __name__ == '__main__':
5
+ main()
@@ -0,0 +1,7 @@
1
+ import os
2
+
3
+ DIR = os.path.dirname(os.path.abspath(__file__))
4
+
5
+
6
+ def get_asset(rel_path: str):
7
+ return os.path.join(DIR, rel_path)
@@ -0,0 +1,20 @@
1
+
2
+ checkpoint() {
3
+ # Usage: checkpoint <flag_file> <command> [arg1] [arg2] ...
4
+ local flag_file="$1"
5
+ shift # Remove the first argument so $@ contains only the command and its arguments
6
+ if [ -f "$flag_file" ]; then
7
+ cat "$flag_file"
8
+ else
9
+ "$@" # Execute the command
10
+ local exit_code=$?
11
+ if [ $exit_code -eq 0 ]; then
12
+ local current_time=$(date '+%Y-%m-%d %H:%M:%S')
13
+ printf 'Command succeeded at %s\n' "$current_time" > "$flag_file"
14
+ echo "Created flag file '$flag_file' with timestamp: $current_time"
15
+ else
16
+ echo "Command `$@` failed with exit code $exit_code"
17
+ return $exit_code
18
+ fi
19
+ fi
20
+ }
oh_my_batch/batch.py ADDED
@@ -0,0 +1,96 @@
1
+ import shlex
2
+ import os
3
+
4
+ from .util import split_list, ensure_dir, expand_globs, mode_translate
5
+ from .assets import get_asset
6
+
7
+
8
+ class BatchMaker:
9
+
10
+ def __init__(self):
11
+ self._work_dirs = []
12
+ self._script_header = []
13
+ self._script_bottom = []
14
+ self._command = []
15
+
16
+ def add_work_dir(self, *dir: str):
17
+ """
18
+ Add working directories
19
+
20
+ :param dir: Directories to work on, can be glob patterns
21
+ """
22
+ self._work_dirs.extend(expand_globs(dir))
23
+ return self
24
+
25
+ def add_header_file(self, file: str, encoding='utf-8'):
26
+ """
27
+ Add script header from files
28
+
29
+ :param file: File path
30
+ :param encoding: File encoding
31
+ """
32
+ with open(file, 'r', encoding=encoding) as f:
33
+ self._script_header.append(f.read())
34
+ return self
35
+
36
+ def add_bottom_file(self, file: str, encoding='utf-8'):
37
+ """
38
+ Add script bottom from files
39
+
40
+ :param file: File path
41
+ :param encoding: File encoding
42
+ """
43
+ with open(file, 'r', encoding=encoding) as f:
44
+ self._script_bottom.append(f.read())
45
+
46
+ def add_command_file(self, file: str, encoding='utf-8'):
47
+ """
48
+ Add commands from files to run under every working directory
49
+
50
+ :param file: File path
51
+ :param encoding: File encoding
52
+ """
53
+ with open(file, 'r', encoding=encoding) as f:
54
+ self._command.append(f.read())
55
+ return self
56
+
57
+ def add_command(self, *cmd: str):
58
+ """
59
+ add commands to run under every working directory
60
+
61
+ :param cmd: Commands to run, can be multiple
62
+ """
63
+ self._command.extend(cmd)
64
+ return self
65
+
66
+ def make(self, path: str, concurrency=1, encoding='utf-8', mode='755'):
67
+ """
68
+ Make batch script files from the previous setup
69
+
70
+ :param path: Path to save batch script files, use {i} to represent index
71
+ :param concurrency: Number of concurrent commands to run
72
+ """
73
+ # inject pre-defined functions
74
+ self.add_header_file(get_asset('functions.sh'))
75
+
76
+ header = '\n'.join(self._script_header)
77
+ bottom = '\n'.join(self._script_bottom)
78
+ for i, work_dirs in enumerate(split_list(self._work_dirs, concurrency)):
79
+ body = []
80
+ work_dirs_arr = "\n".join(shlex.quote(w) for w in work_dirs)
81
+ body.extend([
82
+ '[ -n "$PBS_O_WORKDIR" ] && cd $PBS_O_WORKDIR # fix PBS',
83
+ f'work_dirs=({work_dirs_arr})',
84
+ '',
85
+ 'for work_dir in "${work_dirs[@]}"; do',
86
+ 'pushd $work_dir',
87
+ *self._command,
88
+ 'popd',
89
+ 'done'
90
+ ])
91
+ script = '\n'.join([header, *body, bottom])
92
+ out_path = path.format(i=i)
93
+ ensure_dir(out_path)
94
+ with open(out_path, 'w', encoding=encoding) as f:
95
+ f.write(script)
96
+ os.chmod(out_path, mode_translate(str(mode)))
oh_my_batch/cli.py ADDED
@@ -0,0 +1,28 @@
1
+ import logging
2
+ import fire
3
+
4
+ logging.basicConfig(format='%(asctime)s %(name)s: %(message)s', level=logging.INFO)
5
+
6
+ class JobCli:
7
+
8
+ def slurm(self):
9
+ from .job import Slurm
10
+ return Slurm
11
+
12
+
13
+ class OhMyBatch:
14
+
15
+ def combo(self):
16
+ from .combo import ComboMaker
17
+ return ComboMaker
18
+
19
+ def batch(self):
20
+ from .batch import BatchMaker
21
+ return BatchMaker
22
+
23
+ def job(self):
24
+ return JobCli()
25
+
26
+
27
+ def main():
28
+ fire.Fire(OhMyBatch)
oh_my_batch/combo.py ADDED
@@ -0,0 +1,202 @@
1
+ from itertools import product
2
+ from string import Template
3
+ import random
4
+ import os
5
+
6
+ from .util import expand_globs, mode_translate
7
+
8
+ class ComboMaker:
9
+
10
+ def __init__(self, seed=None):
11
+ """
12
+ ComboMaker constructor
13
+
14
+ :param seed: Seed for random number generator
15
+ """
16
+ self._product_vars = {}
17
+ self._broadcast_vars = {}
18
+ if seed is not None:
19
+ random.seed(seed)
20
+ self._combos = []
21
+
22
+ def add_seq(self, key: str, start: int, stop: int, step: int=1, broadcast=False):
23
+ """
24
+ Add a variable with sequence of integer values
25
+
26
+ :param key: Variable name
27
+ :param start: Start value
28
+ :param stop: Stop value
29
+ :param step: Step
30
+ :param broadcast: If True, values are broadcasted, otherwise they are producted when making combos
31
+ """
32
+ args = list(range(start, stop, step))
33
+ self.add_var(key, *args, broadcast=broadcast)
34
+ return self
35
+
36
+ def add_randint(self, key: str, n: int, a: int, b: int, broadcast=False, seed=None):
37
+ """
38
+ Add a variable with random integer values
39
+
40
+ :param key: Variable name
41
+ :param n: Number of values
42
+ :param a: Lower bound
43
+ :param b: Upper bound
44
+ :param broadcast: If True, values are broadcasted, otherwise they are producted when making combos
45
+ :param seed: Seed for random number generator
46
+ """
47
+ if seed is not None:
48
+ random.seed(seed)
49
+ args = [random.randint(a, b) for _ in range(n)]
50
+ self.add_var(key, *args, broadcast=broadcast)
51
+ return self
52
+
53
+ def add_rand(self, key: str, n: int, a: float, b: float, broadcast=False, seed=None):
54
+ """
55
+ Add a variable with random float values
56
+
57
+ :param key: Variable name
58
+ :param n: Number of values
59
+ :param a: Lower bound
60
+ :param b: Upper bound
61
+ :param broadcast: If True, values are broadcasted, otherwise they are producted when making combos
62
+ :param seed: Seed for random number generator
63
+ """
64
+ if seed is not None:
65
+ random.seed(seed)
66
+ args = [random.uniform(a, b) for _ in range(n)]
67
+ self.add_var(key, *args, broadcast=broadcast)
68
+ return self
69
+
70
+ def add_files(self, key: str, *path: str, broadcast=False, abs=False):
71
+ """
72
+ Add a variable with files by glob pattern
73
+ For example, suppose there are 3 files named 1.txt, 2.txt, 3.txt in data directory,
74
+ then calling add_files('DATA_FILE', 'data/*.txt') will add list ["data/1.txt", "data/2.txt", "data/3.txt"]
75
+ to the variable DATA_FILE.
76
+
77
+ :param key: Variable name
78
+ :param path: Path to files, can include glob pattern
79
+ :param broadcast: If True, values are broadcasted, otherwise they are producted when making combos
80
+ :param abs: If True, path will be turned into absolute path
81
+ """
82
+ args = expand_globs(path, raise_invalid=True)
83
+ if not args:
84
+ raise ValueError(f"No files found for {path}")
85
+ if abs:
86
+ args = [os.path.abspath(p) for p in args]
87
+ self.add_var(key, *args, broadcast=broadcast)
88
+ return self
89
+
90
+ def add_files_as_one(self, key: str, path: str, broadcast=False, sep=' ', abs=False):
91
+ """
92
+ Add a variable with files by glob pattern as one string
93
+ Unlike add_files, this function joins the files with a delimiter.
94
+ For example, suppose there are 1.txt, 2.txt, 3.txt in data directory,
95
+ then calling add_files_as_one('DATA_FILE', 'data/*.txt') will add string "data/1.txt data/2.txt data/3.txt"
96
+ to the variable DATA_FILE.
97
+
98
+ :param key: Variable name
99
+ :param path: Path to files, can include glob pattern
100
+ :param broadcast: If True, values are broadcasted, otherwise they are producted when making combos
101
+ :param sep: Separator to join files
102
+ :param abs: If True, path will be turned into absolute path
103
+ """
104
+ args = expand_globs(path, raise_invalid=True)
105
+ if not args:
106
+ raise ValueError(f"No files found for {path}")
107
+ if abs:
108
+ args = [os.path.abspath(p) for p in args]
109
+ self.add_var(key, sep.join(args), broadcast=broadcast)
110
+ return self
111
+
112
+ def add_var(self, key: str, *args, broadcast=False):
113
+ """
114
+ Add a variable with values
115
+
116
+ :param key: Variable name
117
+ :param args: Values
118
+ :param broadcast: If True, values are broadcasted, otherwise they are producted when making combos
119
+ """
120
+
121
+ if key == 'i':
122
+ raise ValueError("Variable name 'i' is reserved")
123
+
124
+ if broadcast:
125
+ if key in self._product_vars:
126
+ raise ValueError(f"Variable {key} already defined as product variable")
127
+ self._broadcast_vars.setdefault(key, []).extend(args)
128
+ else:
129
+ if key in self._broadcast_vars:
130
+ raise ValueError(f"Variable {key} already defined as broadcast variable")
131
+ self._product_vars.setdefault(key, []).extend(args)
132
+ return self
133
+
134
+ def shuffle(self, *keys: str, seed=None):
135
+ """
136
+ Shuffle variables
137
+ :param keys: Variable names to shuffle
138
+ :param seed: Seed for random number generator
139
+ """
140
+ if seed is not None:
141
+ random.seed(seed)
142
+
143
+ for key in keys:
144
+ if key in self._product_vars:
145
+ random.shuffle(self._product_vars[key])
146
+ elif key in self._broadcast_vars:
147
+ random.shuffle(self._broadcast_vars[key])
148
+ else:
149
+ raise ValueError(f"Variable {key} not found")
150
+ return self
151
+
152
+ def make_files(self, template: str, dest: str, delimiter='$', mode=None):
153
+ """
154
+ Make files from template
155
+ The template file can include variables with delimiter.
156
+ For example, if delimiter is '$', then the template file can include $var1, $var2, ...
157
+
158
+ The destination can also include variables in string format style.
159
+ For example, if dest is 'output/{i}.txt', then files are saved as output/0.txt, output/1.txt, ...
160
+
161
+ :param template: Path to template file
162
+ :param dest: Path pattern to destination file
163
+ :param delimiter: Delimiter for variables in template, default is '$',
164
+ can be changed to other character, e.g $$, @, ...
165
+ """
166
+ _delimiter = delimiter
167
+
168
+ class _Template(Template):
169
+ delimiter = _delimiter
170
+
171
+ combos = self._make_combos()
172
+ for i, combo in enumerate(combos):
173
+ with open(template, 'r') as f:
174
+ template_text = f.read()
175
+ text = _Template(template_text).safe_substitute(combo)
176
+ _dest = dest.format(i=i, **combo)
177
+ os.makedirs(os.path.dirname(_dest), exist_ok=True)
178
+ with open(_dest, 'w') as f:
179
+ f.write(text)
180
+ if mode is not None:
181
+ os.chmod(_dest, mode_translate(str(mode)))
182
+ return self
183
+
184
+ def done(self):
185
+ """
186
+ End of command chain
187
+ """
188
+ pass
189
+
190
+ def _make_combos(self):
191
+ if not self._product_vars and not self._broadcast_vars:
192
+ return self._combos
193
+ keys = self._product_vars.keys()
194
+ values_list = product(*self._product_vars.values())
195
+ combos = [ dict(zip(keys, values)) for values in values_list ]
196
+ for i, combo in enumerate(combos):
197
+ for k, v in self._broadcast_vars.items():
198
+ combo[k] = v[i % len(v)]
199
+ self._combos.extend(combos)
200
+ self._product_vars = {}
201
+ self._broadcast_vars = {}
202
+ return self._combos
oh_my_batch/job.py ADDED
@@ -0,0 +1,171 @@
1
+ from typing import List
2
+
3
+ import logging
4
+ import json
5
+ import time
6
+ import os
7
+ import re
8
+
9
+ from .util import expand_globs, shell_run, parse_csv
10
+
11
+
12
+ logger = logging.getLogger(__name__)
13
+
14
+
15
+ class JobState:
16
+ NULL = 0
17
+ PENDING = 1
18
+ RUNNING = 2
19
+ CANCELLED = 3
20
+ COMPLETED = 4
21
+ FAILED = 5
22
+ UNKNOWN = 6
23
+
24
+ @classmethod
25
+ def is_terminal(cls, state: int):
26
+ return state in (JobState.NULL, JobState.COMPLETED, JobState.FAILED, JobState.CANCELLED)
27
+
28
+ @classmethod
29
+ def is_success(cls, state: int):
30
+ return state == JobState.COMPLETED
31
+
32
+
33
+ def new_job(script: str):
34
+ return {
35
+ 'id': '', 'script': script, 'state': JobState.NULL, 'tries': 0,
36
+ }
37
+
38
+
39
+ class BaseJobManager:
40
+
41
+ def submit(self, *script: str, recovery: str = '', wait=False,
42
+ timeout=None, opts='', max_tries=1, interval=10):
43
+ """
44
+ Submit scripts
45
+
46
+ :param script: Script files to submit, can be glob patterns.
47
+ :param recovery: Recovery file to store the state of the submitted scripts
48
+ :param wait: If True, wait for the job to finish
49
+ :param timeout: Timeout in seconds for waiting
50
+ :param opts: Additional options for submit command
51
+ :param max_tries: Maximum number of tries for each job
52
+ :param interval: Interval in seconds for checking job status
53
+ """
54
+ jobs = []
55
+ if recovery and os.path.exists(recovery):
56
+ with open(recovery, 'r', encoding='utf-8') as f:
57
+ jobs = json.load(f)
58
+
59
+ recover_scripts = set(j['script'] for j in jobs)
60
+ logger.info('Scripts in recovery files: %s', recover_scripts)
61
+
62
+ scripts = set(os.path.normpath(s) for s in expand_globs(script))
63
+ logger.info('Scripts to submit: %s', scripts)
64
+
65
+ for script_file in scripts:
66
+ if script_file not in recover_scripts:
67
+ jobs.append(new_job(script_file))
68
+
69
+ current = time.time()
70
+ while True:
71
+ self._update_jobs(jobs, max_tries, opts)
72
+ if recovery:
73
+ with open(recovery, 'w', encoding='utf-8') as f:
74
+ json.dump(jobs, f, indent=2)
75
+
76
+ if not wait:
77
+ break
78
+
79
+ # stop if all jobs are terminal and not job to be submitted
80
+ if (all(JobState.is_terminal(j['state']) for j in jobs) and
81
+ not any(should_submit(j, max_tries) for j in jobs)):
82
+ break
83
+
84
+ if timeout and time.time() - current > timeout:
85
+ logger.error('Timeout, current state: %s', jobs)
86
+ break
87
+
88
+ time.sleep(interval)
89
+
90
+ def _update_jobs(self, jobs: List[dict], max_tries: int, submit_opts: str):
91
+ raise NotImplementedError
92
+
93
+
94
+ class Slurm(BaseJobManager):
95
+ def __init__(self, sbatch='sbatch', sacct='sacct'):
96
+ self._sbatch_bin = sbatch
97
+ self._sacct_bin = sacct
98
+
99
+ def _update_jobs(self, jobs: List[dict], max_tries: int, submit_opts: str):
100
+ # query job status
101
+ job_ids = [j['id'] for j in jobs if j['id']]
102
+ if job_ids:
103
+ query_cmd = f'{self._sacct_bin} -X -P --format=JobID,JobName,State -j {",".join(job_ids)}'
104
+ user = os.environ.get('USER')
105
+ if user:
106
+ query_cmd += f' -u {user}'
107
+
108
+ cp = shell_run(query_cmd)
109
+ if cp.returncode != 0:
110
+ logger.error('Failed to query job status: %s', cp.stderr.decode('utf-8'))
111
+ return jobs
112
+ logger.info('Job status:\n%s', cp.stdout.decode('utf-8'))
113
+ new_state = parse_csv(cp.stdout.decode('utf-8'))
114
+ else:
115
+ new_state = []
116
+
117
+ for job in jobs:
118
+ for row in new_state:
119
+ if job['id'] == row['JobID']:
120
+ job['state'] = self._map_state(row['State'])
121
+ if job['state'] == JobState.UNKNOWN:
122
+ logger.warning('Unknown job %s state: %s',row['JobID'], row['State'])
123
+ break
124
+ else:
125
+ if job['id']:
126
+ logger.error('Job %s not found in sacct output', job['id'])
127
+
128
+ # check if there are jobs to be (re)submitted
129
+ for job in jobs:
130
+ if should_submit(job, max_tries):
131
+ job['tries'] += 1
132
+ job['id'] = ''
133
+ job['state'] = JobState.NULL
134
+ submit_cmd = f'{self._sbatch_bin} {submit_opts} {job["script"]}'
135
+ cp = shell_run(submit_cmd)
136
+ if cp.returncode != 0:
137
+ job['state'] = JobState.FAILED
138
+ logger.error('Failed to submit job: %s', cp.stderr.decode('utf-8'))
139
+ else:
140
+ job['id'] = self._parse_job_id(cp.stdout.decode('utf-8'))
141
+ assert job['id'], 'Failed to parse job id'
142
+ job['state'] = JobState.PENDING
143
+ logger.info('Job %s submitted', job['id'])
144
+
145
+ def _map_state(self, state: str):
146
+ if state.startswith('CANCELLED'):
147
+ return JobState.CANCELLED
148
+ return {
149
+ 'PENDING': JobState.PENDING,
150
+ 'RUNNING': JobState.RUNNING,
151
+ 'COMPLETED': JobState.COMPLETED,
152
+ 'FAILED': JobState.FAILED,
153
+ 'OUT_OF_MEMORY': JobState.FAILED,
154
+ 'TIMEOUT': JobState.FAILED,
155
+ }.get(state, JobState.UNKNOWN)
156
+
157
+ def _parse_job_id(self, output: str):
158
+ """
159
+ Parse job id from sbatch output
160
+ """
161
+ m = re.search(r'\d+', output)
162
+ return m.group(0) if m else ''
163
+
164
+
165
+ def should_submit(job: dict, max_tries: int):
166
+ state: int = job['state']
167
+ if not JobState.is_terminal(state):
168
+ return False
169
+ if job['tries'] >= max_tries:
170
+ return False
171
+ return state != JobState.COMPLETED
oh_my_batch/util.py ADDED
@@ -0,0 +1,86 @@
1
+ from typing import List, Iterable
2
+ import subprocess as sp
3
+ import logging
4
+ import glob
5
+ import csv
6
+ import os
7
+
8
+
9
+ logger = logging.getLogger(__name__)
10
+
11
+
12
+ def expand_globs(patterns: Iterable[str], raise_invalid=False) -> List[str]:
13
+ """
14
+ Expand glob patterns in paths
15
+
16
+ :param patterns: list of paths or glob patterns
17
+ :param raise_invalid: if True, will raise error if no file found for a glob pattern
18
+ :return: list of expanded paths
19
+ """
20
+ paths = []
21
+ for pattern in patterns:
22
+ result = glob.glob(pattern, recursive=True) if '*' in pattern else [pattern]
23
+ if raise_invalid and len(result) == 0:
24
+ raise FileNotFoundError(f'No file found for {pattern}')
25
+ for p in result:
26
+ if p not in paths:
27
+ paths.append(p)
28
+ else:
29
+ logger.warning('path %s already exists in the list', p)
30
+ return paths
31
+
32
+
33
+ def split_list(l, n):
34
+ """
35
+ Splits a list into n sub-lists.
36
+
37
+ :param l: The list to be split.
38
+ :param n: The number of sub-lists to create.
39
+ :return: A list of sub-lists.
40
+ """
41
+ if n <= 0:
42
+ raise ValueError("Number of sub-lists must be a positive integer")
43
+
44
+ # Calculate the size of each sublist
45
+ k, m = divmod(len(l), n)
46
+
47
+ for i in range(n):
48
+ start = i * k + min(i, m)
49
+ end = (i + 1) * k + min(i + 1, m)
50
+ if start == end:
51
+ break
52
+ yield l[start:end]
53
+
54
+
55
+ def ensure_dir(path: str):
56
+ """
57
+ Ensure the directory exists
58
+
59
+ :param path: Path to directory or file.
60
+ """
61
+ os.makedirs(os.path.dirname(path), exist_ok=True)
62
+
63
+
64
+ def mode_translate(mode: str):
65
+ """
66
+ Translate mode in decimal to octal
67
+ For example, convert 777 -> 0o777, 755 -> 0o755
68
+ """
69
+ return int(mode, 8)
70
+
71
+
72
+ def shell_run(cmd: str):
73
+ """
74
+ Run a shell command
75
+
76
+ :param cmd: Command to run
77
+ """
78
+ return sp.run(cmd, shell=True, stdout=sp.PIPE, stderr=sp.PIPE)
79
+
80
+
81
+ def parse_csv(text: str, delimiter="|"):
82
+ """
83
+ Parse CSV text to list of dictionaries
84
+ """
85
+ reader = csv.DictReader(text.splitlines(), delimiter=delimiter)
86
+ return list(reader)
@@ -0,0 +1,135 @@
1
+ Metadata-Version: 2.1
2
+ Name: oh-my-batch
3
+ Version: 0.1.0
4
+ Summary:
5
+ License: GPL
6
+ Author: weihong.xu
7
+ Author-email: xuweihong.cn@gmail.com
8
+ Requires-Python: >=3.8,<4.0
9
+ Classifier: License :: Other/Proprietary License
10
+ Classifier: Programming Language :: Python :: 3
11
+ Classifier: Programming Language :: Python :: 3.8
12
+ Classifier: Programming Language :: Python :: 3.9
13
+ Classifier: Programming Language :: Python :: 3.10
14
+ Classifier: Programming Language :: Python :: 3.11
15
+ Classifier: Programming Language :: Python :: 3.12
16
+ Requires-Dist: fire (>=0.7.0,<0.8.0)
17
+ Description-Content-Type: text/markdown
18
+
19
+ # oh-my-batch
20
+
21
+ [![PyPI version](https://badge.fury.io/py/oh-my-batch.svg)](https://badge.fury.io/py/oh-my-batch)
22
+ [![PyPI - Downloads](https://img.shields.io/pypi/dm/oh-my-batch)](https://pypi.org/project/oh-my-batch/)
23
+ [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/oh-my-batch)](https://pypi.org/project/oh-my-batch/)
24
+
25
+ A simple tool to manipulate batch tasks designed for scientific computing community.
26
+
27
+ ## Features
28
+ * `omb combo`: generate folders/files from different combinations of parameters
29
+ * `omb batch`: generate batch scripts from multiple working directories
30
+ * `omb job`: track the state of job in job schedular
31
+
32
+ ## Install
33
+ ```bash
34
+ pip install oh-my-batch
35
+ ```
36
+
37
+ ## Use cases
38
+
39
+ ### Generate files from different combinations of parameters
40
+
41
+ It's common to generate files with different combinations of parameters in scientific computing.
42
+ For example, you have 3 LAMMPS data files in `tmp` directory: `tmp/1.data`, `tmp/2.data`, `tmp/3.data`.
43
+ And you want to generate a series of input files with different parameters,
44
+ for example, different temperatures 300K, 400K, 500K, against each data file.
45
+
46
+ In this case, you can use `omb combo` command to generate a series of input files for you.
47
+
48
+ ```bash
49
+ #! /bin/bash
50
+ # prepare fake data files
51
+ mkdir -p tmp/
52
+ touch tmp/1.data tmp/2.data tmp/3.data
53
+
54
+ # prepare a lammps input file template
55
+ cat > tmp/in.lmp.tmp <<EOF
56
+ read_data $DATA_FILE
57
+ velocity all create $TEMP $RANDOM
58
+ run 1000
59
+ EOF
60
+
61
+ # prepare a run script template
62
+ cat > tmp/run.sh.tmp <<EOF
63
+ cat in.lmp # simulate running lammps
64
+ EOF
65
+
66
+ # generate input files
67
+ omb combo \
68
+ add_files DATA_FILE tmp/*.data - \
69
+ add_var TEMP 300 400 500 - \
70
+ add_randint RANDOM -n 3 -a 1 -b 1000 --broadcast - \
71
+ make_files tmp/in.lmp.tmp tmp/tasks/{i}-T-{TEMP}/in.lmp - \
72
+ make_files tmp/run.sh.tmp tmp/tasks/{i}-T-{TEMP}/run.sh --mode 755 - \
73
+ done
74
+ ```
75
+
76
+ The above script will generate 9 folders in `tmp/tasks` directory
77
+ with names from `0-T-300`, `1-T-400`, `2-T-500`, `3-T-300` to `8-T-500`.
78
+ Each folder will contain a `in.lmp` file and a `run.sh` file.
79
+
80
+ The 9 folders are the combinations of 3 data files and 3 temperatures,
81
+ and each input file will have a independent random number between 1 and 1000 as `RANDOM`.
82
+
83
+ You can run the about script by `./examples/omb-combo.sh`,
84
+ and you can also run `omb combo --help` to see the detailed usage of `combo` command.
85
+
86
+ ### Generate batch scripts from multiple working directories
87
+ It's common to submit a lot of jobs to a job scheduler. `omb batch` is designed to help you generate batch scripts from multiple working directories and package them into several batch scripts.
88
+
89
+ Let's continue the above example, now you have 9 folders in `tmp/tasks` directory.
90
+ You want to package them into 2 batch scripts to submit to a job scheduler.
91
+
92
+ You can use `omb batch` to generate batch scripts for you like this:
93
+
94
+ ```bash
95
+ #! /bin/bash
96
+ cat > tmp/lammps_header.sh <<EOF
97
+ #!/bin/bash
98
+ #SBATCH -J lmp
99
+ #SBATCH -n 1
100
+ #SBATCH -t 1:00:00
101
+ EOF
102
+
103
+ omb batch \
104
+ add_work_dir tmp/tasks/* - \
105
+ add_header_file tmp/lammps_header.sh - \
106
+ add_command "checkpoint lmp.done ./run.sh" - \
107
+ make tmp/lmp-{i}.slurm --concurrency 2
108
+ ```
109
+
110
+ You will find batch scripts `tmp/lmp-0.slurm` and `tmp/lmp-1.slurm` in `tmp` directory.
111
+
112
+ `omb batch` will provide some useful functions in the batch script.
113
+ For example, `checkpoint` will check if the job is done and skip the job if it's done.
114
+
115
+ You can run the above script by `./examples/omb-batch.sh`,
116
+
117
+ ### Track the state of job in job schedular
118
+
119
+ Let's continue the above example, now you have submitted the batch scripts to the job scheduler.
120
+
121
+ You can use `omb job` to track the state of the jobs.
122
+
123
+ ```bash
124
+
125
+ omb job slurm \
126
+ submit tmp/*.slurm --max_tries 3 --wait --recovery lammps-jobs.json
127
+ ```
128
+
129
+ The above command will submit the batch scripts to the job scheduler,
130
+ and wait for the jobs to finish. If the job fails, it will retry for at most 3 times.
131
+
132
+ The `--recovery` option will save the job information to `lammps-jobs.json` file,
133
+ if `omb job` is interrupted, you can run the exact same command to recover the job status,
134
+ so that you don't need to resubmit the jobs that are already submitted.
135
+
@@ -0,0 +1,14 @@
1
+ oh_my_batch/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
2
+ oh_my_batch/__main__.py,sha256=sWyFZMwWNvhkanwZSJRGfBBDoIevhC028dTSB67i6yI,61
3
+ oh_my_batch/assets/__init__.py,sha256=Exub46UbQaz2V2eXpQeiVfnThQpXaNeuyjlGY6gBSZc,130
4
+ oh_my_batch/assets/functions.sh,sha256=eORxFefV-XrWbG-2I6u-c8uf1XxOQ31LaeVHBumwzJ4,708
5
+ oh_my_batch/batch.py,sha256=e73N-xwxMvgxnWwFMp33PQD1Dy-T-ATjANlwtPRHPQM,3016
6
+ oh_my_batch/cli.py,sha256=uelW9ms1N30DipJOcsiuG5K-5VN8O6yu1RNEqex00GY,475
7
+ oh_my_batch/combo.py,sha256=AHFD5CLoczqtjcfl2Rb4A2ucoQU40-cWtDOYjtP-yY4,7680
8
+ oh_my_batch/job.py,sha256=XcMEENcJLicvc0hzu5trpgi8dBI_w5TXfuYNoRAJleg,5768
9
+ oh_my_batch/util.py,sha256=H8B4zVNH5xRp-NG_uypgvtmz2YSpXy_6LK5ROv6SYrc,2116
10
+ oh_my_batch-0.1.0.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
11
+ oh_my_batch-0.1.0.dist-info/METADATA,sha256=XJYyoJtYg11hgu6JAkrUfc3BOKY2zLkE7xaDHITuM1g,4771
12
+ oh_my_batch-0.1.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
13
+ oh_my_batch-0.1.0.dist-info/entry_points.txt,sha256=ZY2GutSoNjjSyJ4qO2pTeseKUFgoTYdvmgkuZZkwi68,77
14
+ oh_my_batch-0.1.0.dist-info/RECORD,,
@@ -0,0 +1,4 @@
1
+ [console_scripts]
2
+ oh-my-batch=oh_my_batch.cli:main
3
+ omb=oh_my_batch.cli:main
4
+
@@ -1,20 +0,0 @@
1
- Metadata-Version: 2.1
2
- Name: oh-my-batch
3
- Version: 0.0.1.dev0
4
- Summary:
5
- License: GPL
6
- Author: weihong.xu
7
- Author-email: xuweihong.cn@gmail.com
8
- Requires-Python: >=3.8,<4.0
9
- Classifier: License :: Other/Proprietary License
10
- Classifier: Programming Language :: Python :: 3
11
- Classifier: Programming Language :: Python :: 3.8
12
- Classifier: Programming Language :: Python :: 3.9
13
- Classifier: Programming Language :: Python :: 3.10
14
- Classifier: Programming Language :: Python :: 3.11
15
- Classifier: Programming Language :: Python :: 3.12
16
- Description-Content-Type: text/markdown
17
-
18
- # oh-my-batch
19
- A simple tool to manipulate batch tasks.
20
-
@@ -1,5 +0,0 @@
1
- on_my_batch/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
2
- oh_my_batch-0.0.1.dev0.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
3
- oh_my_batch-0.0.1.dev0.dist-info/METADATA,sha256=j1kg17YPCOJs503Obz4RUb0vlaRLAZhM2BP68J5DcrA,614
4
- oh_my_batch-0.0.1.dev0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
5
- oh_my_batch-0.0.1.dev0.dist-info/RECORD,,
File without changes