oh-my-batch 0.2.1__tar.gz → 0.2.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {oh_my_batch-0.2.1 → oh_my_batch-0.2.3}/PKG-INFO +15 -12
- {oh_my_batch-0.2.1 → oh_my_batch-0.2.3}/README.md +13 -11
- {oh_my_batch-0.2.1 → oh_my_batch-0.2.3}/oh_my_batch/combo.py +53 -15
- {oh_my_batch-0.2.1 → oh_my_batch-0.2.3}/oh_my_batch/job.py +42 -23
- {oh_my_batch-0.2.1 → oh_my_batch-0.2.3}/pyproject.toml +1 -1
- {oh_my_batch-0.2.1 → oh_my_batch-0.2.3}/LICENSE +0 -0
- {oh_my_batch-0.2.1 → oh_my_batch-0.2.3}/oh_my_batch/__init__.py +0 -0
- {oh_my_batch-0.2.1 → oh_my_batch-0.2.3}/oh_my_batch/__main__.py +0 -0
- {oh_my_batch-0.2.1 → oh_my_batch-0.2.3}/oh_my_batch/assets/__init__.py +0 -0
- {oh_my_batch-0.2.1 → oh_my_batch-0.2.3}/oh_my_batch/assets/functions.sh +0 -0
- {oh_my_batch-0.2.1 → oh_my_batch-0.2.3}/oh_my_batch/batch.py +0 -0
- {oh_my_batch-0.2.1 → oh_my_batch-0.2.3}/oh_my_batch/cli.py +0 -0
- {oh_my_batch-0.2.1 → oh_my_batch-0.2.3}/oh_my_batch/misc.py +0 -0
- {oh_my_batch-0.2.1 → oh_my_batch-0.2.3}/oh_my_batch/util.py +0 -0
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: oh-my-batch
|
3
|
-
Version: 0.2.
|
3
|
+
Version: 0.2.3
|
4
4
|
Summary:
|
5
5
|
License: GPL
|
6
6
|
Author: weihong.xu
|
@@ -13,6 +13,7 @@ Classifier: Programming Language :: Python :: 3.9
|
|
13
13
|
Classifier: Programming Language :: Python :: 3.10
|
14
14
|
Classifier: Programming Language :: Python :: 3.11
|
15
15
|
Classifier: Programming Language :: Python :: 3.12
|
16
|
+
Classifier: Programming Language :: Python :: 3.13
|
16
17
|
Requires-Dist: fire (>=0.7.0,<0.8.0)
|
17
18
|
Description-Content-Type: text/markdown
|
18
19
|
|
@@ -28,27 +29,29 @@ A toolkit to manipulate batch tasks with command line. Designed for scientific c
|
|
28
29
|
* `omb combo`: generate folders/files from different combinations of parameters
|
29
30
|
* `omb batch`: generate batch scripts from multiple working directories
|
30
31
|
* `omb job`: track the state of job in job schedular
|
31
|
-
|
32
|
-
## Shell Tips
|
33
|
-
`oh-my-batch` is intended to help you implement computational workflows with shell scripts.
|
34
|
-
To make the best use of `oh-my-batch`, you need to know some shell tips.
|
35
|
-
|
36
|
-
* [Retry commands until success in shell script](https://stackoverflow.com/a/79191004/3099733)
|
32
|
+
* `omb misc`: miscellaneous commands
|
37
33
|
|
38
34
|
## Install
|
39
35
|
```bash
|
40
36
|
pip install oh-my-batch
|
41
37
|
```
|
42
38
|
|
39
|
+
## Shell tips
|
40
|
+
`oh-my-batch` is intended to help you implement computational workflows with shell scripts.
|
41
|
+
To make the best use of `oh-my-batch`, you need to know some shell tips.
|
42
|
+
|
43
|
+
* [Retry commands until success in shell script](https://stackoverflow.com/a/79191004/3099733)
|
44
|
+
* [Run multiple line shell script with ssh](https://stackoverflow.com/a/32082912/3099733)
|
45
|
+
|
43
46
|
## Use cases
|
44
|
-
### Load
|
47
|
+
### Load functions in shell script
|
45
48
|
You can load useful functions from `oh-my-batch` this way:
|
46
49
|
|
47
50
|
```bash
|
48
|
-
omb misc export-shell-func > omb-func.sh && source omb-func.sh
|
51
|
+
omb misc export-shell-func > omb-func.sh && source omb-func.sh && rm omb-func.sh
|
49
52
|
```
|
50
53
|
|
51
|
-
This will load some
|
54
|
+
This will load some functions to your shell script, for example, `checkpoint`.
|
52
55
|
|
53
56
|
### Generate files from different combinations of parameters
|
54
57
|
|
@@ -66,8 +69,8 @@ touch tmp/1.data tmp/2.data tmp/3.data
|
|
66
69
|
|
67
70
|
# prepare a lammps input file template
|
68
71
|
cat > tmp/in.lmp.tmp <<EOF
|
69
|
-
read_data
|
70
|
-
velocity all create
|
72
|
+
read_data @DATA_FILE
|
73
|
+
velocity all create @TEMP @RANDOM
|
71
74
|
run 1000
|
72
75
|
EOF
|
73
76
|
|
@@ -10,27 +10,29 @@ A toolkit to manipulate batch tasks with command line. Designed for scientific c
|
|
10
10
|
* `omb combo`: generate folders/files from different combinations of parameters
|
11
11
|
* `omb batch`: generate batch scripts from multiple working directories
|
12
12
|
* `omb job`: track the state of job in job schedular
|
13
|
-
|
14
|
-
## Shell Tips
|
15
|
-
`oh-my-batch` is intended to help you implement computational workflows with shell scripts.
|
16
|
-
To make the best use of `oh-my-batch`, you need to know some shell tips.
|
17
|
-
|
18
|
-
* [Retry commands until success in shell script](https://stackoverflow.com/a/79191004/3099733)
|
13
|
+
* `omb misc`: miscellaneous commands
|
19
14
|
|
20
15
|
## Install
|
21
16
|
```bash
|
22
17
|
pip install oh-my-batch
|
23
18
|
```
|
24
19
|
|
20
|
+
## Shell tips
|
21
|
+
`oh-my-batch` is intended to help you implement computational workflows with shell scripts.
|
22
|
+
To make the best use of `oh-my-batch`, you need to know some shell tips.
|
23
|
+
|
24
|
+
* [Retry commands until success in shell script](https://stackoverflow.com/a/79191004/3099733)
|
25
|
+
* [Run multiple line shell script with ssh](https://stackoverflow.com/a/32082912/3099733)
|
26
|
+
|
25
27
|
## Use cases
|
26
|
-
### Load
|
28
|
+
### Load functions in shell script
|
27
29
|
You can load useful functions from `oh-my-batch` this way:
|
28
30
|
|
29
31
|
```bash
|
30
|
-
omb misc export-shell-func > omb-func.sh && source omb-func.sh
|
32
|
+
omb misc export-shell-func > omb-func.sh && source omb-func.sh && rm omb-func.sh
|
31
33
|
```
|
32
34
|
|
33
|
-
This will load some
|
35
|
+
This will load some functions to your shell script, for example, `checkpoint`.
|
34
36
|
|
35
37
|
### Generate files from different combinations of parameters
|
36
38
|
|
@@ -48,8 +50,8 @@ touch tmp/1.data tmp/2.data tmp/3.data
|
|
48
50
|
|
49
51
|
# prepare a lammps input file template
|
50
52
|
cat > tmp/in.lmp.tmp <<EOF
|
51
|
-
read_data
|
52
|
-
velocity all create
|
53
|
+
read_data @DATA_FILE
|
54
|
+
velocity all create @TEMP @RANDOM
|
53
55
|
run 1000
|
54
56
|
EOF
|
55
57
|
|
@@ -1,9 +1,10 @@
|
|
1
1
|
from itertools import product
|
2
2
|
from string import Template
|
3
3
|
import random
|
4
|
+
import json
|
4
5
|
import os
|
5
6
|
|
6
|
-
from .util import expand_globs, mode_translate
|
7
|
+
from .util import expand_globs, mode_translate, ensure_dir
|
7
8
|
|
8
9
|
class ComboMaker:
|
9
10
|
|
@@ -67,7 +68,7 @@ class ComboMaker:
|
|
67
68
|
self.add_var(key, *args, broadcast=broadcast)
|
68
69
|
return self
|
69
70
|
|
70
|
-
def add_files(self, key: str, *path: str, broadcast=False, abs=False):
|
71
|
+
def add_files(self, key: str, *path: str, broadcast=False, abs=False, raise_invalid=False):
|
71
72
|
"""
|
72
73
|
Add a variable with files by glob pattern
|
73
74
|
For example, suppose there are 3 files named 1.txt, 2.txt, 3.txt in data directory,
|
@@ -78,8 +79,9 @@ class ComboMaker:
|
|
78
79
|
:param path: Path to files, can include glob pattern
|
79
80
|
:param broadcast: If True, values are broadcasted, otherwise they are producted when making combos
|
80
81
|
:param abs: If True, path will be turned into absolute path
|
82
|
+
:param raise_invalid: If True, will raise error if no file found for a glob pattern
|
81
83
|
"""
|
82
|
-
args = expand_globs(path, raise_invalid=
|
84
|
+
args = expand_globs(path, raise_invalid=raise_invalid)
|
83
85
|
if not args:
|
84
86
|
raise ValueError(f"No files found for {path}")
|
85
87
|
if abs:
|
@@ -87,7 +89,8 @@ class ComboMaker:
|
|
87
89
|
self.add_var(key, *args, broadcast=broadcast)
|
88
90
|
return self
|
89
91
|
|
90
|
-
def add_files_as_one(self, key: str, path: str, broadcast=False,
|
92
|
+
def add_files_as_one(self, key: str, *path: str, broadcast=False, format=None,
|
93
|
+
sep=' ', abs=False, raise_invalid=False):
|
91
94
|
"""
|
92
95
|
Add a variable with files by glob pattern as one string
|
93
96
|
Unlike add_files, this function joins the files with a delimiter.
|
@@ -98,15 +101,25 @@ class ComboMaker:
|
|
98
101
|
:param key: Variable name
|
99
102
|
:param path: Path to files, can include glob pattern
|
100
103
|
:param broadcast: If True, values are broadcasted, otherwise they are producted when making combos
|
104
|
+
:param format: the way to format the files, can be None, 'json-list','json-item'
|
101
105
|
:param sep: Separator to join files
|
102
106
|
:param abs: If True, path will be turned into absolute path
|
107
|
+
:param raise_invalid: If True, will raise error if no file found for a glob pattern
|
103
108
|
"""
|
104
|
-
args = expand_globs(path, raise_invalid=
|
109
|
+
args = expand_globs(path, raise_invalid=raise_invalid)
|
105
110
|
if not args:
|
106
111
|
raise ValueError(f"No files found for {path}")
|
107
112
|
if abs:
|
108
113
|
args = [os.path.abspath(p) for p in args]
|
109
|
-
|
114
|
+
if format is None:
|
115
|
+
value = sep.join(args)
|
116
|
+
elif format == 'json-list':
|
117
|
+
value = json.dumps(args)
|
118
|
+
elif format == 'json-item':
|
119
|
+
value = json.dumps(args).strip('[]')
|
120
|
+
else:
|
121
|
+
raise ValueError(f"Invalid format: {format}")
|
122
|
+
self.add_var(key, value, broadcast=broadcast)
|
110
123
|
return self
|
111
124
|
|
112
125
|
def add_var(self, key: str, *args, broadcast=False):
|
@@ -148,19 +161,22 @@ class ComboMaker:
|
|
148
161
|
raise ValueError(f"Variable {key} not found")
|
149
162
|
return self
|
150
163
|
|
151
|
-
def make_files(self, template: str, dest: str, delimiter='
|
164
|
+
def make_files(self, template: str, dest: str, delimiter='@', mode=None, encoding='utf-8'):
|
152
165
|
"""
|
153
|
-
Make files from template
|
166
|
+
Make files from template against each combo
|
154
167
|
The template file can include variables with delimiter.
|
155
|
-
For example, if delimiter is '
|
168
|
+
For example, if delimiter is '@', then the template file can include @var1, @var2, ...
|
156
169
|
|
157
170
|
The destination can also include variables in string format style.
|
158
|
-
For example, if dest is 'output/{i}.txt',
|
171
|
+
For example, if dest is 'output/{i}-{TEMP}.txt',
|
172
|
+
then files are saved as output/0-300K.txt, output/1-400K.txt, ...
|
159
173
|
|
160
174
|
:param template: Path to template file
|
161
175
|
:param dest: Path pattern to destination file
|
162
|
-
:param delimiter: Delimiter for variables in template, default is '$'
|
163
|
-
can be changed to other character, e.g $$,
|
176
|
+
:param delimiter: Delimiter for variables in template, default is '@', as '$' is popular in shell scripts
|
177
|
+
can be changed to other character, e.g $, $$, ...
|
178
|
+
:param mode: File mode, e.g. 755, 644, ...
|
179
|
+
:param encoding: File encoding
|
164
180
|
"""
|
165
181
|
_delimiter = delimiter
|
166
182
|
|
@@ -173,12 +189,34 @@ class ComboMaker:
|
|
173
189
|
template_text = f.read()
|
174
190
|
text = _Template(template_text).safe_substitute(combo)
|
175
191
|
_dest = dest.format(i=i, **combo)
|
176
|
-
|
177
|
-
with open(_dest, 'w') as f:
|
192
|
+
ensure_dir(_dest)
|
193
|
+
with open(_dest, 'w', encoding=encoding) as f:
|
178
194
|
f.write(text)
|
179
195
|
if mode is not None:
|
180
196
|
os.chmod(_dest, mode_translate(str(mode)))
|
181
197
|
return self
|
198
|
+
|
199
|
+
def print(self, *line: str, file: str = '', mode=None, encoding='utf-8'):
|
200
|
+
"""
|
201
|
+
Print lines to a file against each combo
|
202
|
+
|
203
|
+
:param line: Lines to print, can include format style variables, e.g. {i}, {i:03d}, {TEMP}
|
204
|
+
:param file: File to save the output, if not provided, print to stdout
|
205
|
+
"""
|
206
|
+
combos = self._make_combos()
|
207
|
+
out_lines = []
|
208
|
+
for i, combo in enumerate(combos):
|
209
|
+
out_lines.extend(l.format(i=i, **combo) for l in line)
|
210
|
+
out = '\n'.join(out_lines)
|
211
|
+
if file:
|
212
|
+
ensure_dir(file)
|
213
|
+
with open(file, 'w', encoding=encoding) as f:
|
214
|
+
f.write(out)
|
215
|
+
if mode is not None:
|
216
|
+
os.chmod(file, mode_translate(str(mode)))
|
217
|
+
else:
|
218
|
+
print(out)
|
219
|
+
return self
|
182
220
|
|
183
221
|
def done(self):
|
184
222
|
"""
|
@@ -187,7 +225,7 @@ class ComboMaker:
|
|
187
225
|
pass
|
188
226
|
|
189
227
|
def _make_combos(self):
|
190
|
-
if not self._product_vars
|
228
|
+
if not self._product_vars:
|
191
229
|
return self._combos
|
192
230
|
keys = self._product_vars.keys()
|
193
231
|
values_list = product(*self._product_vars.values())
|
@@ -68,7 +68,7 @@ class BaseJobManager:
|
|
68
68
|
|
69
69
|
current = time.time()
|
70
70
|
while True:
|
71
|
-
self._update_jobs(jobs, max_tries, opts)
|
71
|
+
jobs = self._update_jobs(jobs, max_tries, opts)
|
72
72
|
if recovery:
|
73
73
|
ensure_dir(recovery)
|
74
74
|
with open(recovery, 'w', encoding='utf-8') as f:
|
@@ -77,7 +77,7 @@ class BaseJobManager:
|
|
77
77
|
if not wait:
|
78
78
|
break
|
79
79
|
|
80
|
-
# stop if all jobs are terminal and
|
80
|
+
# stop if all jobs are terminal and no job to be submitted
|
81
81
|
if (all(JobState.is_terminal(j['state']) for j in jobs) and
|
82
82
|
not any(should_submit(j, max_tries) for j in jobs)):
|
83
83
|
break
|
@@ -88,7 +88,30 @@ class BaseJobManager:
|
|
88
88
|
|
89
89
|
time.sleep(interval)
|
90
90
|
|
91
|
+
|
91
92
|
def _update_jobs(self, jobs: List[dict], max_tries: int, submit_opts: str):
|
93
|
+
jobs = self._update_state(jobs)
|
94
|
+
|
95
|
+
# check if there are jobs to be (re)submitted
|
96
|
+
for job in jobs:
|
97
|
+
if should_submit(job, max_tries):
|
98
|
+
job['tries'] += 1
|
99
|
+
job['id'] = ''
|
100
|
+
job['state'] = JobState.NULL
|
101
|
+
job_id = self._submit_job(job, submit_opts)
|
102
|
+
if job_id:
|
103
|
+
job['state'] = JobState.PENDING
|
104
|
+
job['id'] = job_id
|
105
|
+
logger.info('Job %s submitted', job['id'])
|
106
|
+
else:
|
107
|
+
job['state'] = JobState.FAILED
|
108
|
+
logger.error('Failed to submit job %s', job['script'])
|
109
|
+
return jobs
|
110
|
+
|
111
|
+
def _update_state(self, jobs: List[dict]):
|
112
|
+
raise NotImplementedError
|
113
|
+
|
114
|
+
def _submit_job(self, job: dict, submit_opts: str):
|
92
115
|
raise NotImplementedError
|
93
116
|
|
94
117
|
|
@@ -97,8 +120,7 @@ class Slurm(BaseJobManager):
|
|
97
120
|
self._sbatch_bin = sbatch
|
98
121
|
self._sacct_bin = sacct
|
99
122
|
|
100
|
-
def
|
101
|
-
# query job status
|
123
|
+
def _update_state(self, jobs: List[dict]):
|
102
124
|
job_ids = [j['id'] for j in jobs if j['id']]
|
103
125
|
if job_ids:
|
104
126
|
query_cmd = f'{self._sacct_bin} -X -P --format=JobID,JobName,State -j {",".join(job_ids)}'
|
@@ -106,8 +128,9 @@ class Slurm(BaseJobManager):
|
|
106
128
|
if cp.returncode != 0:
|
107
129
|
logger.error('Failed to query job status: %s', log_cp(cp))
|
108
130
|
return jobs
|
109
|
-
|
110
|
-
|
131
|
+
out = cp.stdout.decode('utf-8')
|
132
|
+
logger.info('Job status:\n%s', out)
|
133
|
+
new_state = parse_csv(out)
|
111
134
|
else:
|
112
135
|
new_state = []
|
113
136
|
|
@@ -122,23 +145,19 @@ class Slurm(BaseJobManager):
|
|
122
145
|
break
|
123
146
|
else:
|
124
147
|
logger.error('Job %s not found in sacct output', job['id'])
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
job['id'] = self._parse_job_id(cp.stdout.decode('utf-8'))
|
139
|
-
assert job['id'], 'Failed to parse job id'
|
140
|
-
job['state'] = JobState.PENDING
|
141
|
-
logger.info('Job %s submitted', job['id'])
|
148
|
+
return jobs
|
149
|
+
|
150
|
+
def _submit_job(self, job:dict, submit_opts:str):
|
151
|
+
submit_cmd = f'{self._sbatch_bin} {submit_opts} {job["script"]}'
|
152
|
+
cp = shell_run(submit_cmd)
|
153
|
+
if cp.returncode != 0:
|
154
|
+
logger.error('Failed to submit job: %s', log_cp(cp))
|
155
|
+
return ''
|
156
|
+
out = cp.stdout.decode('utf-8')
|
157
|
+
job_id = self._parse_job_id(out)
|
158
|
+
if not job_id:
|
159
|
+
raise ValueError(f'Unexpected sbatch output: {out}')
|
160
|
+
return job_id
|
142
161
|
|
143
162
|
def _map_state(self, state: str):
|
144
163
|
if state.startswith('CANCELLED'):
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|