oh-my-batch 0.2.2__py3-none-any.whl → 0.2.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- oh_my_batch/cli.py +2 -1
- oh_my_batch/combo.py +30 -6
- oh_my_batch/job.py +42 -23
- oh_my_batch/util.py +3 -1
- {oh_my_batch-0.2.2.dist-info → oh_my_batch-0.2.4.dist-info}/METADATA +1 -2
- oh_my_batch-0.2.4.dist-info/RECORD +15 -0
- {oh_my_batch-0.2.2.dist-info → oh_my_batch-0.2.4.dist-info}/WHEEL +1 -1
- oh_my_batch-0.2.2.dist-info/RECORD +0 -15
- {oh_my_batch-0.2.2.dist-info → oh_my_batch-0.2.4.dist-info}/LICENSE +0 -0
- {oh_my_batch-0.2.2.dist-info → oh_my_batch-0.2.4.dist-info}/entry_points.txt +0 -0
oh_my_batch/cli.py
CHANGED
@@ -1,7 +1,6 @@
|
|
1
1
|
import logging
|
2
2
|
import fire
|
3
3
|
|
4
|
-
logging.basicConfig(format='%(asctime)s %(name)s: %(message)s', level=logging.INFO)
|
5
4
|
|
6
5
|
class JobCli:
|
7
6
|
|
@@ -27,5 +26,7 @@ class OhMyBatch:
|
|
27
26
|
from .misc import Misc
|
28
27
|
return Misc()
|
29
28
|
|
29
|
+
|
30
30
|
def main():
|
31
|
+
logging.basicConfig(format='%(asctime)s %(name)s: %(message)s', level=logging.INFO)
|
31
32
|
fire.Fire(OhMyBatch)
|
oh_my_batch/combo.py
CHANGED
@@ -4,7 +4,7 @@ import random
|
|
4
4
|
import json
|
5
5
|
import os
|
6
6
|
|
7
|
-
from .util import expand_globs, mode_translate
|
7
|
+
from .util import expand_globs, mode_translate, ensure_dir
|
8
8
|
|
9
9
|
class ComboMaker:
|
10
10
|
|
@@ -161,9 +161,9 @@ class ComboMaker:
|
|
161
161
|
raise ValueError(f"Variable {key} not found")
|
162
162
|
return self
|
163
163
|
|
164
|
-
def make_files(self, template: str, dest: str, delimiter='@', mode=None):
|
164
|
+
def make_files(self, template: str, dest: str, delimiter='@', mode=None, encoding='utf-8'):
|
165
165
|
"""
|
166
|
-
Make files from template
|
166
|
+
Make files from template against each combo
|
167
167
|
The template file can include variables with delimiter.
|
168
168
|
For example, if delimiter is '@', then the template file can include @var1, @var2, ...
|
169
169
|
|
@@ -175,6 +175,8 @@ class ComboMaker:
|
|
175
175
|
:param dest: Path pattern to destination file
|
176
176
|
:param delimiter: Delimiter for variables in template, default is '@', as '$' is popular in shell scripts
|
177
177
|
can be changed to other character, e.g $, $$, ...
|
178
|
+
:param mode: File mode, e.g. 755, 644, ...
|
179
|
+
:param encoding: File encoding
|
178
180
|
"""
|
179
181
|
_delimiter = delimiter
|
180
182
|
|
@@ -187,12 +189,34 @@ class ComboMaker:
|
|
187
189
|
template_text = f.read()
|
188
190
|
text = _Template(template_text).safe_substitute(combo)
|
189
191
|
_dest = dest.format(i=i, **combo)
|
190
|
-
|
191
|
-
with open(_dest, 'w') as f:
|
192
|
+
ensure_dir(_dest)
|
193
|
+
with open(_dest, 'w', encoding=encoding) as f:
|
192
194
|
f.write(text)
|
193
195
|
if mode is not None:
|
194
196
|
os.chmod(_dest, mode_translate(str(mode)))
|
195
197
|
return self
|
198
|
+
|
199
|
+
def print(self, *line: str, file: str = '', mode=None, encoding='utf-8'):
|
200
|
+
"""
|
201
|
+
Print lines to a file against each combo
|
202
|
+
|
203
|
+
:param line: Lines to print, can include format style variables, e.g. {i}, {i:03d}, {TEMP}
|
204
|
+
:param file: File to save the output, if not provided, print to stdout
|
205
|
+
"""
|
206
|
+
combos = self._make_combos()
|
207
|
+
out_lines = []
|
208
|
+
for i, combo in enumerate(combos):
|
209
|
+
out_lines.extend(l.format(i=i, **combo) for l in line)
|
210
|
+
out = '\n'.join(out_lines)
|
211
|
+
if file:
|
212
|
+
ensure_dir(file)
|
213
|
+
with open(file, 'w', encoding=encoding) as f:
|
214
|
+
f.write(out)
|
215
|
+
if mode is not None:
|
216
|
+
os.chmod(file, mode_translate(str(mode)))
|
217
|
+
else:
|
218
|
+
print(out)
|
219
|
+
return self
|
196
220
|
|
197
221
|
def done(self):
|
198
222
|
"""
|
@@ -201,7 +225,7 @@ class ComboMaker:
|
|
201
225
|
pass
|
202
226
|
|
203
227
|
def _make_combos(self):
|
204
|
-
if not self._product_vars
|
228
|
+
if not self._product_vars:
|
205
229
|
return self._combos
|
206
230
|
keys = self._product_vars.keys()
|
207
231
|
values_list = product(*self._product_vars.values())
|
oh_my_batch/job.py
CHANGED
@@ -68,7 +68,7 @@ class BaseJobManager:
|
|
68
68
|
|
69
69
|
current = time.time()
|
70
70
|
while True:
|
71
|
-
self._update_jobs(jobs, max_tries, opts)
|
71
|
+
jobs = self._update_jobs(jobs, max_tries, opts)
|
72
72
|
if recovery:
|
73
73
|
ensure_dir(recovery)
|
74
74
|
with open(recovery, 'w', encoding='utf-8') as f:
|
@@ -77,7 +77,7 @@ class BaseJobManager:
|
|
77
77
|
if not wait:
|
78
78
|
break
|
79
79
|
|
80
|
-
# stop if all jobs are terminal and
|
80
|
+
# stop if all jobs are terminal and no job to be submitted
|
81
81
|
if (all(JobState.is_terminal(j['state']) for j in jobs) and
|
82
82
|
not any(should_submit(j, max_tries) for j in jobs)):
|
83
83
|
break
|
@@ -88,7 +88,30 @@ class BaseJobManager:
|
|
88
88
|
|
89
89
|
time.sleep(interval)
|
90
90
|
|
91
|
+
|
91
92
|
def _update_jobs(self, jobs: List[dict], max_tries: int, submit_opts: str):
|
93
|
+
jobs = self._update_state(jobs)
|
94
|
+
|
95
|
+
# check if there are jobs to be (re)submitted
|
96
|
+
for job in jobs:
|
97
|
+
if should_submit(job, max_tries):
|
98
|
+
job['tries'] += 1
|
99
|
+
job['id'] = ''
|
100
|
+
job['state'] = JobState.NULL
|
101
|
+
job_id = self._submit_job(job, submit_opts)
|
102
|
+
if job_id:
|
103
|
+
job['state'] = JobState.PENDING
|
104
|
+
job['id'] = job_id
|
105
|
+
logger.info('Job %s submitted', job['id'])
|
106
|
+
else:
|
107
|
+
job['state'] = JobState.FAILED
|
108
|
+
logger.error('Failed to submit job %s', job['script'])
|
109
|
+
return jobs
|
110
|
+
|
111
|
+
def _update_state(self, jobs: List[dict]):
|
112
|
+
raise NotImplementedError
|
113
|
+
|
114
|
+
def _submit_job(self, job: dict, submit_opts: str):
|
92
115
|
raise NotImplementedError
|
93
116
|
|
94
117
|
|
@@ -97,8 +120,7 @@ class Slurm(BaseJobManager):
|
|
97
120
|
self._sbatch_bin = sbatch
|
98
121
|
self._sacct_bin = sacct
|
99
122
|
|
100
|
-
def
|
101
|
-
# query job status
|
123
|
+
def _update_state(self, jobs: List[dict]):
|
102
124
|
job_ids = [j['id'] for j in jobs if j['id']]
|
103
125
|
if job_ids:
|
104
126
|
query_cmd = f'{self._sacct_bin} -X -P --format=JobID,JobName,State -j {",".join(job_ids)}'
|
@@ -106,8 +128,9 @@ class Slurm(BaseJobManager):
|
|
106
128
|
if cp.returncode != 0:
|
107
129
|
logger.error('Failed to query job status: %s', log_cp(cp))
|
108
130
|
return jobs
|
109
|
-
|
110
|
-
|
131
|
+
out = cp.stdout.decode('utf-8')
|
132
|
+
logger.info('Job status:\n%s', out)
|
133
|
+
new_state = parse_csv(out)
|
111
134
|
else:
|
112
135
|
new_state = []
|
113
136
|
|
@@ -122,23 +145,19 @@ class Slurm(BaseJobManager):
|
|
122
145
|
break
|
123
146
|
else:
|
124
147
|
logger.error('Job %s not found in sacct output', job['id'])
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
job['id'] = self._parse_job_id(cp.stdout.decode('utf-8'))
|
139
|
-
assert job['id'], 'Failed to parse job id'
|
140
|
-
job['state'] = JobState.PENDING
|
141
|
-
logger.info('Job %s submitted', job['id'])
|
148
|
+
return jobs
|
149
|
+
|
150
|
+
def _submit_job(self, job:dict, submit_opts:str):
|
151
|
+
submit_cmd = f'{self._sbatch_bin} {submit_opts} {job["script"]}'
|
152
|
+
cp = shell_run(submit_cmd)
|
153
|
+
if cp.returncode != 0:
|
154
|
+
logger.error('Failed to submit job: %s', log_cp(cp))
|
155
|
+
return ''
|
156
|
+
out = cp.stdout.decode('utf-8')
|
157
|
+
job_id = self._parse_job_id(out)
|
158
|
+
if not job_id:
|
159
|
+
raise ValueError(f'Unexpected sbatch output: {out}')
|
160
|
+
return job_id
|
142
161
|
|
143
162
|
def _map_state(self, state: str):
|
144
163
|
if state.startswith('CANCELLED'):
|
oh_my_batch/util.py
CHANGED
@@ -58,7 +58,9 @@ def ensure_dir(path: str):
|
|
58
58
|
|
59
59
|
:param path: Path to directory or file.
|
60
60
|
"""
|
61
|
-
os.
|
61
|
+
d = os.path.dirname(path)
|
62
|
+
if d:
|
63
|
+
os.makedirs(os.path.dirname(d), exist_ok=True)
|
62
64
|
|
63
65
|
|
64
66
|
def mode_translate(mode: str):
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: oh-my-batch
|
3
|
-
Version: 0.2.
|
3
|
+
Version: 0.2.4
|
4
4
|
Summary:
|
5
5
|
License: GPL
|
6
6
|
Author: weihong.xu
|
@@ -13,7 +13,6 @@ Classifier: Programming Language :: Python :: 3.9
|
|
13
13
|
Classifier: Programming Language :: Python :: 3.10
|
14
14
|
Classifier: Programming Language :: Python :: 3.11
|
15
15
|
Classifier: Programming Language :: Python :: 3.12
|
16
|
-
Classifier: Programming Language :: Python :: 3.13
|
17
16
|
Requires-Dist: fire (>=0.7.0,<0.8.0)
|
18
17
|
Description-Content-Type: text/markdown
|
19
18
|
|
@@ -0,0 +1,15 @@
|
|
1
|
+
oh_my_batch/__init__.py,sha256=BsRNxZbqDWfaIZJGxzIDqCubRWztMGFDceW08TECuFs,98
|
2
|
+
oh_my_batch/__main__.py,sha256=sWyFZMwWNvhkanwZSJRGfBBDoIevhC028dTSB67i6yI,61
|
3
|
+
oh_my_batch/assets/__init__.py,sha256=Exub46UbQaz2V2eXpQeiVfnThQpXaNeuyjlGY6gBSZc,130
|
4
|
+
oh_my_batch/assets/functions.sh,sha256=dvPGpOKz4CyUSlE5IAewQc8HUrQJuaZ0j-WZIuxB3Tg,1002
|
5
|
+
oh_my_batch/batch.py,sha256=6qnaXEVyA493heGzzbCrdZXCcnYk8zgl7WP0rmo7KlU,3690
|
6
|
+
oh_my_batch/cli.py,sha256=Jyz8q2pUYke3mfJS6F_G9S9hApddgXxQw1BsN6Kfkjc,553
|
7
|
+
oh_my_batch/combo.py,sha256=-qgqT7XSvEeyefMPD3-HuX-aozefjSKhbZbPm1pm_18,9324
|
8
|
+
oh_my_batch/job.py,sha256=q-1WxZQxQNqDV4xZXz5V9RQQ787rhdX18tQWTNBKBjQ,6268
|
9
|
+
oh_my_batch/misc.py,sha256=G_iOovRCrShBJJCc82QLN0CvMqW4adOefEoY1GedEiw,452
|
10
|
+
oh_my_batch/util.py,sha256=K-XAndE5dgpxBrohAlUhVpeJGT7Q9vUG9ghHc9L0HaU,2426
|
11
|
+
oh_my_batch-0.2.4.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
|
12
|
+
oh_my_batch-0.2.4.dist-info/METADATA,sha256=6pn6dkc4pAN3FlH9erNTWKwYnOJl9Y-S7PsocwDc33c,5445
|
13
|
+
oh_my_batch-0.2.4.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
|
14
|
+
oh_my_batch-0.2.4.dist-info/entry_points.txt,sha256=ZY2GutSoNjjSyJ4qO2pTeseKUFgoTYdvmgkuZZkwi68,77
|
15
|
+
oh_my_batch-0.2.4.dist-info/RECORD,,
|
@@ -1,15 +0,0 @@
|
|
1
|
-
oh_my_batch/__init__.py,sha256=BsRNxZbqDWfaIZJGxzIDqCubRWztMGFDceW08TECuFs,98
|
2
|
-
oh_my_batch/__main__.py,sha256=sWyFZMwWNvhkanwZSJRGfBBDoIevhC028dTSB67i6yI,61
|
3
|
-
oh_my_batch/assets/__init__.py,sha256=Exub46UbQaz2V2eXpQeiVfnThQpXaNeuyjlGY6gBSZc,130
|
4
|
-
oh_my_batch/assets/functions.sh,sha256=dvPGpOKz4CyUSlE5IAewQc8HUrQJuaZ0j-WZIuxB3Tg,1002
|
5
|
-
oh_my_batch/batch.py,sha256=6qnaXEVyA493heGzzbCrdZXCcnYk8zgl7WP0rmo7KlU,3690
|
6
|
-
oh_my_batch/cli.py,sha256=jMSM_orAamNoEP7jPsbp0UYPX-UDY1-kHSHO31zELHs,548
|
7
|
-
oh_my_batch/combo.py,sha256=UQYyZAGFi-siUN13XVcpQwuqSZ0ZYPJl3hdVCORECng,8407
|
8
|
-
oh_my_batch/job.py,sha256=8kZnWtvpr1rAl4tc9I_Vlhi-T0o3rh4RQZZgMNBCGho,5800
|
9
|
-
oh_my_batch/misc.py,sha256=G_iOovRCrShBJJCc82QLN0CvMqW4adOefEoY1GedEiw,452
|
10
|
-
oh_my_batch/util.py,sha256=okg_kY8dJouyJ2BYCXRl7bxDUAtNH6GLh2UjXRnkoW0,2385
|
11
|
-
oh_my_batch-0.2.2.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
|
12
|
-
oh_my_batch-0.2.2.dist-info/METADATA,sha256=7Y9HPRolb4gljmDpHLaqkfdh1bfPYR4WdzFuh8DeKVw,5496
|
13
|
-
oh_my_batch-0.2.2.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
|
14
|
-
oh_my_batch-0.2.2.dist-info/entry_points.txt,sha256=ZY2GutSoNjjSyJ4qO2pTeseKUFgoTYdvmgkuZZkwi68,77
|
15
|
-
oh_my_batch-0.2.2.dist-info/RECORD,,
|
File without changes
|
File without changes
|