oh-my-batch 0.2.2__py3-none-any.whl → 0.2.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
oh_my_batch/combo.py CHANGED
@@ -4,7 +4,7 @@ import random
4
4
  import json
5
5
  import os
6
6
 
7
- from .util import expand_globs, mode_translate
7
+ from .util import expand_globs, mode_translate, ensure_dir
8
8
 
9
9
  class ComboMaker:
10
10
 
@@ -161,9 +161,9 @@ class ComboMaker:
161
161
  raise ValueError(f"Variable {key} not found")
162
162
  return self
163
163
 
164
- def make_files(self, template: str, dest: str, delimiter='@', mode=None):
164
+ def make_files(self, template: str, dest: str, delimiter='@', mode=None, encoding='utf-8'):
165
165
  """
166
- Make files from template
166
+ Make files from template against each combo
167
167
  The template file can include variables with delimiter.
168
168
  For example, if delimiter is '@', then the template file can include @var1, @var2, ...
169
169
 
@@ -175,6 +175,8 @@ class ComboMaker:
175
175
  :param dest: Path pattern to destination file
176
176
  :param delimiter: Delimiter for variables in template, default is '@', as '$' is popular in shell scripts
177
177
  can be changed to other character, e.g $, $$, ...
178
+ :param mode: File mode, e.g. 755, 644, ...
179
+ :param encoding: File encoding
178
180
  """
179
181
  _delimiter = delimiter
180
182
 
@@ -187,12 +189,34 @@ class ComboMaker:
187
189
  template_text = f.read()
188
190
  text = _Template(template_text).safe_substitute(combo)
189
191
  _dest = dest.format(i=i, **combo)
190
- os.makedirs(os.path.dirname(_dest), exist_ok=True)
191
- with open(_dest, 'w') as f:
192
+ ensure_dir(_dest)
193
+ with open(_dest, 'w', encoding=encoding) as f:
192
194
  f.write(text)
193
195
  if mode is not None:
194
196
  os.chmod(_dest, mode_translate(str(mode)))
195
197
  return self
198
+
199
+ def print(self, *line: str, file: str = '', mode=None, encoding='utf-8'):
200
+ """
201
+ Print lines to a file against each combo
202
+
203
+ :param line: Lines to print, can include format style variables, e.g. {i}, {i:03d}, {TEMP}
204
+ :param file: File to save the output, if not provided, print to stdout
205
+ """
206
+ combos = self._make_combos()
207
+ out_lines = []
208
+ for i, combo in enumerate(combos):
209
+ out_lines.extend(l.format(i=i, **combo) for l in line)
210
+ out = '\n'.join(out_lines)
211
+ if file:
212
+ ensure_dir(file)
213
+ with open(file, 'w', encoding=encoding) as f:
214
+ f.write(out)
215
+ if mode is not None:
216
+ os.chmod(file, mode_translate(str(mode)))
217
+ else:
218
+ print(out)
219
+ return self
196
220
 
197
221
  def done(self):
198
222
  """
@@ -201,7 +225,7 @@ class ComboMaker:
201
225
  pass
202
226
 
203
227
  def _make_combos(self):
204
- if not self._product_vars and not self._broadcast_vars:
228
+ if not self._product_vars:
205
229
  return self._combos
206
230
  keys = self._product_vars.keys()
207
231
  values_list = product(*self._product_vars.values())
oh_my_batch/job.py CHANGED
@@ -68,7 +68,7 @@ class BaseJobManager:
68
68
 
69
69
  current = time.time()
70
70
  while True:
71
- self._update_jobs(jobs, max_tries, opts)
71
+ jobs = self._update_jobs(jobs, max_tries, opts)
72
72
  if recovery:
73
73
  ensure_dir(recovery)
74
74
  with open(recovery, 'w', encoding='utf-8') as f:
@@ -77,7 +77,7 @@ class BaseJobManager:
77
77
  if not wait:
78
78
  break
79
79
 
80
- # stop if all jobs are terminal and not job to be submitted
80
+ # stop if all jobs are terminal and no job to be submitted
81
81
  if (all(JobState.is_terminal(j['state']) for j in jobs) and
82
82
  not any(should_submit(j, max_tries) for j in jobs)):
83
83
  break
@@ -88,7 +88,30 @@ class BaseJobManager:
88
88
 
89
89
  time.sleep(interval)
90
90
 
91
+
91
92
  def _update_jobs(self, jobs: List[dict], max_tries: int, submit_opts: str):
93
+ jobs = self._update_state(jobs)
94
+
95
+ # check if there are jobs to be (re)submitted
96
+ for job in jobs:
97
+ if should_submit(job, max_tries):
98
+ job['tries'] += 1
99
+ job['id'] = ''
100
+ job['state'] = JobState.NULL
101
+ job_id = self._submit_job(job, submit_opts)
102
+ if job_id:
103
+ job['state'] = JobState.PENDING
104
+ job['id'] = job_id
105
+ logger.info('Job %s submitted', job['id'])
106
+ else:
107
+ job['state'] = JobState.FAILED
108
+ logger.error('Failed to submit job %s', job['script'])
109
+ return jobs
110
+
111
+ def _update_state(self, jobs: List[dict]):
112
+ raise NotImplementedError
113
+
114
+ def _submit_job(self, job: dict, submit_opts: str):
92
115
  raise NotImplementedError
93
116
 
94
117
 
@@ -97,8 +120,7 @@ class Slurm(BaseJobManager):
97
120
  self._sbatch_bin = sbatch
98
121
  self._sacct_bin = sacct
99
122
 
100
- def _update_jobs(self, jobs: List[dict], max_tries: int, submit_opts: str):
101
- # query job status
123
+ def _update_state(self, jobs: List[dict]):
102
124
  job_ids = [j['id'] for j in jobs if j['id']]
103
125
  if job_ids:
104
126
  query_cmd = f'{self._sacct_bin} -X -P --format=JobID,JobName,State -j {",".join(job_ids)}'
@@ -106,8 +128,9 @@ class Slurm(BaseJobManager):
106
128
  if cp.returncode != 0:
107
129
  logger.error('Failed to query job status: %s', log_cp(cp))
108
130
  return jobs
109
- logger.info('Job status:\n%s', cp.stdout.decode('utf-8'))
110
- new_state = parse_csv(cp.stdout.decode('utf-8'))
131
+ out = cp.stdout.decode('utf-8')
132
+ logger.info('Job status:\n%s', out)
133
+ new_state = parse_csv(out)
111
134
  else:
112
135
  new_state = []
113
136
 
@@ -122,23 +145,19 @@ class Slurm(BaseJobManager):
122
145
  break
123
146
  else:
124
147
  logger.error('Job %s not found in sacct output', job['id'])
125
-
126
- # check if there are jobs to be (re)submitted
127
- for job in jobs:
128
- if should_submit(job, max_tries):
129
- job['tries'] += 1
130
- job['id'] = ''
131
- job['state'] = JobState.NULL
132
- submit_cmd = f'{self._sbatch_bin} {submit_opts} {job["script"]}'
133
- cp = shell_run(submit_cmd)
134
- if cp.returncode != 0:
135
- job['state'] = JobState.FAILED
136
- logger.error('Failed to submit job: %s', log_cp(cp))
137
- else:
138
- job['id'] = self._parse_job_id(cp.stdout.decode('utf-8'))
139
- assert job['id'], 'Failed to parse job id'
140
- job['state'] = JobState.PENDING
141
- logger.info('Job %s submitted', job['id'])
148
+ return jobs
149
+
150
+ def _submit_job(self, job:dict, submit_opts:str):
151
+ submit_cmd = f'{self._sbatch_bin} {submit_opts} {job["script"]}'
152
+ cp = shell_run(submit_cmd)
153
+ if cp.returncode != 0:
154
+ logger.error('Failed to submit job: %s', log_cp(cp))
155
+ return ''
156
+ out = cp.stdout.decode('utf-8')
157
+ job_id = self._parse_job_id(out)
158
+ if not job_id:
159
+ raise ValueError(f'Unexpected sbatch output: {out}')
160
+ return job_id
142
161
 
143
162
  def _map_state(self, state: str):
144
163
  if state.startswith('CANCELLED'):
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: oh-my-batch
3
- Version: 0.2.2
3
+ Version: 0.2.3
4
4
  Summary:
5
5
  License: GPL
6
6
  Author: weihong.xu
@@ -4,12 +4,12 @@ oh_my_batch/assets/__init__.py,sha256=Exub46UbQaz2V2eXpQeiVfnThQpXaNeuyjlGY6gBSZ
4
4
  oh_my_batch/assets/functions.sh,sha256=dvPGpOKz4CyUSlE5IAewQc8HUrQJuaZ0j-WZIuxB3Tg,1002
5
5
  oh_my_batch/batch.py,sha256=6qnaXEVyA493heGzzbCrdZXCcnYk8zgl7WP0rmo7KlU,3690
6
6
  oh_my_batch/cli.py,sha256=jMSM_orAamNoEP7jPsbp0UYPX-UDY1-kHSHO31zELHs,548
7
- oh_my_batch/combo.py,sha256=UQYyZAGFi-siUN13XVcpQwuqSZ0ZYPJl3hdVCORECng,8407
8
- oh_my_batch/job.py,sha256=8kZnWtvpr1rAl4tc9I_Vlhi-T0o3rh4RQZZgMNBCGho,5800
7
+ oh_my_batch/combo.py,sha256=-qgqT7XSvEeyefMPD3-HuX-aozefjSKhbZbPm1pm_18,9324
8
+ oh_my_batch/job.py,sha256=q-1WxZQxQNqDV4xZXz5V9RQQ787rhdX18tQWTNBKBjQ,6268
9
9
  oh_my_batch/misc.py,sha256=G_iOovRCrShBJJCc82QLN0CvMqW4adOefEoY1GedEiw,452
10
10
  oh_my_batch/util.py,sha256=okg_kY8dJouyJ2BYCXRl7bxDUAtNH6GLh2UjXRnkoW0,2385
11
- oh_my_batch-0.2.2.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
12
- oh_my_batch-0.2.2.dist-info/METADATA,sha256=7Y9HPRolb4gljmDpHLaqkfdh1bfPYR4WdzFuh8DeKVw,5496
13
- oh_my_batch-0.2.2.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
14
- oh_my_batch-0.2.2.dist-info/entry_points.txt,sha256=ZY2GutSoNjjSyJ4qO2pTeseKUFgoTYdvmgkuZZkwi68,77
15
- oh_my_batch-0.2.2.dist-info/RECORD,,
11
+ oh_my_batch-0.2.3.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
12
+ oh_my_batch-0.2.3.dist-info/METADATA,sha256=sRJJinfObbY4AVBpO1i86Ds2ognYtQ7upeJ4Q_q3WG8,5496
13
+ oh_my_batch-0.2.3.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
14
+ oh_my_batch-0.2.3.dist-info/entry_points.txt,sha256=ZY2GutSoNjjSyJ4qO2pTeseKUFgoTYdvmgkuZZkwi68,77
15
+ oh_my_batch-0.2.3.dist-info/RECORD,,