oh-my-batch 0.2.1__tar.gz → 0.2.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: oh-my-batch
3
- Version: 0.2.1
3
+ Version: 0.2.3
4
4
  Summary:
5
5
  License: GPL
6
6
  Author: weihong.xu
@@ -13,6 +13,7 @@ Classifier: Programming Language :: Python :: 3.9
13
13
  Classifier: Programming Language :: Python :: 3.10
14
14
  Classifier: Programming Language :: Python :: 3.11
15
15
  Classifier: Programming Language :: Python :: 3.12
16
+ Classifier: Programming Language :: Python :: 3.13
16
17
  Requires-Dist: fire (>=0.7.0,<0.8.0)
17
18
  Description-Content-Type: text/markdown
18
19
 
@@ -28,27 +29,29 @@ A toolkit to manipulate batch tasks with command line. Designed for scientific c
28
29
  * `omb combo`: generate folders/files from different combinations of parameters
29
30
  * `omb batch`: generate batch scripts from multiple working directories
30
31
  * `omb job`: track the state of job in job schedular
31
-
32
- ## Shell Tips
33
- `oh-my-batch` is intended to help you implement computational workflows with shell scripts.
34
- To make the best use of `oh-my-batch`, you need to know some shell tips.
35
-
36
- * [Retry commands until success in shell script](https://stackoverflow.com/a/79191004/3099733)
32
+ * `omb misc`: miscellaneous commands
37
33
 
38
34
  ## Install
39
35
  ```bash
40
36
  pip install oh-my-batch
41
37
  ```
42
38
 
39
+ ## Shell tips
40
+ `oh-my-batch` is intended to help you implement computational workflows with shell scripts.
41
+ To make the best use of `oh-my-batch`, you need to know some shell tips.
42
+
43
+ * [Retry commands until success in shell script](https://stackoverflow.com/a/79191004/3099733)
44
+ * [Run multiple line shell script with ssh](https://stackoverflow.com/a/32082912/3099733)
45
+
43
46
  ## Use cases
44
- ### Load useful functions to your shell script
47
+ ### Load functions in shell script
45
48
  You can load useful functions from `oh-my-batch` this way:
46
49
 
47
50
  ```bash
48
- omb misc export-shell-func > omb-func.sh && source omb-func.sh
51
+ omb misc export-shell-func > omb-func.sh && source omb-func.sh && rm omb-func.sh
49
52
  ```
50
53
 
51
- This will load some useful functions to your shell script, for example, `checkpoint` function.
54
+ This will load some functions to your shell script, for example, `checkpoint`.
52
55
 
53
56
  ### Generate files from different combinations of parameters
54
57
 
@@ -66,8 +69,8 @@ touch tmp/1.data tmp/2.data tmp/3.data
66
69
 
67
70
  # prepare a lammps input file template
68
71
  cat > tmp/in.lmp.tmp <<EOF
69
- read_data $DATA_FILE
70
- velocity all create $TEMP $RANDOM
72
+ read_data @DATA_FILE
73
+ velocity all create @TEMP @RANDOM
71
74
  run 1000
72
75
  EOF
73
76
 
@@ -10,27 +10,29 @@ A toolkit to manipulate batch tasks with command line. Designed for scientific c
10
10
  * `omb combo`: generate folders/files from different combinations of parameters
11
11
  * `omb batch`: generate batch scripts from multiple working directories
12
12
  * `omb job`: track the state of job in job schedular
13
-
14
- ## Shell Tips
15
- `oh-my-batch` is intended to help you implement computational workflows with shell scripts.
16
- To make the best use of `oh-my-batch`, you need to know some shell tips.
17
-
18
- * [Retry commands until success in shell script](https://stackoverflow.com/a/79191004/3099733)
13
+ * `omb misc`: miscellaneous commands
19
14
 
20
15
  ## Install
21
16
  ```bash
22
17
  pip install oh-my-batch
23
18
  ```
24
19
 
20
+ ## Shell tips
21
+ `oh-my-batch` is intended to help you implement computational workflows with shell scripts.
22
+ To make the best use of `oh-my-batch`, you need to know some shell tips.
23
+
24
+ * [Retry commands until success in shell script](https://stackoverflow.com/a/79191004/3099733)
25
+ * [Run multiple line shell script with ssh](https://stackoverflow.com/a/32082912/3099733)
26
+
25
27
  ## Use cases
26
- ### Load useful functions to your shell script
28
+ ### Load functions in shell script
27
29
  You can load useful functions from `oh-my-batch` this way:
28
30
 
29
31
  ```bash
30
- omb misc export-shell-func > omb-func.sh && source omb-func.sh
32
+ omb misc export-shell-func > omb-func.sh && source omb-func.sh && rm omb-func.sh
31
33
  ```
32
34
 
33
- This will load some useful functions to your shell script, for example, `checkpoint` function.
35
+ This will load some functions to your shell script, for example, `checkpoint`.
34
36
 
35
37
  ### Generate files from different combinations of parameters
36
38
 
@@ -48,8 +50,8 @@ touch tmp/1.data tmp/2.data tmp/3.data
48
50
 
49
51
  # prepare a lammps input file template
50
52
  cat > tmp/in.lmp.tmp <<EOF
51
- read_data $DATA_FILE
52
- velocity all create $TEMP $RANDOM
53
+ read_data @DATA_FILE
54
+ velocity all create @TEMP @RANDOM
53
55
  run 1000
54
56
  EOF
55
57
 
@@ -1,9 +1,10 @@
1
1
  from itertools import product
2
2
  from string import Template
3
3
  import random
4
+ import json
4
5
  import os
5
6
 
6
- from .util import expand_globs, mode_translate
7
+ from .util import expand_globs, mode_translate, ensure_dir
7
8
 
8
9
  class ComboMaker:
9
10
 
@@ -67,7 +68,7 @@ class ComboMaker:
67
68
  self.add_var(key, *args, broadcast=broadcast)
68
69
  return self
69
70
 
70
- def add_files(self, key: str, *path: str, broadcast=False, abs=False):
71
+ def add_files(self, key: str, *path: str, broadcast=False, abs=False, raise_invalid=False):
71
72
  """
72
73
  Add a variable with files by glob pattern
73
74
  For example, suppose there are 3 files named 1.txt, 2.txt, 3.txt in data directory,
@@ -78,8 +79,9 @@ class ComboMaker:
78
79
  :param path: Path to files, can include glob pattern
79
80
  :param broadcast: If True, values are broadcasted, otherwise they are producted when making combos
80
81
  :param abs: If True, path will be turned into absolute path
82
+ :param raise_invalid: If True, will raise error if no file found for a glob pattern
81
83
  """
82
- args = expand_globs(path, raise_invalid=True)
84
+ args = expand_globs(path, raise_invalid=raise_invalid)
83
85
  if not args:
84
86
  raise ValueError(f"No files found for {path}")
85
87
  if abs:
@@ -87,7 +89,8 @@ class ComboMaker:
87
89
  self.add_var(key, *args, broadcast=broadcast)
88
90
  return self
89
91
 
90
- def add_files_as_one(self, key: str, path: str, broadcast=False, sep=' ', abs=False):
92
+ def add_files_as_one(self, key: str, *path: str, broadcast=False, format=None,
93
+ sep=' ', abs=False, raise_invalid=False):
91
94
  """
92
95
  Add a variable with files by glob pattern as one string
93
96
  Unlike add_files, this function joins the files with a delimiter.
@@ -98,15 +101,25 @@ class ComboMaker:
98
101
  :param key: Variable name
99
102
  :param path: Path to files, can include glob pattern
100
103
  :param broadcast: If True, values are broadcasted, otherwise they are producted when making combos
104
+ :param format: the way to format the files, can be None, 'json-list','json-item'
101
105
  :param sep: Separator to join files
102
106
  :param abs: If True, path will be turned into absolute path
107
+ :param raise_invalid: If True, will raise error if no file found for a glob pattern
103
108
  """
104
- args = expand_globs(path, raise_invalid=True)
109
+ args = expand_globs(path, raise_invalid=raise_invalid)
105
110
  if not args:
106
111
  raise ValueError(f"No files found for {path}")
107
112
  if abs:
108
113
  args = [os.path.abspath(p) for p in args]
109
- self.add_var(key, sep.join(args), broadcast=broadcast)
114
+ if format is None:
115
+ value = sep.join(args)
116
+ elif format == 'json-list':
117
+ value = json.dumps(args)
118
+ elif format == 'json-item':
119
+ value = json.dumps(args).strip('[]')
120
+ else:
121
+ raise ValueError(f"Invalid format: {format}")
122
+ self.add_var(key, value, broadcast=broadcast)
110
123
  return self
111
124
 
112
125
  def add_var(self, key: str, *args, broadcast=False):
@@ -148,19 +161,22 @@ class ComboMaker:
148
161
  raise ValueError(f"Variable {key} not found")
149
162
  return self
150
163
 
151
- def make_files(self, template: str, dest: str, delimiter='$', mode=None):
164
+ def make_files(self, template: str, dest: str, delimiter='@', mode=None, encoding='utf-8'):
152
165
  """
153
- Make files from template
166
+ Make files from template against each combo
154
167
  The template file can include variables with delimiter.
155
- For example, if delimiter is '$', then the template file can include $var1, $var2, ...
168
+ For example, if delimiter is '@', then the template file can include @var1, @var2, ...
156
169
 
157
170
  The destination can also include variables in string format style.
158
- For example, if dest is 'output/{i}.txt', then files are saved as output/0.txt, output/1.txt, ...
171
+ For example, if dest is 'output/{i}-{TEMP}.txt',
172
+ then files are saved as output/0-300K.txt, output/1-400K.txt, ...
159
173
 
160
174
  :param template: Path to template file
161
175
  :param dest: Path pattern to destination file
162
- :param delimiter: Delimiter for variables in template, default is '$',
163
- can be changed to other character, e.g $$, @, ...
176
+ :param delimiter: Delimiter for variables in template, default is '@', as '$' is popular in shell scripts
177
+ can be changed to other character, e.g $, $$, ...
178
+ :param mode: File mode, e.g. 755, 644, ...
179
+ :param encoding: File encoding
164
180
  """
165
181
  _delimiter = delimiter
166
182
 
@@ -173,12 +189,34 @@ class ComboMaker:
173
189
  template_text = f.read()
174
190
  text = _Template(template_text).safe_substitute(combo)
175
191
  _dest = dest.format(i=i, **combo)
176
- os.makedirs(os.path.dirname(_dest), exist_ok=True)
177
- with open(_dest, 'w') as f:
192
+ ensure_dir(_dest)
193
+ with open(_dest, 'w', encoding=encoding) as f:
178
194
  f.write(text)
179
195
  if mode is not None:
180
196
  os.chmod(_dest, mode_translate(str(mode)))
181
197
  return self
198
+
199
+ def print(self, *line: str, file: str = '', mode=None, encoding='utf-8'):
200
+ """
201
+ Print lines to a file against each combo
202
+
203
+ :param line: Lines to print, can include format style variables, e.g. {i}, {i:03d}, {TEMP}
204
+ :param file: File to save the output, if not provided, print to stdout
205
+ """
206
+ combos = self._make_combos()
207
+ out_lines = []
208
+ for i, combo in enumerate(combos):
209
+ out_lines.extend(l.format(i=i, **combo) for l in line)
210
+ out = '\n'.join(out_lines)
211
+ if file:
212
+ ensure_dir(file)
213
+ with open(file, 'w', encoding=encoding) as f:
214
+ f.write(out)
215
+ if mode is not None:
216
+ os.chmod(file, mode_translate(str(mode)))
217
+ else:
218
+ print(out)
219
+ return self
182
220
 
183
221
  def done(self):
184
222
  """
@@ -187,7 +225,7 @@ class ComboMaker:
187
225
  pass
188
226
 
189
227
  def _make_combos(self):
190
- if not self._product_vars and not self._broadcast_vars:
228
+ if not self._product_vars:
191
229
  return self._combos
192
230
  keys = self._product_vars.keys()
193
231
  values_list = product(*self._product_vars.values())
@@ -68,7 +68,7 @@ class BaseJobManager:
68
68
 
69
69
  current = time.time()
70
70
  while True:
71
- self._update_jobs(jobs, max_tries, opts)
71
+ jobs = self._update_jobs(jobs, max_tries, opts)
72
72
  if recovery:
73
73
  ensure_dir(recovery)
74
74
  with open(recovery, 'w', encoding='utf-8') as f:
@@ -77,7 +77,7 @@ class BaseJobManager:
77
77
  if not wait:
78
78
  break
79
79
 
80
- # stop if all jobs are terminal and not job to be submitted
80
+ # stop if all jobs are terminal and no job to be submitted
81
81
  if (all(JobState.is_terminal(j['state']) for j in jobs) and
82
82
  not any(should_submit(j, max_tries) for j in jobs)):
83
83
  break
@@ -88,7 +88,30 @@ class BaseJobManager:
88
88
 
89
89
  time.sleep(interval)
90
90
 
91
+
91
92
  def _update_jobs(self, jobs: List[dict], max_tries: int, submit_opts: str):
93
+ jobs = self._update_state(jobs)
94
+
95
+ # check if there are jobs to be (re)submitted
96
+ for job in jobs:
97
+ if should_submit(job, max_tries):
98
+ job['tries'] += 1
99
+ job['id'] = ''
100
+ job['state'] = JobState.NULL
101
+ job_id = self._submit_job(job, submit_opts)
102
+ if job_id:
103
+ job['state'] = JobState.PENDING
104
+ job['id'] = job_id
105
+ logger.info('Job %s submitted', job['id'])
106
+ else:
107
+ job['state'] = JobState.FAILED
108
+ logger.error('Failed to submit job %s', job['script'])
109
+ return jobs
110
+
111
+ def _update_state(self, jobs: List[dict]):
112
+ raise NotImplementedError
113
+
114
+ def _submit_job(self, job: dict, submit_opts: str):
92
115
  raise NotImplementedError
93
116
 
94
117
 
@@ -97,8 +120,7 @@ class Slurm(BaseJobManager):
97
120
  self._sbatch_bin = sbatch
98
121
  self._sacct_bin = sacct
99
122
 
100
- def _update_jobs(self, jobs: List[dict], max_tries: int, submit_opts: str):
101
- # query job status
123
+ def _update_state(self, jobs: List[dict]):
102
124
  job_ids = [j['id'] for j in jobs if j['id']]
103
125
  if job_ids:
104
126
  query_cmd = f'{self._sacct_bin} -X -P --format=JobID,JobName,State -j {",".join(job_ids)}'
@@ -106,8 +128,9 @@ class Slurm(BaseJobManager):
106
128
  if cp.returncode != 0:
107
129
  logger.error('Failed to query job status: %s', log_cp(cp))
108
130
  return jobs
109
- logger.info('Job status:\n%s', cp.stdout.decode('utf-8'))
110
- new_state = parse_csv(cp.stdout.decode('utf-8'))
131
+ out = cp.stdout.decode('utf-8')
132
+ logger.info('Job status:\n%s', out)
133
+ new_state = parse_csv(out)
111
134
  else:
112
135
  new_state = []
113
136
 
@@ -122,23 +145,19 @@ class Slurm(BaseJobManager):
122
145
  break
123
146
  else:
124
147
  logger.error('Job %s not found in sacct output', job['id'])
125
-
126
- # check if there are jobs to be (re)submitted
127
- for job in jobs:
128
- if should_submit(job, max_tries):
129
- job['tries'] += 1
130
- job['id'] = ''
131
- job['state'] = JobState.NULL
132
- submit_cmd = f'{self._sbatch_bin} {submit_opts} {job["script"]}'
133
- cp = shell_run(submit_cmd)
134
- if cp.returncode != 0:
135
- job['state'] = JobState.FAILED
136
- logger.error('Failed to submit job: %s', log_cp(cp))
137
- else:
138
- job['id'] = self._parse_job_id(cp.stdout.decode('utf-8'))
139
- assert job['id'], 'Failed to parse job id'
140
- job['state'] = JobState.PENDING
141
- logger.info('Job %s submitted', job['id'])
148
+ return jobs
149
+
150
+ def _submit_job(self, job:dict, submit_opts:str):
151
+ submit_cmd = f'{self._sbatch_bin} {submit_opts} {job["script"]}'
152
+ cp = shell_run(submit_cmd)
153
+ if cp.returncode != 0:
154
+ logger.error('Failed to submit job: %s', log_cp(cp))
155
+ return ''
156
+ out = cp.stdout.decode('utf-8')
157
+ job_id = self._parse_job_id(out)
158
+ if not job_id:
159
+ raise ValueError(f'Unexpected sbatch output: {out}')
160
+ return job_id
142
161
 
143
162
  def _map_state(self, state: str):
144
163
  if state.startswith('CANCELLED'):
@@ -1,6 +1,6 @@
1
1
  [tool.poetry]
2
2
  name = "oh-my-batch"
3
- version = "0.2.1"
3
+ version = "0.2.3"
4
4
  description = ""
5
5
  authors = ["weihong.xu <xuweihong.cn@gmail.com>"]
6
6
  license = "GPL"
File without changes