oh-my-batch 0.4.7__py3-none-any.whl → 0.4.9__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- oh_my_batch/batch.py +9 -2
- oh_my_batch/job.py +20 -1
- {oh_my_batch-0.4.7.dist-info → oh_my_batch-0.4.9.dist-info}/METADATA +11 -19
- {oh_my_batch-0.4.7.dist-info → oh_my_batch-0.4.9.dist-info}/RECORD +7 -7
- {oh_my_batch-0.4.7.dist-info → oh_my_batch-0.4.9.dist-info}/WHEEL +1 -1
- {oh_my_batch-0.4.7.dist-info → oh_my_batch-0.4.9.dist-info}/LICENSE +0 -0
- {oh_my_batch-0.4.7.dist-info → oh_my_batch-0.4.9.dist-info}/entry_points.txt +0 -0
oh_my_batch/batch.py
CHANGED
@@ -79,15 +79,22 @@ class BatchMaker:
|
|
79
79
|
self._command.extend(cmd)
|
80
80
|
return self
|
81
81
|
|
82
|
-
def make(self, path: str, concurrency=
|
82
|
+
def make(self, path: str, concurrency=0, encoding='utf-8', mode='755'):
|
83
83
|
"""
|
84
84
|
Make batch script files from the previous setup
|
85
85
|
|
86
86
|
:param path: Path to save batch script files, use {i} to represent index
|
87
|
-
:param concurrency: Number of scripts to to make
|
87
|
+
:param concurrency: Number of scripts to to make, default is 0, which means make one script for each working directory
|
88
|
+
:param encoding: File encoding
|
89
|
+
:param mode: File mode, default is 755
|
88
90
|
"""
|
91
|
+
|
89
92
|
header = '\n'.join(self._script_header)
|
90
93
|
bottom = '\n'.join(self._script_bottom)
|
94
|
+
|
95
|
+
if concurrency < 1:
|
96
|
+
concurrency = len(self._work_dirs)
|
97
|
+
|
91
98
|
for i, work_dirs in enumerate(split_list(self._work_dirs, concurrency)):
|
92
99
|
body = []
|
93
100
|
work_dirs_arr = "\n".join(shlex.quote(w) for w in work_dirs)
|
oh_my_batch/job.py
CHANGED
@@ -94,6 +94,25 @@ class BaseJobManager:
|
|
94
94
|
error = True
|
95
95
|
if error:
|
96
96
|
raise RuntimeError('Some jobs failed')
|
97
|
+
|
98
|
+
def wait(self, *job_ids, timeout=None, interval=10):
|
99
|
+
"""
|
100
|
+
Wait for jobs to finish
|
101
|
+
|
102
|
+
:param job_ids: Job ids to wait for
|
103
|
+
:param timeout: Timeout in seconds
|
104
|
+
:param interval: Interval in seconds for checking job status
|
105
|
+
"""
|
106
|
+
current = time.time()
|
107
|
+
while True:
|
108
|
+
jobs = [{'id': j, 'state': JobState.NULL} for j in job_ids]
|
109
|
+
jobs = self._update_state(jobs)
|
110
|
+
if all(JobState.is_terminal(j['state']) for j in jobs):
|
111
|
+
break
|
112
|
+
if timeout and time.time() - current > timeout:
|
113
|
+
logger.error('Timeout, current state: %s', jobs)
|
114
|
+
break
|
115
|
+
time.sleep(interval)
|
97
116
|
|
98
117
|
def _update_jobs(self, jobs: List[dict], max_tries: int, submit_opts: str):
|
99
118
|
jobs = self._update_state(jobs)
|
@@ -135,7 +154,7 @@ class Slurm(BaseJobManager):
|
|
135
154
|
logger.error('Failed to query job status: %s', log_cp(cp))
|
136
155
|
return jobs
|
137
156
|
out = cp.stdout.decode('utf-8')
|
138
|
-
logger.
|
157
|
+
logger.debug('Job status:\n%s', out)
|
139
158
|
new_state = parse_csv(out)
|
140
159
|
else:
|
141
160
|
new_state = []
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: oh-my-batch
|
3
|
-
Version: 0.4.
|
3
|
+
Version: 0.4.9
|
4
4
|
Summary:
|
5
5
|
License: GPL
|
6
6
|
Author: weihong.xu
|
@@ -13,7 +13,6 @@ Classifier: Programming Language :: Python :: 3.9
|
|
13
13
|
Classifier: Programming Language :: Python :: 3.10
|
14
14
|
Classifier: Programming Language :: Python :: 3.11
|
15
15
|
Classifier: Programming Language :: Python :: 3.12
|
16
|
-
Classifier: Programming Language :: Python :: 3.13
|
17
16
|
Requires-Dist: fire (>=0.5.0,<0.6.0)
|
18
17
|
Description-Content-Type: text/markdown
|
19
18
|
|
@@ -28,7 +27,7 @@ A toolkit to manipulate batch tasks with command line. Designed for scientific c
|
|
28
27
|
## Features
|
29
28
|
* `omb combo`: generate folders/files from different combinations of parameters
|
30
29
|
* `omb batch`: generate batch scripts from multiple working directories
|
31
|
-
* `omb job`: track the state of job in job
|
30
|
+
* `omb job`: track the state of job in job scheduler
|
32
31
|
* `omb misc`: miscellaneous commands
|
33
32
|
|
34
33
|
## Install
|
@@ -36,24 +35,10 @@ A toolkit to manipulate batch tasks with command line. Designed for scientific c
|
|
36
35
|
pip install oh-my-batch
|
37
36
|
```
|
38
37
|
|
39
|
-
##
|
40
|
-
|
41
|
-
To make the best use of `oh-my-batch`, you need to know some shell tips.
|
42
|
-
|
43
|
-
* [Retry commands until success in shell script](https://stackoverflow.com/a/79191004/3099733)
|
44
|
-
* [Run multiple line shell script with ssh](https://stackoverflow.com/a/32082912/3099733)
|
38
|
+
## Examples
|
39
|
+
* [TESLA workflow](./examples/tesla/): A customizable active learning workflow for training machine learning potentials.
|
45
40
|
|
46
41
|
## Use cases
|
47
|
-
### Load functions in shell script
|
48
|
-
You can load useful functions from `oh-my-batch` this way:
|
49
|
-
|
50
|
-
```bash
|
51
|
-
eval "$(omb misc export-shell-func)"
|
52
|
-
# or
|
53
|
-
omb misc export-shell-func > omb-func.sh && source omb-func.sh
|
54
|
-
```
|
55
|
-
|
56
|
-
This will load extra functions to your shell script, for example, `checkpoint`.
|
57
42
|
|
58
43
|
### Generate files from different combinations of parameters
|
59
44
|
|
@@ -145,3 +130,10 @@ The `--recovery` option will save the job information to `lammps-jobs.json` file
|
|
145
130
|
If `omb job` is interrupted, you can rerun the exact same command to recover the job status,
|
146
131
|
so that you don't need to resubmit the jobs that are still running or completed.
|
147
132
|
|
133
|
+
|
134
|
+
## Shell tips
|
135
|
+
`oh-my-batch` is intended to help you implement computational workflows with shell scripts.
|
136
|
+
To make the best use of `oh-my-batch`, you need to know some shell tips.
|
137
|
+
|
138
|
+
* [Retry commands until success in shell script](https://stackoverflow.com/a/79191004/3099733)
|
139
|
+
* [Run multiple line shell script with ssh](https://stackoverflow.com/a/32082912/3099733)
|
@@ -2,14 +2,14 @@ oh_my_batch/__init__.py,sha256=BsRNxZbqDWfaIZJGxzIDqCubRWztMGFDceW08TECuFs,98
|
|
2
2
|
oh_my_batch/__main__.py,sha256=sWyFZMwWNvhkanwZSJRGfBBDoIevhC028dTSB67i6yI,61
|
3
3
|
oh_my_batch/assets/__init__.py,sha256=Exub46UbQaz2V2eXpQeiVfnThQpXaNeuyjlGY6gBSZc,130
|
4
4
|
oh_my_batch/assets/functions.sh,sha256=LaiavZBu84D7C1r-dWhf91vPTuHXCMV1DQZUIPVQnjE,1001
|
5
|
-
oh_my_batch/batch.py,sha256=
|
5
|
+
oh_my_batch/batch.py,sha256=BBrRlkTw9sLS4s3xmRGW-Ok3-qZi68_iaNpfDO7iYv4,3827
|
6
6
|
oh_my_batch/cli.py,sha256=Jyz8q2pUYke3mfJS6F_G9S9hApddgXxQw1BsN6Kfkjc,553
|
7
7
|
oh_my_batch/combo.py,sha256=A5flYsjugcelj105P6Nq4M3kNGtdqh3c7QaSIL3hUcg,9954
|
8
|
-
oh_my_batch/job.py,sha256=
|
8
|
+
oh_my_batch/job.py,sha256=8qFYscHGzIQhX-HrAGxCCv7h2T8Ls1Fj0gmNEBdP5Go,7227
|
9
9
|
oh_my_batch/misc.py,sha256=G_iOovRCrShBJJCc82QLN0CvMqW4adOefEoY1GedEiw,452
|
10
10
|
oh_my_batch/util.py,sha256=5ve2QcviuF0UHFLrsXmjMTj0ogXJ4g05q1y-yWCFuOk,2409
|
11
|
-
oh_my_batch-0.4.
|
12
|
-
oh_my_batch-0.4.
|
13
|
-
oh_my_batch-0.4.
|
14
|
-
oh_my_batch-0.4.
|
15
|
-
oh_my_batch-0.4.
|
11
|
+
oh_my_batch-0.4.9.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
|
12
|
+
oh_my_batch-0.4.9.dist-info/METADATA,sha256=nnVCB-BUnDfpNBHX5LUNeLMl7aWTlMZ3bOtbd1WdyVE,5174
|
13
|
+
oh_my_batch-0.4.9.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
|
14
|
+
oh_my_batch-0.4.9.dist-info/entry_points.txt,sha256=ZY2GutSoNjjSyJ4qO2pTeseKUFgoTYdvmgkuZZkwi68,77
|
15
|
+
oh_my_batch-0.4.9.dist-info/RECORD,,
|
File without changes
|
File without changes
|