rda-python-miscs 1.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rda_python_miscs/__init__.py +1 -0
- rda_python_miscs/bashqsub.py +213 -0
- rda_python_miscs/bashqsub.usg +64 -0
- rda_python_miscs/pgwget.py +186 -0
- rda_python_miscs/rdacp.py +212 -0
- rda_python_miscs/rdacp.usg +62 -0
- rda_python_miscs/rdakill.py +267 -0
- rda_python_miscs/rdakill.usg +36 -0
- rda_python_miscs/rdals.py +278 -0
- rda_python_miscs/rdals.usg +61 -0
- rda_python_miscs/rdamod.py +172 -0
- rda_python_miscs/rdamod.usg +51 -0
- rda_python_miscs/rdaown.py +169 -0
- rda_python_miscs/rdaown.usg +46 -0
- rda_python_miscs/rdaps.py +194 -0
- rda_python_miscs/rdaps.usg +29 -0
- rda_python_miscs/rdasub.py +117 -0
- rda_python_miscs/rdasub.usg +12 -0
- rda_python_miscs/rdazip.py +64 -0
- rda_python_miscs/rdazip.usg +20 -0
- rda_python_miscs/tcshqsub.py +213 -0
- rda_python_miscs/tcshqsub.usg +62 -0
- rda_python_miscs-1.0.1.dist-info/LICENSE +21 -0
- rda_python_miscs-1.0.1.dist-info/METADATA +17 -0
- rda_python_miscs-1.0.1.dist-info/RECORD +28 -0
- rda_python_miscs-1.0.1.dist-info/WHEEL +5 -0
- rda_python_miscs-1.0.1.dist-info/entry_points.txt +11 -0
- rda_python_miscs-1.0.1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
|
|
2
|
+
List matching process information for local or batch processes if they are running.
|
|
3
|
+
|
|
4
|
+
Usage: rdaps [-h HostName] [-p ProcessID] [-P ParentProcessID] \
|
|
5
|
+
[-u ProcessOnwer] [-a ApplicationName]
|
|
6
|
+
|
|
7
|
+
- Option -a, application name of the process;
|
|
8
|
+
|
|
9
|
+
- Option -h, hostname the process is on; omit it for local process,
|
|
10
|
+
but it is mandatory if the process id is a SLURM/PBS bactch id.
|
|
11
|
+
|
|
12
|
+
- Option -p, the local process or batch job id to be checked;
|
|
13
|
+
|
|
14
|
+
- Option -P, the parent process id on the local machine;
|
|
15
|
+
|
|
16
|
+
- Option -u, use login name for the process owner. It defaults to 'all' for
|
|
17
|
+
all user login names.
|
|
18
|
+
|
|
19
|
+
This utility program can be executed on selected machines. This help document
|
|
20
|
+
is displayed if no option is provided.
|
|
21
|
+
|
|
22
|
+
For examples, to list process information of a dsrqst process with pid 13199 locally,
|
|
23
|
+
|
|
24
|
+
rdaps -p 13199 -a dsrqst
|
|
25
|
+
|
|
26
|
+
To list a PBS bactch process with a bid = 334323,
|
|
27
|
+
|
|
28
|
+
rdaps -h PBS -p 334323
|
|
29
|
+
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
#
|
|
3
|
+
##################################################################################
|
|
4
|
+
#
|
|
5
|
+
# Title: rdasub
|
|
6
|
+
# Author: Zaihua Ji, zji@ucar.edu
|
|
7
|
+
# Date: 03/51/2021
|
|
8
|
+
# 2025-03-10 transferred to package rda_python_miscs from
|
|
9
|
+
# https://github.com/NCAR/rda-utility-programs.git
|
|
10
|
+
# Purpose: python script to submit a nohup bachground execution
|
|
11
|
+
#
|
|
12
|
+
# Github: https://github.com/NCAR/rda-python-miscs.git
|
|
13
|
+
#
|
|
14
|
+
##################################################################################
|
|
15
|
+
|
|
16
|
+
import os
|
|
17
|
+
import sys
|
|
18
|
+
import re
|
|
19
|
+
import time
|
|
20
|
+
from rda_python_common import PgLOG
|
|
21
|
+
from rda_python_common import PgFile
|
|
22
|
+
from rda_python_common import PgUtil
|
|
23
|
+
|
|
24
|
+
#
|
|
25
|
+
# main function to excecute this script
|
|
26
|
+
#
|
|
27
|
+
def main():
|
|
28
|
+
|
|
29
|
+
aname = 'rdasub'
|
|
30
|
+
PgLOG.set_help_path(__file__)
|
|
31
|
+
coptions = {'cmd' : None, 'cwd' : None, 'env' : None} # customized options
|
|
32
|
+
copts = '|'.join(coptions)
|
|
33
|
+
option = None
|
|
34
|
+
argv = sys.argv[1:]
|
|
35
|
+
if not argv: PgLOG.show_usage(aname)
|
|
36
|
+
PgLOG.PGLOG['LOGFILE'] = aname + ".log"
|
|
37
|
+
PgLOG.cmdlog("{} {}".format(aname, ' '.join(argv)))
|
|
38
|
+
|
|
39
|
+
while argv:
|
|
40
|
+
arg = argv.pop(0)
|
|
41
|
+
if arg == "-b":
|
|
42
|
+
PgLOG.PGLOG['BCKGRND'] = 1
|
|
43
|
+
option = None
|
|
44
|
+
continue
|
|
45
|
+
ms = re.match(r'^-({})$'.format(copts), arg)
|
|
46
|
+
if ms:
|
|
47
|
+
option = ms.group(1)
|
|
48
|
+
continue
|
|
49
|
+
if not option: PgLOG.pglog("{}: Value passed in without leading option for {}".format(arg, aname), PgLOG.LGEREX)
|
|
50
|
+
if arg.find(' ') > -1 and not re.match(r'^[\'\"].*[\'\"]$', arg): # quote string with space but not quoted yet
|
|
51
|
+
if arg.find("'") > -1:
|
|
52
|
+
arg = '"{}"'.format(arg)
|
|
53
|
+
else:
|
|
54
|
+
arg = "'{}'".format(arg)
|
|
55
|
+
|
|
56
|
+
coptions[option] = arg
|
|
57
|
+
if option == "cmd": break
|
|
58
|
+
option = None
|
|
59
|
+
|
|
60
|
+
if not coptions['cmd']: PgLOG.pglog(aname + ": specify command via option -cmd to run", PgLOG.LGWNEX)
|
|
61
|
+
args = PgLOG.argv_to_string(argv, 0) # append command options
|
|
62
|
+
msg = "{}-{}{}".format(PgLOG.PGLOG['HOSTNAME'], PgLOG.PGLOG['CURUID'], PgLOG.current_datetime())
|
|
63
|
+
if coptions['cwd']:
|
|
64
|
+
if coptions['cwd'].find('$'): coptions['cwd'] = PgLOG.replace_environments(coptions['cwd'], '', PgLOG.LGWNEX)
|
|
65
|
+
msg += "-" + coptions['cwd']
|
|
66
|
+
PgFile.change_local_directory(coptions['cwd'], PgLOG.LGEREX)
|
|
67
|
+
else:
|
|
68
|
+
coptions['cwd'] = PgLOG.PGLOG['CURDIR']
|
|
69
|
+
cmd = PgLOG.valid_command(coptions['cmd'])
|
|
70
|
+
if not cmd and not re.match(r'^/', coptions['cmd']): cmd = PgLOG.valid_command('./' + coptions['cmd'])
|
|
71
|
+
if not cmd: PgLOG.pglog(coptions['cmd'] + ": Cannot find given command to run", PgLOG.LGWNEX)
|
|
72
|
+
if args: cmd += " " + args
|
|
73
|
+
|
|
74
|
+
msg += ": " + cmd
|
|
75
|
+
PgLOG.pglog(msg, PgLOG.LOGWRN)
|
|
76
|
+
os.system("nohup " + cmd + " > /dev/null 2>&1 &")
|
|
77
|
+
display_process_info(coptions['cmd'], cmd)
|
|
78
|
+
|
|
79
|
+
sys.exit(0)
|
|
80
|
+
|
|
81
|
+
#
|
|
82
|
+
# display the the most recent matching process info
|
|
83
|
+
#
|
|
84
|
+
def display_process_info(cname, cmd):
|
|
85
|
+
|
|
86
|
+
ctime = time.time()
|
|
87
|
+
RTIME = PID = 0
|
|
88
|
+
pscmd = "ps -u {},{} -f | grep {} | grep ' 1 ' | grep -v ' grep '".format(PgLOG.PGLOG['CURUID'], PgLOG.PGLOG['RDAUSER'], cname)
|
|
89
|
+
|
|
90
|
+
for i in range(2):
|
|
91
|
+
buf = PgLOG.pgsystem(pscmd, PgLOG.LOGWRN, 20)
|
|
92
|
+
if buf:
|
|
93
|
+
lines = buf.split("\n")
|
|
94
|
+
for line in lines:
|
|
95
|
+
mp = "\s+(\d+)\s+1\s+.*\s(\d+:\d+)\s.*{}\S*\s*(.*)$".format(cname)
|
|
96
|
+
ms = re.search(mp, line)
|
|
97
|
+
if ms:
|
|
98
|
+
pid = ms.group(1)
|
|
99
|
+
rtm = ms.group(2)
|
|
100
|
+
arg = ms.group(3)
|
|
101
|
+
if not arg or cmd.find(arg) > -1:
|
|
102
|
+
rtime = PgUtil.unixtime(rtm + ':00')
|
|
103
|
+
if rtime > ctime: rtime -= 24*60*60
|
|
104
|
+
if rtime > RTIME:
|
|
105
|
+
PID = pid
|
|
106
|
+
RTIME = rtime
|
|
107
|
+
if PID:
|
|
108
|
+
return PgLOG.pglog("Job <{}> is submitted to background <{}>".format(PID, PgLOG.PgLOG['HOSTNAME']), PgLOG.LOGWRN)
|
|
109
|
+
elif i == 0:
|
|
110
|
+
time.sleep(2)
|
|
111
|
+
else:
|
|
112
|
+
return PgLOG.pglog("{}: No job information found, It may have finished".format(cmd), PgLOG.LOGWRN)
|
|
113
|
+
|
|
114
|
+
#
|
|
115
|
+
# call main() to start program
|
|
116
|
+
#
|
|
117
|
+
if __name__ == "__main__": main()
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
|
|
2
|
+
To submit a nohup background execution job on a Linux machine.
|
|
3
|
+
|
|
4
|
+
Usage: rdasub [-env EnvironmentPairs] [-cwd WorkDir] -cmd Command [cmd-options]
|
|
5
|
+
|
|
6
|
+
- Option -cwd, set the working directory a Command to be executed;
|
|
7
|
+
|
|
8
|
+
- Option -cmd, mandatory option to lead a Command to be executed;
|
|
9
|
+
|
|
10
|
+
- Option -env, set environment name/value pairs separated by ',';
|
|
11
|
+
|
|
12
|
+
- cmd-options, specifies options that can be passed to the Command.
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
#
|
|
3
|
+
##################################################################################
|
|
4
|
+
#
|
|
5
|
+
# Title: rdazip
|
|
6
|
+
# Author: Zaihua Ji, zji@ucar.edu
|
|
7
|
+
# Date: 10/24/2020
|
|
8
|
+
# 2025-03-17 transferred to package rda_python_miscs from
|
|
9
|
+
# https://github.com/NCAR/rda-utility-programs.git
|
|
10
|
+
# Purpose: compress/uncompress given file names
|
|
11
|
+
#
|
|
12
|
+
# Github: https://github.com/NCAR/rda-python-miscs.git
|
|
13
|
+
#
|
|
14
|
+
##################################################################################
|
|
15
|
+
|
|
16
|
+
import re
|
|
17
|
+
import os
|
|
18
|
+
import sys
|
|
19
|
+
from rda_python_common import PgLOG
|
|
20
|
+
from rda_python_common import PgFile
|
|
21
|
+
|
|
22
|
+
#
|
|
23
|
+
# main function to run the application
|
|
24
|
+
#
|
|
25
|
+
def main():
|
|
26
|
+
|
|
27
|
+
act = 0
|
|
28
|
+
argv = sys.argv[1:]
|
|
29
|
+
PgLOG.set_help_path(__file__)
|
|
30
|
+
PgLOG.PGLOG['LOGFILE'] = "rdazip.log" # set different log file
|
|
31
|
+
PgLOG.cmdlog("rdazip {}".format(' '.join(argv)))
|
|
32
|
+
files = []
|
|
33
|
+
fmt = option = None
|
|
34
|
+
for arg in argv:
|
|
35
|
+
ms = re.match(r'-(\w+)$', arg)
|
|
36
|
+
if ms:
|
|
37
|
+
option = ms.group(1)
|
|
38
|
+
if option == "b":
|
|
39
|
+
PgLOG.PGLOG['BCKGRND'] = 1
|
|
40
|
+
option = None
|
|
41
|
+
elif option == "f":
|
|
42
|
+
act = 1
|
|
43
|
+
else:
|
|
44
|
+
PgLOG.pglog(arg + ": Unknown Option", PgLOG.LGEREX)
|
|
45
|
+
elif option:
|
|
46
|
+
if fmt: PgLOG.pglog("{}: compression format '{}' provided already".format(arg, fmt), PgLOG.LGEREX)
|
|
47
|
+
fmt = arg
|
|
48
|
+
if not files: option = None
|
|
49
|
+
else:
|
|
50
|
+
if not os.path.isfile(arg): PgLOG.pglog(arg + ": file not exists", PgLOG.LGEREX)
|
|
51
|
+
files.append(arg)
|
|
52
|
+
|
|
53
|
+
if not files: PgLOG.show_usage("rdazip")
|
|
54
|
+
|
|
55
|
+
for file in files:
|
|
56
|
+
PgFile.compress_local_file(file, fmt, act, PgLOG.LGWNEX)
|
|
57
|
+
|
|
58
|
+
PgLOG.cmdlog()
|
|
59
|
+
sys.exit(0)
|
|
60
|
+
|
|
61
|
+
#
|
|
62
|
+
# call main() to start program
|
|
63
|
+
#
|
|
64
|
+
if __name__ == "__main__": main()
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
|
|
2
|
+
Compress/Uncompress given files.
|
|
3
|
+
|
|
4
|
+
Usage: rdazip [-f CompressFormat] FileList
|
|
5
|
+
|
|
6
|
+
- Option -f, compression format, Z, gz, zip or bz2. Specify this format
|
|
7
|
+
to compress, otherwise to umcompress
|
|
8
|
+
|
|
9
|
+
For examples, to compress a file using gzip, testfile.txt => testfile.txt.gz
|
|
10
|
+
|
|
11
|
+
rdazip -f gz testfile.txt
|
|
12
|
+
|
|
13
|
+
and to uncompress a gzipped file, testfile.txt.gz => get testfile.txt
|
|
14
|
+
|
|
15
|
+
rdazip testfile.txt.gz
|
|
16
|
+
|
|
17
|
+
and to change compression from Z to gz, testfile.txt.Z => testfile.txt.gz
|
|
18
|
+
|
|
19
|
+
rdazip -f gz testfile.txt.Z
|
|
20
|
+
|
|
@@ -0,0 +1,213 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
#
|
|
3
|
+
##################################################################################
|
|
4
|
+
#
|
|
5
|
+
# Title: tcshqsub
|
|
6
|
+
# Author: Zaihua Ji, zji@ucar.edu
|
|
7
|
+
# Date: 11/19/2020
|
|
8
|
+
# 2025-03-07 transferred to package rda_python_miscs from
|
|
9
|
+
# https://github.com/NCAR/rda-utility-programs.git
|
|
10
|
+
# Purpose: python script to submit a batch job on PBS node via tcsh script
|
|
11
|
+
#
|
|
12
|
+
# Github: https://github.com/NCAR/rda-python-miscs.git
|
|
13
|
+
#
|
|
14
|
+
##################################################################################
|
|
15
|
+
|
|
16
|
+
import os
|
|
17
|
+
import sys
|
|
18
|
+
import re
|
|
19
|
+
from os import path as op
|
|
20
|
+
from rda_python_common import PgLOG
|
|
21
|
+
|
|
22
|
+
DEFMODS = {
|
|
23
|
+
'default' : "ncarenv,netcdf,ncl,nco,cdo,conda,grib-util,wgrib2",
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
DEFLIBS = {
|
|
27
|
+
'default' : "conda activate /glade/work/rdadata/conda-envs/pg-casper",
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
SWAPMODS = {
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
RESOURCES = { # resource list for option -l
|
|
34
|
+
'walltime' : '6:00:00', # if this is changed, change defpbstime in PgCheck.py too
|
|
35
|
+
'select' : '1:ncpus=1:mem=1gb'
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
SOPTIONS = { # single-dash option values
|
|
39
|
+
'o' : None, # will set to default if not provided
|
|
40
|
+
'e' : None,
|
|
41
|
+
'A' : "P43713000",
|
|
42
|
+
'q' : "rda@casper-pbs",
|
|
43
|
+
# 'm' : 'a',
|
|
44
|
+
'm' : 'n',
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
#
|
|
48
|
+
# main function to excecute this script
|
|
49
|
+
#
|
|
50
|
+
def main():
|
|
51
|
+
|
|
52
|
+
aname = 'tcshqsub'
|
|
53
|
+
pname = 'rdaqsub'
|
|
54
|
+
PgLOG.set_help_path(__file__)
|
|
55
|
+
rdasub = PgLOG.BCHCMDS['PBS']
|
|
56
|
+
coptions = {'cmd' : None, 'cwd' : None, 'env' : None, 'mod' : None, 'res' : 'default'} # customized options
|
|
57
|
+
copts = '|'.join(coptions)
|
|
58
|
+
option = None
|
|
59
|
+
dcount = 0
|
|
60
|
+
argv = sys.argv[1:]
|
|
61
|
+
if not argv: PgLOG.show_usage(aname)
|
|
62
|
+
PgLOG.PGLOG['LOGFILE'] = pname + ".log"
|
|
63
|
+
PgLOG.cmdlog("{} {}".format(aname, ' '.join(argv)))
|
|
64
|
+
if not PgLOG.valid_command(rdasub): PgLOG.pglog("{}: miss {} command to submit batch job".format(rdasub, PgLOG.PGLOG['PBSNAME']), PgLOG.LGWNEX)
|
|
65
|
+
|
|
66
|
+
while argv:
|
|
67
|
+
arg = argv.pop(0)
|
|
68
|
+
ms = re.match(r'^-(\w)$', arg)
|
|
69
|
+
if ms:
|
|
70
|
+
option = ms.group(1)
|
|
71
|
+
if option == "b":
|
|
72
|
+
PgLOG.PGLOG['BCKGRND'] = 1
|
|
73
|
+
option = None
|
|
74
|
+
else:
|
|
75
|
+
SOPTIONS[option] = ''
|
|
76
|
+
continue
|
|
77
|
+
ms = re.match(r'^-({})$'.format(copts), arg)
|
|
78
|
+
if ms:
|
|
79
|
+
option = ms.group(1)
|
|
80
|
+
if option == "env": option = 'v'
|
|
81
|
+
continue
|
|
82
|
+
|
|
83
|
+
if not option: PgLOG.pglog("{}: Value passed in without leading option for {}".format(arg, rdasub), PgLOG.LGEREX)
|
|
84
|
+
if arg.find(' ') > -1 and not re.match(r'^[\'\"].*[\'\"]$', arg): # quote string with space but not quoted yet
|
|
85
|
+
if arg.find("'") > -1:
|
|
86
|
+
arg = '"{}"'.format(arg)
|
|
87
|
+
else:
|
|
88
|
+
arg = "'{}'".format(arg)
|
|
89
|
+
|
|
90
|
+
if option in coptions:
|
|
91
|
+
coptions[option] = arg
|
|
92
|
+
if option == "cmd": break
|
|
93
|
+
else:
|
|
94
|
+
SOPTIONS[option] = arg
|
|
95
|
+
option = None
|
|
96
|
+
|
|
97
|
+
if not coptions['cmd']: PgLOG.pglog(aname + ": specify command via option -cmd to run", PgLOG.LGWNEX)
|
|
98
|
+
args = PgLOG.argv_to_string(argv, 0) # append command options
|
|
99
|
+
if not SOPTIONS['o']: SOPTIONS['o'] = "{}/{}/".format(PgLOG.PGLOG['LOGPATH'], pname)
|
|
100
|
+
if not SOPTIONS['e']: SOPTIONS['e'] = "{}/{}/".format(PgLOG.PGLOG['LOGPATH'], pname)
|
|
101
|
+
if 'N' not in SOPTIONS: SOPTIONS['N'] = op.basename(coptions['cmd'])
|
|
102
|
+
msg = "{}-{}{}".format(PgLOG.PGLOG['HOSTNAME'], PgLOG.PGLOG['CURUID'], PgLOG.current_datetime())
|
|
103
|
+
|
|
104
|
+
if coptions['cwd']:
|
|
105
|
+
if coptions['cwd'].find('$'): coptions['cwd'] = PgLOG.replace_environments(coptions['cwd'], '', PgLOG.LGWNEX)
|
|
106
|
+
msg += "-" + coptions['cwd']
|
|
107
|
+
os.chdir(coptions['cwd'])
|
|
108
|
+
|
|
109
|
+
cmd = PgLOG.valid_command(coptions['cmd'])
|
|
110
|
+
if not cmd and not re.match(r'^/', coptions['cmd']): cmd = PgLOG.valid_command('./' + coptions['cmd'])
|
|
111
|
+
if not cmd: PgLOG.pglog(coptions['cmd'] + ": Cannot find given command to run", PgLOG.LGWNEX)
|
|
112
|
+
if args: cmd += " " + args
|
|
113
|
+
|
|
114
|
+
sbuf = build_tcsh_script(cmd, coptions, rdasub)
|
|
115
|
+
PgLOG.pglog(sbuf, PgLOG.MSGLOG)
|
|
116
|
+
PgLOG.PGLOG['ERR2STD'] = ['bind mouting']
|
|
117
|
+
PgLOG.pgsystem(rdasub, PgLOG.LOGWRN, 6, sbuf)
|
|
118
|
+
PgLOG.PGLOG['ERR2STD'] = []
|
|
119
|
+
|
|
120
|
+
sys.exit(0)
|
|
121
|
+
|
|
122
|
+
#
|
|
123
|
+
# build tcsh script to submit a PBS batch job
|
|
124
|
+
#
|
|
125
|
+
def build_tcsh_script(cmd, coptions, rdasub):
|
|
126
|
+
|
|
127
|
+
buf = "#!/bin/tcsh\n\n" # sbatch starting tcsh script
|
|
128
|
+
|
|
129
|
+
if 'l' in SOPTIONS: add_resources()
|
|
130
|
+
# add options to tcsh script for qsub
|
|
131
|
+
for option in SOPTIONS:
|
|
132
|
+
buf += "#PBS -" + option
|
|
133
|
+
if SOPTIONS[option]: buf += " {}".format(SOPTIONS[option])
|
|
134
|
+
buf += "\n"
|
|
135
|
+
for option in RESOURCES:
|
|
136
|
+
buf += "#PBS -l"
|
|
137
|
+
if RESOURCES[option]: buf += " {}={}".format(option, RESOURCES[option])
|
|
138
|
+
buf += "\n"
|
|
139
|
+
|
|
140
|
+
# always include the login user's tcsh resource file
|
|
141
|
+
homedir = "{}/{}".format(PgLOG.PGLOG['USRHOME'], PgLOG.PGLOG['CURUID'])
|
|
142
|
+
buf += "setenv HOME {}\n".format(homedir)
|
|
143
|
+
buf += "source /etc/profile.d/z00_modules.csh\n"
|
|
144
|
+
buf += "source /glade/u/apps/opt/conda/etc/profile.d/conda.csh\n"
|
|
145
|
+
buf += "source {}/.tcshrc\n".format(homedir)
|
|
146
|
+
buf += "pwd; hostname; date\n"
|
|
147
|
+
buf += add_modules(coptions['res'], coptions['mod'])
|
|
148
|
+
buf += set_vm_libs(coptions['res'])
|
|
149
|
+
buf += "\necho {}\n{}\n\ndate\n".format(cmd, cmd)
|
|
150
|
+
|
|
151
|
+
return buf
|
|
152
|
+
|
|
153
|
+
#
|
|
154
|
+
# check and add resource options
|
|
155
|
+
#
|
|
156
|
+
def add_resources():
|
|
157
|
+
|
|
158
|
+
for res in re.split(',', SOPTIONS['l']):
|
|
159
|
+
ms = re.match(r'^([^=]+)=(.+)$', res)
|
|
160
|
+
if ms:
|
|
161
|
+
RESOURCES[ms.group(1)] = ms.group(2)
|
|
162
|
+
else:
|
|
163
|
+
PgLOG.pglog(res + ": use '=' to separate resource name & value", PgLOG.LGEREX)
|
|
164
|
+
del SOPTIONS['l']
|
|
165
|
+
|
|
166
|
+
#
|
|
167
|
+
# add module loads for modules provided
|
|
168
|
+
#
|
|
169
|
+
def add_modules(res, mods):
|
|
170
|
+
|
|
171
|
+
mbuf = "\n"
|
|
172
|
+
defmods = DEFMODS[res] if res in DEFMODS else DEFMODS['default']
|
|
173
|
+
|
|
174
|
+
dmods = re.split(',', defmods)
|
|
175
|
+
for dmod in dmods:
|
|
176
|
+
ms = re.match(r'^(.+)/', dmod)
|
|
177
|
+
smod = ms.group(1) if ms else dmod
|
|
178
|
+
if smod in SWAPMODS: mbuf += "module unload {}\n".format(SWAPMODS[smod])
|
|
179
|
+
mbuf += "module load {}\n".format(dmod)
|
|
180
|
+
|
|
181
|
+
if mods:
|
|
182
|
+
amods = re.split(',', mods)
|
|
183
|
+
for amod in amods:
|
|
184
|
+
if re.match(r'^/', amod):
|
|
185
|
+
mbuf += "module use {}\n".format(amod)
|
|
186
|
+
else:
|
|
187
|
+
ms = re.match(r'^(.+)/', amod)
|
|
188
|
+
smod = ms.group(1) if ms else amod
|
|
189
|
+
if smod in dmods: continue
|
|
190
|
+
if smod in SWAPMODS: mbuf += "module unload {}\n".format(SWAPMODS[smod])
|
|
191
|
+
mbuf += "module load {}\n".format(amod)
|
|
192
|
+
|
|
193
|
+
return mbuf
|
|
194
|
+
|
|
195
|
+
#
|
|
196
|
+
# set virtual machine libraries
|
|
197
|
+
#
|
|
198
|
+
def set_vm_libs(res):
|
|
199
|
+
|
|
200
|
+
deflibs = DEFLIBS[res] if res in DEFLIBS else DEFLIBS['default']
|
|
201
|
+
if not deflibs: return ''
|
|
202
|
+
|
|
203
|
+
dlibs = re.split(',', deflibs)
|
|
204
|
+
libbuf = "\n"
|
|
205
|
+
for dlib in dlibs:
|
|
206
|
+
libbuf += dlib + "\n"
|
|
207
|
+
|
|
208
|
+
return libbuf
|
|
209
|
+
|
|
210
|
+
#
|
|
211
|
+
# call main() to start program
|
|
212
|
+
#
|
|
213
|
+
if __name__ == "__main__": main()
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
|
|
2
|
+
To submit a job execution as a batch job in tcsh script on a PBS node via 'qsub'.
|
|
3
|
+
|
|
4
|
+
Usage: rdaqsub [qsub-options] [-cwd WorkDir] [-env EnvironmentPairs] \
|
|
5
|
+
[-mod Modules] [-res Reservation] -cmd Command [cmd-options]
|
|
6
|
+
|
|
7
|
+
- qsub-options, accepts options that can be passed to 'qsub' in a tcsh script.
|
|
8
|
+
Check qsub man page for help on the PBS batch options. Here is the list of
|
|
9
|
+
qsub options that are included at default:
|
|
10
|
+
-o LOGPATH/rdaqsub/
|
|
11
|
+
-e LOGPATH/rdaqsub/
|
|
12
|
+
-A P43713000
|
|
13
|
+
-m a
|
|
14
|
+
-q regular
|
|
15
|
+
-l walltime=6:00:00,select=1:node=1:mem=1gb
|
|
16
|
+
|
|
17
|
+
- Option -cwd, set the working directory for the Command to be executed. If
|
|
18
|
+
it is not specified, it defaults to the current directory where qsub
|
|
19
|
+
command is executed.
|
|
20
|
+
|
|
21
|
+
- Option -env, set environment name/value pairs separated by ',' dynamically
|
|
22
|
+
for the Command to be executed. This is equivilent to qsub-option -v.
|
|
23
|
+
|
|
24
|
+
- Option -mod, set module list separated by ',' for loading modules to execute
|
|
25
|
+
the command.
|
|
26
|
+
|
|
27
|
+
- Option -res, set Reservation name to load modules and start VM libs;
|
|
28
|
+
|
|
29
|
+
- Option -cmd, mandatory option to lead a Command to be executed;
|
|
30
|
+
|
|
31
|
+
- cmd-options, specifies options that can be passed to the Command.
|
|
32
|
+
|
|
33
|
+
A tash script example:
|
|
34
|
+
#!/bin/tcsh
|
|
35
|
+
|
|
36
|
+
#PBS -o /gpfs/u/home/rdadata/dssdb/log/rdaqsub/
|
|
37
|
+
#PBS -e /gpfs/u/home/rdadata/dssdb/log/rdaqsub/
|
|
38
|
+
#PBS -A P43713000
|
|
39
|
+
#PBS -q rda@casper-pbs
|
|
40
|
+
#PBS -m a
|
|
41
|
+
#PBS -N dsrqst
|
|
42
|
+
#PBS -l walltime=12:00:00
|
|
43
|
+
#PBS -l select=1:mem=20gb
|
|
44
|
+
setenv HOME /gpfs/u/home/davestep
|
|
45
|
+
source /etc/profile.d/z00_modules.tsh
|
|
46
|
+
source /glade/u/apps/opt/conda/etc/profile.d/conda.tsh
|
|
47
|
+
source /gpfs/u/home/davestep/.tcshrc
|
|
48
|
+
pwd; hostname; date
|
|
49
|
+
|
|
50
|
+
module load ncarenv
|
|
51
|
+
module load netcdf
|
|
52
|
+
module load ncl
|
|
53
|
+
module load nco
|
|
54
|
+
module load cdo
|
|
55
|
+
module load conda
|
|
56
|
+
module load grib-util
|
|
57
|
+
module load wgrib2
|
|
58
|
+
|
|
59
|
+
conda activate /glade/work/rdadata/conda-envs/pg-casper
|
|
60
|
+
|
|
61
|
+
echo /gpfs/u/home/rdadata/bin/dsrqst d628000 SP -RI 729708 -NP -b -d
|
|
62
|
+
/gpfs/u/home/rdadata/bin/dsrqst d628000 SP -RI 729708 -NP -b -d
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2023 National Center for Atmospheric Research
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
Metadata-Version: 2.2
|
|
2
|
+
Name: rda_python_miscs
|
|
3
|
+
Version: 1.0.1
|
|
4
|
+
Summary: RDA Python package to hold RDA miscellaneous utility programs
|
|
5
|
+
Author-email: Zaihua Ji <zji@ucar.edu>
|
|
6
|
+
Project-URL: Homepage, https://github.com/NCAR/rda-python-miscs
|
|
7
|
+
Classifier: Programming Language :: Python :: 3
|
|
8
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
9
|
+
Classifier: Operating System :: OS Independent
|
|
10
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
11
|
+
Requires-Python: >=3.7
|
|
12
|
+
Description-Content-Type: text/markdown
|
|
13
|
+
License-File: LICENSE
|
|
14
|
+
Requires-Dist: rda_python_common
|
|
15
|
+
Requires-Dist: rda_python_setuid
|
|
16
|
+
|
|
17
|
+
RDA python package to hold miscellaneous utility programs.
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
rda_python_miscs/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
|
2
|
+
rda_python_miscs/bashqsub.py,sha256=bW2YZKjPDgGBZ0TR9qvhZ_tvznpP1T38YFuZcrwanK0,6659
|
|
3
|
+
rda_python_miscs/bashqsub.usg,sha256=5m35b6JfZGJ0appFRcen8FLjdFgsAVg_lRM8gnEcfZM,2111
|
|
4
|
+
rda_python_miscs/pgwget.py,sha256=DHBOL9CMP0cRvLR8mEG5HqX2nRfgzb2C5WvIbgqCrOE,6815
|
|
5
|
+
rda_python_miscs/rdacp.py,sha256=FXWY0KAStV-6PiBXNWJautaK1Tx5mMkdU_MNyboE9ng,7126
|
|
6
|
+
rda_python_miscs/rdacp.usg,sha256=xADB569OY7t1hGlOwYWjNVY6qwRTz47pYM8mY9tJeMQ,2780
|
|
7
|
+
rda_python_miscs/rdakill.py,sha256=XOUS36QkdsNN8ac-VuePDKrJ1YbeMXCywhCTg4B0QiY,8705
|
|
8
|
+
rda_python_miscs/rdakill.usg,sha256=1gQ7OVp9o0HSr9T2KHyqpGyFiaF1HEh2qaMk21HXgfM,1367
|
|
9
|
+
rda_python_miscs/rdals.py,sha256=2TzHOyyIkgeyY-6flXT9x5kp4hKJal-LzD1N7GpEhGU,8793
|
|
10
|
+
rda_python_miscs/rdals.usg,sha256=ChF-nn3Qb2pds3wMXIWubB_tjwZslxHfSha0dTDqOiY,2984
|
|
11
|
+
rda_python_miscs/rdamod.py,sha256=Tvmf6Z5zH9P2KdckB7H69c63yrSvoOyedWfKqK6SmvM,5368
|
|
12
|
+
rda_python_miscs/rdamod.usg,sha256=BFCWqYDybqLLWGYTfWRj-G3b-itOBJgE-yxnmxB-JMA,2528
|
|
13
|
+
rda_python_miscs/rdaown.py,sha256=bm2qvMWCFe5xbTriue6ZeVvfikGBoICjeOwAyt-Ct-g,5592
|
|
14
|
+
rda_python_miscs/rdaown.usg,sha256=KUEDMZT6mkQ_vblwZEoL_JAW5ti44ic4n_Wlzy7-ihE,2313
|
|
15
|
+
rda_python_miscs/rdaps.py,sha256=SQQonJ5lyvGeqo_BqOpeosY5ACNXKFag5JXyRdEoYa0,5591
|
|
16
|
+
rda_python_miscs/rdaps.usg,sha256=Tge9aZVTqCGVASDEXE11VnJhgvksQPEbwyBXKviwGf4,986
|
|
17
|
+
rda_python_miscs/rdasub.py,sha256=Oa1xohozShVxLgooUEQj_DvyD0d3JDkknrJ4XkTq4H0,4083
|
|
18
|
+
rda_python_miscs/rdasub.usg,sha256=zHPHURJXVK2u8z9g4rL_TqT4ApRSRN72tIe4VOiSY90,444
|
|
19
|
+
rda_python_miscs/rdazip.py,sha256=mm1mTSflEJX72by9rAHZ5vjDJZ9q7xjREBXYHoJJ6k4,1776
|
|
20
|
+
rda_python_miscs/rdazip.usg,sha256=V4tCymlIZObIXRT3wPILKBWfmpV0mre1SlXBlg4ZG1k,551
|
|
21
|
+
rda_python_miscs/tcshqsub.py,sha256=Of3j7FF78oLOMBCmM-vj3rb06qcIzZJAMRS4cOxRkJ0,6660
|
|
22
|
+
rda_python_miscs/tcshqsub.usg,sha256=vl5mHyPe0ca589hRBIC8yb3myQtHby2TpqbQICavW2o,2115
|
|
23
|
+
rda_python_miscs-1.0.1.dist-info/LICENSE,sha256=1dck4EAQwv8QweDWCXDx-4Or0S8YwiCstaso_H57Pno,1097
|
|
24
|
+
rda_python_miscs-1.0.1.dist-info/METADATA,sha256=F8jcbpXEoCp2gUOmQAaqrxdMjNGHnXhKTDhTicAWaYk,647
|
|
25
|
+
rda_python_miscs-1.0.1.dist-info/WHEEL,sha256=beeZ86-EfXScwlR_HKu4SllMC9wUEj_8Z_4FJ3egI2w,91
|
|
26
|
+
rda_python_miscs-1.0.1.dist-info/entry_points.txt,sha256=KFt_k_EGp_HEggEbVK2pXygaol1MlqLvDq8ZwWNG7v0,411
|
|
27
|
+
rda_python_miscs-1.0.1.dist-info/top_level.txt,sha256=W5rz7DrWb7hXABUbGgWcwe6D644X338LR8_zdgmtLhg,17
|
|
28
|
+
rda_python_miscs-1.0.1.dist-info/RECORD,,
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
[console_scripts]
|
|
2
|
+
bashqsub = rda_python_miscs.bashqsub:main
|
|
3
|
+
pgwget = rda_python_miscs.pgwget:main
|
|
4
|
+
rdacp.py = rda_python_miscs.rdacp:main
|
|
5
|
+
rdakill.py = rda_python_miscs.rdakill:main
|
|
6
|
+
rdals = rda_python_miscs.rdals:main
|
|
7
|
+
rdamod.py = rda_python_miscs.rdamod:main
|
|
8
|
+
rdaown = rda_python_miscs.rdaown:main
|
|
9
|
+
rdaps = rda_python_miscs.rdaps:main
|
|
10
|
+
rdasub = rda_python_miscs.rdasub:main
|
|
11
|
+
tcshqsub = rda_python_miscs.tcshqsub:main
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
rda_python_miscs
|