rda-python-miscs 1.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rda_python_miscs/__init__.py +1 -0
- rda_python_miscs/bashqsub.py +213 -0
- rda_python_miscs/bashqsub.usg +64 -0
- rda_python_miscs/pgwget.py +186 -0
- rda_python_miscs/rdacp.py +212 -0
- rda_python_miscs/rdacp.usg +62 -0
- rda_python_miscs/rdakill.py +267 -0
- rda_python_miscs/rdakill.usg +36 -0
- rda_python_miscs/rdals.py +278 -0
- rda_python_miscs/rdals.usg +61 -0
- rda_python_miscs/rdamod.py +172 -0
- rda_python_miscs/rdamod.usg +51 -0
- rda_python_miscs/rdaown.py +169 -0
- rda_python_miscs/rdaown.usg +46 -0
- rda_python_miscs/rdaps.py +194 -0
- rda_python_miscs/rdaps.usg +29 -0
- rda_python_miscs/rdasub.py +117 -0
- rda_python_miscs/rdasub.usg +12 -0
- rda_python_miscs/rdazip.py +64 -0
- rda_python_miscs/rdazip.usg +20 -0
- rda_python_miscs/tcshqsub.py +213 -0
- rda_python_miscs/tcshqsub.usg +62 -0
- rda_python_miscs-1.0.1.dist-info/LICENSE +21 -0
- rda_python_miscs-1.0.1.dist-info/METADATA +17 -0
- rda_python_miscs-1.0.1.dist-info/RECORD +28 -0
- rda_python_miscs-1.0.1.dist-info/WHEEL +5 -0
- rda_python_miscs-1.0.1.dist-info/entry_points.txt +11 -0
- rda_python_miscs-1.0.1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
|
|
2
|
+
List directory and file information of the current or specified directories
|
|
3
|
+
with metadata information recorded in RDADB if matched. Four columns are
|
|
4
|
+
listed, they are Directory Name, Data Volume, File Count, and Brief
|
|
5
|
+
Description if the listed item is a directory, and they are File Name,
|
|
6
|
+
Data Size, Data Format, and Brief Description if the listed item is a
|
|
7
|
+
file.
|
|
8
|
+
|
|
9
|
+
A leading letter is displayed on each line to indicate what type item is listed;
|
|
10
|
+
including 'D' for a whole dataset, 'G' for a group or subgroup in a dataset,
|
|
11
|
+
and 'F' for a data file.
|
|
12
|
+
|
|
13
|
+
The output of directory/file list is formatted as default with double spaces
|
|
14
|
+
as delimiter and each column lined up vertically at least for the files under each
|
|
15
|
+
directory. Provide Option -N to display list without format. A delimiter symbol '|'
|
|
16
|
+
is defaulted if Option -N is present.
|
|
17
|
+
|
|
18
|
+
Usage: rdals [-d] [-f] [-N] [-h] [-r] [-D DelimitSymbols] [-R RecursiveLevel] [Directory/File List]
|
|
19
|
+
|
|
20
|
+
- Option -d, list directory information only. Directory information
|
|
21
|
+
is included as default. Add this option to exclude file information;
|
|
22
|
+
|
|
23
|
+
- Option -f, list file information only. File information
|
|
24
|
+
is included as default. Add this option to exclude directory information;
|
|
25
|
+
|
|
26
|
+
- Option -N, list files unformatted;
|
|
27
|
+
|
|
28
|
+
- Option -h, display this help document;
|
|
29
|
+
|
|
30
|
+
- Option -r, list directories and files recursively;
|
|
31
|
+
|
|
32
|
+
- Option -R, list directories and files recursively up to the level
|
|
33
|
+
provided with this Option;
|
|
34
|
+
|
|
35
|
+
- Option -D, specify delimiting symbols for dividing the columns.
|
|
36
|
+
It defaults to " " for formatted output and '|' for unformatted output.
|
|
37
|
+
Make sure quote the symbols if any character in the symbols has Unix
|
|
38
|
+
meaning, for example -D '<:>';
|
|
39
|
+
|
|
40
|
+
- Directory/file List is optional; without specification, all directories
|
|
41
|
+
and files in the current directory are listed. Unix command line
|
|
42
|
+
wildcards are supported.
|
|
43
|
+
|
|
44
|
+
This utility program can be executed anywhere. Nothing is displayed if neither
|
|
45
|
+
directory nor file information matches RDADB information.
|
|
46
|
+
|
|
47
|
+
For examples, to check directories and files of ds277.6, you can
|
|
48
|
+
|
|
49
|
+
1. Change into the dataset home data directory as 'cd /PathTo/ds277.6' and
|
|
50
|
+
execute 'rdals'; add recursive option '-r' to check directories and files
|
|
51
|
+
further into the sub-directories, or change directory into a sub-directory
|
|
52
|
+
to check files inside of it.
|
|
53
|
+
|
|
54
|
+
2. Pass an absolute path to rdals as 'rdals /PathTo/ds277.6/' or as
|
|
55
|
+
'rdals /PathTo/ds277.6/*'; without the ending by '/' or an appended
|
|
56
|
+
wildcard symbol '*' information of the dataset itself is check unless
|
|
57
|
+
the recursive option '-r' or '-R RecursiveLevel' is present
|
|
58
|
+
|
|
59
|
+
3. If the current directory is in another dataset home data directory,
|
|
60
|
+
such as /PathTo/ds277.7, you can pass a relative path to rdals
|
|
61
|
+
as 'rdals ../ds277.6/' or as 'rdals ../ds277.6/*'
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
#
|
|
3
|
+
##################################################################################
|
|
4
|
+
#
|
|
5
|
+
# Title: rdamod
|
|
6
|
+
# Author: Zaihua Ji, zji@ucar.edu
|
|
7
|
+
# Date: 10/24/2020
|
|
8
|
+
# 2025-03-10 transferred to package rda_python_miscs from
|
|
9
|
+
# https://github.com/NCAR/rda-utility-programs.git
|
|
10
|
+
# Purpose: change file/directory modes in given one or mutilple local directories
|
|
11
|
+
# owned by 'rdadata'
|
|
12
|
+
#
|
|
13
|
+
# Github: https://github.com/NCAR/rda-python-miscs.git
|
|
14
|
+
#
|
|
15
|
+
##################################################################################
|
|
16
|
+
#
|
|
17
|
+
import re
|
|
18
|
+
import os
|
|
19
|
+
import sys
|
|
20
|
+
from os import path as op
|
|
21
|
+
from rda_python_common import PgLOG
|
|
22
|
+
from rda_python_common import PgUtil
|
|
23
|
+
from rda_python_common import PgFile
|
|
24
|
+
from rda_python_common import PgDBI
|
|
25
|
+
|
|
26
|
+
RDAMOD = {
|
|
27
|
+
'd' : 0, # 1 to change directory mode
|
|
28
|
+
'f' : 0, # 1 to change file mode
|
|
29
|
+
'h' : 0, # 1 to show help message
|
|
30
|
+
'r' : 0, # 1 if recursive all
|
|
31
|
+
'R' : 0, # > 0 to set recursive limit
|
|
32
|
+
'F' : 0o664, # to chnage file mode, default to 664
|
|
33
|
+
'D' : 0o775, # to chnge directory mode, default to 775
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
MINFO = {
|
|
37
|
+
'files' : [],
|
|
38
|
+
'curdir' : os.getcwd(),
|
|
39
|
+
'tpath' : None,
|
|
40
|
+
'dcnt' : 0,
|
|
41
|
+
'fcnt' : 0
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
#
|
|
45
|
+
# main function to run the application
|
|
46
|
+
#
|
|
47
|
+
def main():
|
|
48
|
+
|
|
49
|
+
PgDBI.dssdb_dbname()
|
|
50
|
+
PgLOG.set_suid(PgLOG.PGLOG['EUID'])
|
|
51
|
+
PgLOG.set_help_path(__file__)
|
|
52
|
+
PgLOG.PGLOG['LOGFILE'] = "rdamod.log" # set different log file
|
|
53
|
+
argv = sys.argv[1:]
|
|
54
|
+
PgLOG.cmdlog("rdamod {} ({})".format(' '.join(argv), MINFO['curdir']))
|
|
55
|
+
option = defopt = 'l'
|
|
56
|
+
for arg in argv:
|
|
57
|
+
ms = re.match(r'-(\w)$', arg)
|
|
58
|
+
if ms:
|
|
59
|
+
option = ms.group(1)
|
|
60
|
+
if option not in RDAMOD: PgLOG.pglog(arg + ": Unknown Option", PgLOG.LGEREX)
|
|
61
|
+
if 'dfhr'.find(option) > -1:
|
|
62
|
+
RDAMOD[option] = 1
|
|
63
|
+
option = defopt
|
|
64
|
+
continue
|
|
65
|
+
if not option: PgLOG.pglog(arg + ": Value provided without option", PgLOG.LGEREX)
|
|
66
|
+
if option == 'l':
|
|
67
|
+
MINFO['files'].append(arg)
|
|
68
|
+
defopt = None
|
|
69
|
+
else:
|
|
70
|
+
if option == 'R':
|
|
71
|
+
RDAMOD[option] = int(arg)
|
|
72
|
+
elif 'FD'.find(option) > -1:
|
|
73
|
+
RDAMOD[option] = PgLOG.base2int(arg, 8)
|
|
74
|
+
else:
|
|
75
|
+
RDAMOD[option] = arg
|
|
76
|
+
option = defopt
|
|
77
|
+
|
|
78
|
+
if RDAMOD['h'] or not MINFO['files']: PgLOG.show_usage("rdamod")
|
|
79
|
+
if not (RDAMOD['d'] or RDAMOD['f']):
|
|
80
|
+
RDAMOD['d'] = RDAMOD['f'] = 1 # both directories and files as default
|
|
81
|
+
if not RDAMOD['R'] and RDAMOD['r']: RDAMOD['R'] = 1000
|
|
82
|
+
PgDBI.validate_decs_group('rdamod', PgLOG.PGLOG['CURUID'], 1)
|
|
83
|
+
|
|
84
|
+
change_top_list(MINFO['files'])
|
|
85
|
+
|
|
86
|
+
if (MINFO['dcnt'] + MINFO['fcnt']) > 1:
|
|
87
|
+
msg = ''
|
|
88
|
+
if MINFO['dcnt'] > 0:
|
|
89
|
+
s = ('ies' if MINFO['dcnt'] else 'y')
|
|
90
|
+
msg = "{} Director{}".format(MINFO['dcnt'], s)
|
|
91
|
+
if MINFO['fcnt'] > 0:
|
|
92
|
+
s = ('s' if MINFO['fcnt'] > 1 else '')
|
|
93
|
+
if msg: msg += " & "
|
|
94
|
+
msg += "{} File{}".format(MINFO['fcnt'], s)
|
|
95
|
+
PgLOG.pglog("Total {} changed Mode".format(msg), PgLOG.LOGWRN)
|
|
96
|
+
elif (MINFO['dcnt'] + MINFO['fcnt']) == 0:
|
|
97
|
+
PgLOG.pglog((MINFO['tpath'] if MINFO['tpath'] else MINFO['curdir']) + ": No Mode changed", PgLOG.LOGWRN)
|
|
98
|
+
|
|
99
|
+
PgLOG.cmdlog()
|
|
100
|
+
PgLOG.pgexit(0)
|
|
101
|
+
|
|
102
|
+
#
|
|
103
|
+
# change mode for the top level list
|
|
104
|
+
#
|
|
105
|
+
def change_top_list(files):
|
|
106
|
+
|
|
107
|
+
for file in files:
|
|
108
|
+
info = PgFile.check_local_file(file, 6, PgLOG.LOGWRN)
|
|
109
|
+
if not info:
|
|
110
|
+
PgLOG.pglog(file + ": NOT exists", PgLOG.LOGERR)
|
|
111
|
+
continue
|
|
112
|
+
|
|
113
|
+
change = 1
|
|
114
|
+
if not info['isfile'] and re.search(r'/$', file):
|
|
115
|
+
change = 0 # do not change the directory mode if it is ended by '/'
|
|
116
|
+
file = re.sub(r'/$', '', file, 1)
|
|
117
|
+
|
|
118
|
+
if not re.match(r'^/', file): file = PgLOG.join_paths(MINFO['curdir'], file)
|
|
119
|
+
MINFO['tpath'] = (op.dirname(file) if change else file) + "/"
|
|
120
|
+
if change: change_mode(file, info)
|
|
121
|
+
if not info['isfile'] and (RDAMOD['R'] > 0 or not change):
|
|
122
|
+
fs = PgFile.local_glob(file, 6, PgLOG.LOGWRN)
|
|
123
|
+
change_list(fs, 1, file)
|
|
124
|
+
|
|
125
|
+
#
|
|
126
|
+
# recursively change directory/file mode
|
|
127
|
+
#
|
|
128
|
+
def change_list(files, level, cdir):
|
|
129
|
+
|
|
130
|
+
fcnt = 0
|
|
131
|
+
|
|
132
|
+
for file in files:
|
|
133
|
+
info = files[file]
|
|
134
|
+
fcnt += change_mode(file, info)
|
|
135
|
+
if not info['isfile'] and level < RDAMOD['R']:
|
|
136
|
+
fs = PgFile.local_glob(file, 6, PgLOG.LOGWRN)
|
|
137
|
+
change_list(fs, level+1, file)
|
|
138
|
+
|
|
139
|
+
if fcnt > 1: # display sub count if two more files are changed mode
|
|
140
|
+
PgLOG.pglog("{}: {} Files changed Mode".format(cdir, fcnt), PgLOG.LOGWRN)
|
|
141
|
+
|
|
142
|
+
#
|
|
143
|
+
# change mode of a single directory/file
|
|
144
|
+
#
|
|
145
|
+
def change_mode(file, info):
|
|
146
|
+
|
|
147
|
+
fname = re.sub(r'^{}'.format(MINFO['tpath']), '', file, 1)
|
|
148
|
+
if info['isfile']:
|
|
149
|
+
if not RDAMOD['d']: return 0
|
|
150
|
+
fname = "F" + fname
|
|
151
|
+
mode = RDAMOD['F']
|
|
152
|
+
else:
|
|
153
|
+
if not RDAMOD['d']: return 0
|
|
154
|
+
fname = "D" + fname
|
|
155
|
+
mode = RDAMOD['D']
|
|
156
|
+
|
|
157
|
+
if info['logname'] != "rdadata":
|
|
158
|
+
return PgLOG.pglog("{}: owner {} not rdadata".format(fname, info['logname']), PgLOG.LOGERR)
|
|
159
|
+
if info['mode'] == mode: return 0 # no need change mode
|
|
160
|
+
|
|
161
|
+
if PgFile.set_local_mode(file, info['isfile'], mode, info['mode'], info['logname'], PgLOG.LOGWRN):
|
|
162
|
+
if info['isfile']:
|
|
163
|
+
MINFO['fcnt'] += 1
|
|
164
|
+
return 1
|
|
165
|
+
else:
|
|
166
|
+
MINFO['dcnt'] += 1
|
|
167
|
+
return 0
|
|
168
|
+
|
|
169
|
+
#
|
|
170
|
+
# call main() to start program
|
|
171
|
+
#
|
|
172
|
+
if __name__ == "__main__": main()
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
|
|
2
|
+
Change modes for directories and files in the current or specified directories.
|
|
3
|
+
The owner of the directories and files must be 'rdadata' for their modes being
|
|
4
|
+
changed. For directories and files with modes changed successfully or with error,
|
|
5
|
+
a leading letter is displayed in font of the relative file names to indicate
|
|
6
|
+
file types; 'D' for a directory and 'F' for a data file.
|
|
7
|
+
|
|
8
|
+
Usage: rdamod [-d] [-f] [-D DiretoryMode] [-F FileMode] [-h HostName] [-r] [-R RecursiveLevel] \
|
|
9
|
+
[Directory/File List]
|
|
10
|
+
|
|
11
|
+
- Option -d, change directory modes only. Changing Directory mode is included
|
|
12
|
+
as default. Add this option to exclude changing file mode;
|
|
13
|
+
|
|
14
|
+
- Option -f, change file modes only. Changing File mode is included
|
|
15
|
+
as default. Add this option to exclude changing Directory mode;
|
|
16
|
+
|
|
17
|
+
- Option -h, pass in the remote host name; hpss for change file mode on HPSS;
|
|
18
|
+
|
|
19
|
+
- Option -r, change modes for directories and files recursively;
|
|
20
|
+
|
|
21
|
+
- Option -R, change modes for directories and files recursively up to
|
|
22
|
+
the level provided with this Option;
|
|
23
|
+
|
|
24
|
+
- Option -D, change directory mode to a value provided by this Option.
|
|
25
|
+
It defaults to "755";
|
|
26
|
+
|
|
27
|
+
- Option -F, change file mode to a value provided by this Option.
|
|
28
|
+
It defaults to "644";
|
|
29
|
+
|
|
30
|
+
- Directory/file List is mandatory; this help document is displayed
|
|
31
|
+
without it. Unix command line wildcards are supported. Use './' or '*'
|
|
32
|
+
for all directories and files in the current directory to be considered.
|
|
33
|
+
|
|
34
|
+
This utility program can be executed anywhere. No Mode is changed if neither
|
|
35
|
+
directory nor file are owned by user 'rdadata'.
|
|
36
|
+
|
|
37
|
+
For examples, to change modes for directories and files under ds277.6, you can
|
|
38
|
+
|
|
39
|
+
1. Change into the dataset home data directory as 'cd /PathTo/ds277.6' and
|
|
40
|
+
execute 'rdamod ./'; add recursive option '-r' to change modes for directories
|
|
41
|
+
and files further into the sub-directories, or change directory into
|
|
42
|
+
a sub-directory to change mode for files inside of it.
|
|
43
|
+
|
|
44
|
+
2. Pass an absolute path to rdamod as 'rdamod /PathTo/ds277.6/';
|
|
45
|
+
without the ending by '/', mode of top directory itself is
|
|
46
|
+
changed only unless the recursive option '-r' or '-R RecursiveLevel'
|
|
47
|
+
is present.
|
|
48
|
+
|
|
49
|
+
3. If the current directory is in another dataset home data directory,
|
|
50
|
+
such as /PathTo/ds277.7, you can pass a relative path to rdamod
|
|
51
|
+
as 'rdamod ../ds277.6/' or as 'rdamod ../ds277.6/*'
|
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
#
|
|
3
|
+
##################################################################################
|
|
4
|
+
#
|
|
5
|
+
# Title: rdaown
|
|
6
|
+
# Author: Zaihua Ji, zji@ucar.edu
|
|
7
|
+
# Date: 10/24/2020
|
|
8
|
+
# 2025-03-10 transferred to package rda_python_miscs from
|
|
9
|
+
# https://github.com/NCAR/rda-utility-programs.git
|
|
10
|
+
# Purpose: change file/directory ownership to 'rdadata' in given one or mutilple
|
|
11
|
+
# local directories that are owned by decs specialists. it needs
|
|
12
|
+
# super user privilege to execute.
|
|
13
|
+
#
|
|
14
|
+
# Github: https://github.com/NCAR/rda-python-miscs.git
|
|
15
|
+
#
|
|
16
|
+
##################################################################################
|
|
17
|
+
#
|
|
18
|
+
import re
|
|
19
|
+
import os
|
|
20
|
+
import sys
|
|
21
|
+
import glob
|
|
22
|
+
from os import path as op
|
|
23
|
+
from rda_python_common import PgLOG
|
|
24
|
+
from rda_python_common import PgUtil
|
|
25
|
+
from rda_python_common import PgFile
|
|
26
|
+
from rda_python_common import PgDBI
|
|
27
|
+
|
|
28
|
+
RDAOWN = {
|
|
29
|
+
'd' : 0, # 1 to change directory owner
|
|
30
|
+
'f' : 0, # 1 to change file owner
|
|
31
|
+
'h' : 0, # 1 to show help message
|
|
32
|
+
'r' : 0, # 1 if recursive all
|
|
33
|
+
'R' : 0, # > 0 to set recursive limit
|
|
34
|
+
'F' : 0o664, # to change file mode, default to 664
|
|
35
|
+
'D' : 0o775, # to change directory mode, default to 775
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
OINFO = {
|
|
39
|
+
'files' : [],
|
|
40
|
+
'curdir' : os.getcwd(),
|
|
41
|
+
'tpath' : None,
|
|
42
|
+
'dcnt' : 0,
|
|
43
|
+
'fcnt' : 0
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
#
|
|
47
|
+
# main function to run the application
|
|
48
|
+
#
|
|
49
|
+
def main():
|
|
50
|
+
|
|
51
|
+
argv = sys.argv[1:]
|
|
52
|
+
PgDBI.dssdb_scname()
|
|
53
|
+
PgLOG.set_help_path(__file__)
|
|
54
|
+
PgLOG.PGLOG['LOGFILE'] = "rdaown.log" # set different log file
|
|
55
|
+
PgLOG.cmdlog("rdaown {} ({})".format(' '.join(argv), OINFO['curdir']))
|
|
56
|
+
option = defopt = 'l'
|
|
57
|
+
for arg in argv:
|
|
58
|
+
ms = re.match(r'-(\w+)$', arg)
|
|
59
|
+
if ms:
|
|
60
|
+
option = ms.group(1)
|
|
61
|
+
if option not in RDAOWN: PgLOG.pglog(arg + ": Unknown Option", PgLOG.LGEREX)
|
|
62
|
+
if 'dfhr'.find(option) > -1:
|
|
63
|
+
RDAOWN[option] = 1
|
|
64
|
+
option = defopt
|
|
65
|
+
continue
|
|
66
|
+
if not option: PgLOG.pglog(arg + ": Value provided without option", PgLOG.LGEREX)
|
|
67
|
+
if option == 'R':
|
|
68
|
+
RDAOWN['R'] = int(arg)
|
|
69
|
+
option = defopt
|
|
70
|
+
else:
|
|
71
|
+
OINFO['files'].append(arg)
|
|
72
|
+
defopt = None
|
|
73
|
+
|
|
74
|
+
if RDAOWN['h'] or not OINFO['files']: PgLOG.show_usage("rdaown")
|
|
75
|
+
if PgLOG.PGLOG['CURUID'] != "root":
|
|
76
|
+
PgLOG.pglog(PgLOG.PGLOG['CURUID'] + ": you must execute 'rdaown' as 'root'!", PgLOG.LGEREX)
|
|
77
|
+
if not (RDAOWN['d'] or RDAOWN['f']):
|
|
78
|
+
RDAOWN['d'] = RDAOWN['f'] = 1 # list both directories and files as default
|
|
79
|
+
if not RDAOWN['R'] and RDAOWN['r']: RDAOWN['R'] = 1000
|
|
80
|
+
|
|
81
|
+
change_top_list(OINFO['files'])
|
|
82
|
+
|
|
83
|
+
if (OINFO['dcnt'] + OINFO['fcnt']) > 1:
|
|
84
|
+
msg = ""
|
|
85
|
+
if OINFO['dcnt'] > 0:
|
|
86
|
+
s = ("ies" if OINFO['dcnt'] > 1 else "y")
|
|
87
|
+
msg = "{} Director{}".format(OINFO['dcnt'], s)
|
|
88
|
+
if OINFO['fcnt'] > 0:
|
|
89
|
+
s = ('s' if OINFO['fcnt'] > 1 else '')
|
|
90
|
+
if msg: msg += " & "
|
|
91
|
+
msg += "{} File{}".format(OINFO['fcnt'], s)
|
|
92
|
+
PgLOG.pglog("Total {} changed owner".format(msg), PgLOG.LOGWRN)
|
|
93
|
+
elif (OINFO['dcnt'] + OINFO['fcnt']) == 0:
|
|
94
|
+
PgLOG.pglog((OINFO['tpath'] if OINFO['tpath'] else OINFO['curdir']) + ": No Owner changed", PgLOG.LOGWRN)
|
|
95
|
+
|
|
96
|
+
PgLOG.cmdlog()
|
|
97
|
+
PgLOG.pgexit(0)
|
|
98
|
+
|
|
99
|
+
#
|
|
100
|
+
# change owner for the top level list
|
|
101
|
+
#
|
|
102
|
+
def change_top_list(files):
|
|
103
|
+
|
|
104
|
+
for file in files:
|
|
105
|
+
info = PgFile.check_local_file(file, 2, PgLOG.LOGWRN)
|
|
106
|
+
if not info:
|
|
107
|
+
PgLOG.pglog(file + ": NOT exists", PgLOG.LOGERR)
|
|
108
|
+
continue
|
|
109
|
+
change = 1
|
|
110
|
+
if not info['isfile'] and re.search(r'/$', file):
|
|
111
|
+
change = 0 # do not change the directory owner if it is ended by '/'
|
|
112
|
+
file = re.sub(r'/$', '', file, 1)
|
|
113
|
+
|
|
114
|
+
if not re.match(r'^/', file): file = PgLOG.join_paths(OINFO['curdir'], file)
|
|
115
|
+
OINFO['tpath'] = (op.dirname(file) if change else file) + "/"
|
|
116
|
+
if change: change_owner(file, info)
|
|
117
|
+
if not info['isfile'] and (RDAOWN['R'] or not change):
|
|
118
|
+
fs = glob.glob(file + "/*")
|
|
119
|
+
change_list(fs, 1, file)
|
|
120
|
+
|
|
121
|
+
#
|
|
122
|
+
# recursively change directory/file owner
|
|
123
|
+
#
|
|
124
|
+
def change_list(files, level, cdir):
|
|
125
|
+
|
|
126
|
+
fcnt = 0
|
|
127
|
+
for file in files:
|
|
128
|
+
info = PgFile.check_local_file(file, 2, PgLOG.LOGWRN)
|
|
129
|
+
if not info: continue # should not happen
|
|
130
|
+
fcnt += change_owner(file, info)
|
|
131
|
+
if not info['isfile'] and level < RDAOWN['R']:
|
|
132
|
+
fs = glob.glob(file + "/*")
|
|
133
|
+
change_list(fs, level+1, file)
|
|
134
|
+
|
|
135
|
+
if fcnt > 1: # display sub count if two more files are changed mode
|
|
136
|
+
PgLOG.pglog("{}: {} Files changed owner in the directory".format(cdir, fcnt), PgLOG.LOGWRN)
|
|
137
|
+
|
|
138
|
+
#
|
|
139
|
+
# change owner for a single directory/file
|
|
140
|
+
#
|
|
141
|
+
def change_owner(file, info):
|
|
142
|
+
|
|
143
|
+
fname = re.sub(r'^{}'.format(OINFO['tpath']), '', file, 1)
|
|
144
|
+
if info['isfile']:
|
|
145
|
+
if not RDAOWN['f']: return 0
|
|
146
|
+
fname = "F" + fname
|
|
147
|
+
else:
|
|
148
|
+
if not RDAOWN['d']: return 0
|
|
149
|
+
fname = "D" + fname
|
|
150
|
+
|
|
151
|
+
if info['logname'] == "rdadata": return 0
|
|
152
|
+
if not PgLOG.pgget("dssgrp", "", "logname = '{}'".format(info['logname']), PgLOG.LGEREX):
|
|
153
|
+
return PgLOG.pglog("{}: owner {} not a DECS Specialist!".format(fname, info['logname']), PgLOG.LOGERR)
|
|
154
|
+
|
|
155
|
+
if PgLOG.pgsystem("su root -c 'chown rdadata {}'".format(file), PgLOG.LOGWRN, 4):
|
|
156
|
+
PgLOG.pglog("{}: {} => rdadata".format(fname, info['logname']), PgLOG.LOGWRN)
|
|
157
|
+
if info['isfile']:
|
|
158
|
+
OINFO['fcnt'] += 1
|
|
159
|
+
return 1
|
|
160
|
+
else:
|
|
161
|
+
OINFO['dcnt'] += 1
|
|
162
|
+
return 0
|
|
163
|
+
|
|
164
|
+
return PgLOG.pglog("{}: Error change owner {} to rdadata".format(fname, info['logname']), PgLOG.LOGERR)
|
|
165
|
+
|
|
166
|
+
#
|
|
167
|
+
# call main() to start program
|
|
168
|
+
#
|
|
169
|
+
if __name__ == "__main__": main()
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
|
|
2
|
+
Change owner to 'rdadata' for directories and files in the current or specified
|
|
3
|
+
directories. You must execute this program as 'root' to be able to change owner.
|
|
4
|
+
The owner of the directories and files must be DSS specialists to be
|
|
5
|
+
changed. For directories and files with ownership changed successfully or with error,
|
|
6
|
+
a leading letter is displayed in font of the relative file names to indicate
|
|
7
|
+
file types; 'D' for a directory and 'F' for a data file.
|
|
8
|
+
|
|
9
|
+
Usage: rdaown [-d] [-f] [-h] [-r] [-R RecursiveLevel] [Directory/File List]
|
|
10
|
+
|
|
11
|
+
- Option -d, change directory owner only. Changing Directory owner is included
|
|
12
|
+
as default. Add this option to exclude changing file owner;
|
|
13
|
+
|
|
14
|
+
- Option -f, change file owner only. Changing File owner is included
|
|
15
|
+
as default. Add this option to exclude changing Directory owner;
|
|
16
|
+
|
|
17
|
+
- Option -h, display this help document;
|
|
18
|
+
|
|
19
|
+
- Option -r, change owner for directories and files recursively;
|
|
20
|
+
|
|
21
|
+
- Option -R, change owner for directories and files recursively up to
|
|
22
|
+
the level provided with this Option;
|
|
23
|
+
|
|
24
|
+
- Directory/file List is mandatory; this help document is displayed
|
|
25
|
+
without it. Unix command line wildcards are supported. Use './' or '*'
|
|
26
|
+
for all directories and files in the current directory to be considered.
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
This utility program can be executed anywhere. No Mode is changed if neither
|
|
30
|
+
directory nor file are owned by user 'rdadata'.
|
|
31
|
+
|
|
32
|
+
For examples, to change owner for directories and files under ds277.6, you can
|
|
33
|
+
|
|
34
|
+
1. Change into the dataset home data directory as 'cd /PathTo/ds277.6' and
|
|
35
|
+
execute 'rdaown ./'; add recursive option '-r' to change owner for directories
|
|
36
|
+
and files further into the sub-directories, or change directory into
|
|
37
|
+
a sub-directory to change owner for files inside of it.
|
|
38
|
+
|
|
39
|
+
2. Pass an absolute path to rdaown as 'rdaown /PathTo/ds277.6/';
|
|
40
|
+
without the ending by '/', owner of top directory itself is
|
|
41
|
+
changed only unless the recursive option '-r' or '-R RecursiveLevel'
|
|
42
|
+
is present.
|
|
43
|
+
|
|
44
|
+
3. If the current directory is in another dataset home data directory,
|
|
45
|
+
such as /PathTo/ds277.7, you can pass a relative path to rdaown
|
|
46
|
+
as 'rdaown ../ds277.6/' or as 'rdaown ../ds277.6/*'
|
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
#
|
|
3
|
+
##################################################################################
|
|
4
|
+
#
|
|
5
|
+
# Title: rdaps
|
|
6
|
+
# Author: Zaihua Ji, zji@ucar.edu
|
|
7
|
+
# Date: 10/24/2020
|
|
8
|
+
# 2025-03-10 transferred to package rda_python_miscs from
|
|
9
|
+
# https://github.com/NCAR/rda-utility-programs.git
|
|
10
|
+
# Purpose: run ps against running process ID locally or remotely
|
|
11
|
+
#
|
|
12
|
+
# Github: https://github.com/NCAR/rda-python-miscs.git
|
|
13
|
+
#
|
|
14
|
+
##################################################################################
|
|
15
|
+
#
|
|
16
|
+
import re
|
|
17
|
+
import os
|
|
18
|
+
import sys
|
|
19
|
+
from rda_python_common import PgLOG
|
|
20
|
+
from rda_python_common import PgSIG
|
|
21
|
+
from rda_python_common import PgUtil
|
|
22
|
+
from rda_python_common import PgFile
|
|
23
|
+
from rda_python_common import PgDBI
|
|
24
|
+
|
|
25
|
+
RDAPS = {
|
|
26
|
+
'a' : None, # application name
|
|
27
|
+
'h' : None, # remote hostname
|
|
28
|
+
'p' : 0, # process id to be checked
|
|
29
|
+
'P' : 0, # parent process id to be checked
|
|
30
|
+
'u' : None, # login user name
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
#
|
|
34
|
+
# main function to run the application
|
|
35
|
+
#
|
|
36
|
+
def main():
|
|
37
|
+
|
|
38
|
+
optcnt = 0
|
|
39
|
+
argv = sys.argv[1:]
|
|
40
|
+
PgDBI.dssdb_dbname()
|
|
41
|
+
PgLOG.set_suid(PgLOG.PGLOG['EUID'])
|
|
42
|
+
PgLOG.set_help_path(__file__)
|
|
43
|
+
PgLOG.PGLOG['LOGFILE'] = "rdaps.log" # set different log file
|
|
44
|
+
PgLOG.cmdlog("rdaps {}".format(' '.join(argv)))
|
|
45
|
+
|
|
46
|
+
for arg in argv:
|
|
47
|
+
ms = re.match(r'-([ahpPtu])$', arg)
|
|
48
|
+
if ms:
|
|
49
|
+
option = ms.group(1)
|
|
50
|
+
elif re.match(r'-\w+$', arg):
|
|
51
|
+
PgLOG.pglog(arg + ": Unknown Option", PgLOG.LGEREX)
|
|
52
|
+
elif option:
|
|
53
|
+
if RDAPS[option]: PgLOG.pglog("{}: value passed to Option -{} already".format(arg, option), PgLOG.LGEREX)
|
|
54
|
+
if 'pPt'.find(option) > -1:
|
|
55
|
+
RDAPS[option] = int(arg)
|
|
56
|
+
elif option == 'h':
|
|
57
|
+
RDAPS[option] = PgLOG.get_short_host(arg)
|
|
58
|
+
else:
|
|
59
|
+
RDAPS[option] = arg
|
|
60
|
+
option = None
|
|
61
|
+
optcnt += 1
|
|
62
|
+
else:
|
|
63
|
+
ms = re.match(r'^(\d+)$', arg)
|
|
64
|
+
if ms and not RDAPS['p']:
|
|
65
|
+
RDAPS['p'] = int(ms.group(1)) # pid allow value only without leading option
|
|
66
|
+
optcnt += 1
|
|
67
|
+
else:
|
|
68
|
+
PgLOG.pglog(arg + ": Value passed in without Option", PgLOG.LGEREX)
|
|
69
|
+
|
|
70
|
+
if not optcnt: PgLOG.show_usage("rdaps")
|
|
71
|
+
chkloc = 1
|
|
72
|
+
if RDAPS['h']:
|
|
73
|
+
PgFile.local_host_action(RDAPS['h'], "check processes", PgLOG.PGLOG['HOSTNAME'], PgLOG.LGEREX)
|
|
74
|
+
if not PgUtil.pgcmp(RDAPS['h'], PgLOG.PGLOG['SLMNAME'], 1):
|
|
75
|
+
slurm_snapshot()
|
|
76
|
+
chkloc = 0
|
|
77
|
+
elif not PgUtil.pgcmp(RDAPS['h'], PgLOG.PGLOG['PBSNAME'], 1):
|
|
78
|
+
pbs_snapshot()
|
|
79
|
+
chkloc = 0
|
|
80
|
+
if chkloc: process_snapshot()
|
|
81
|
+
|
|
82
|
+
PgLOG.cmdlog()
|
|
83
|
+
PgLOG.pgexit(0)
|
|
84
|
+
|
|
85
|
+
#
|
|
86
|
+
# get a snapshot of a process status
|
|
87
|
+
#
|
|
88
|
+
def process_snapshot():
|
|
89
|
+
|
|
90
|
+
if RDAPS['p']:
|
|
91
|
+
cmd = "ps -p {} -f".format(RDAPS['p'])
|
|
92
|
+
elif RDAPS['P']:
|
|
93
|
+
cmd = "ps --ppid {} -f".format(RDAPS['P'])
|
|
94
|
+
elif RDAPS['u']:
|
|
95
|
+
cmd = "ps -u {} -f".format(RDAPS['u'])
|
|
96
|
+
else:
|
|
97
|
+
cmd = "ps -ef"
|
|
98
|
+
|
|
99
|
+
buf = PgLOG.pgsystem(cmd, PgLOG.LGWNEX, 20)
|
|
100
|
+
|
|
101
|
+
for line in re.split('\n', buf):
|
|
102
|
+
ms = re.match(r'\s*(\w+)\s+(\d+)\s+(\d+)\s+(.*)$', line)
|
|
103
|
+
if ms:
|
|
104
|
+
uid = ms.group(1)
|
|
105
|
+
pid = int(ms.group(2))
|
|
106
|
+
ppid = int(ms.group(3))
|
|
107
|
+
aname = ms.group(4)
|
|
108
|
+
if RDAPS['u'] and RDAPS['u'] != uid: continue
|
|
109
|
+
if RDAPS['p'] and RDAPS['p'] != pid: continue
|
|
110
|
+
if RDAPS['P'] and RDAPS['P'] != ppid: continue
|
|
111
|
+
if RDAPS['a'] and aname.find(RDAPS['a']) < 0: continue
|
|
112
|
+
PgLOG.pglog(re.sub(r' +', ' ', line), PgLOG.LOGWRN)
|
|
113
|
+
|
|
114
|
+
#
|
|
115
|
+
# get a snapshot of a SLURM batch process status
|
|
116
|
+
#
|
|
117
|
+
def slurm_snapshot():
|
|
118
|
+
|
|
119
|
+
qopts = ''
|
|
120
|
+
if RDAPS['u']: qopts += " -u " + RDAPS['u']
|
|
121
|
+
if RDAPS['p']:
|
|
122
|
+
qopts += " -j {}".format(RDAPS['p'])
|
|
123
|
+
else:
|
|
124
|
+
qopts = " -p rda"
|
|
125
|
+
cmd = "squeue -l" + qopts
|
|
126
|
+
|
|
127
|
+
buf = PgLOG.pgsystem(cmd, PgLOG.LOGWRN, 272)
|
|
128
|
+
if not buf:
|
|
129
|
+
if PgLOG.PGLOG['SYSERR'] and PgLOG.PGLOG['SYSERR'].find('Invalid job id specified') < 0:
|
|
130
|
+
PgLOG.pglog(PgLOG.PGLOG['SYSERR'], PgLOG.LGEREX)
|
|
131
|
+
return
|
|
132
|
+
|
|
133
|
+
lines = re.split(r'\n', buf)
|
|
134
|
+
lcnt = len(lines)
|
|
135
|
+
if lcnt < 3: return
|
|
136
|
+
dochk = 1
|
|
137
|
+
for line in lines:
|
|
138
|
+
if not line: continue
|
|
139
|
+
if dochk:
|
|
140
|
+
if re.match(r'^\s*JOBID\s', line): dochk = 0
|
|
141
|
+
else:
|
|
142
|
+
vals = re.split(r'\s+', PgLOG.pgtrim(line))
|
|
143
|
+
if RDAPS['a'] and vals[2] and RDAPS['a'] != vals[2]: continue
|
|
144
|
+
# move user name to front
|
|
145
|
+
val = vals[3]
|
|
146
|
+
vals[3] = vals[2]
|
|
147
|
+
vals[2] = vals[1]
|
|
148
|
+
vals[1] = vals[0]
|
|
149
|
+
vals[0] = val
|
|
150
|
+
PgLOG.pglog(' '.join(vals), PgLOG.LOGWRN)
|
|
151
|
+
|
|
152
|
+
#
|
|
153
|
+
# get a snapshot of a PBS batch process status
|
|
154
|
+
#
|
|
155
|
+
def pbs_snapshot():
|
|
156
|
+
|
|
157
|
+
qopts = ''
|
|
158
|
+
if RDAPS['u']:
|
|
159
|
+
qopts = "-u {}".format(RDAPS['u'])
|
|
160
|
+
if RDAPS['p']:
|
|
161
|
+
if qopts: qopts += ' '
|
|
162
|
+
qopts += str(RDAPS['p'])
|
|
163
|
+
if not qopts: qopts = 'rda'
|
|
164
|
+
|
|
165
|
+
stat = PgSIG.get_pbs_info(qopts, 1, PgLOG.LOGWRN)
|
|
166
|
+
if not stat:
|
|
167
|
+
if PgLOG.PGLOG['SYSERR']: PgLOG.pglog(PgLOG.PGLOG['SYSERR'], PgLOG.LGEREX)
|
|
168
|
+
return
|
|
169
|
+
|
|
170
|
+
lcnt = len(stat['JobID'])
|
|
171
|
+
|
|
172
|
+
ckeys = list(stat.keys())
|
|
173
|
+
kcnt = len(ckeys)
|
|
174
|
+
# moving 'UserName' to the first
|
|
175
|
+
for i in range(kcnt):
|
|
176
|
+
if i > 0 and ckeys[i] == 'UserName':
|
|
177
|
+
j = i
|
|
178
|
+
while j > 0:
|
|
179
|
+
ckeys[j] = ckeys[j-1]
|
|
180
|
+
j -= 1
|
|
181
|
+
ckeys[0] = 'UserName'
|
|
182
|
+
break
|
|
183
|
+
|
|
184
|
+
for i in range(lcnt):
|
|
185
|
+
if RDAPS['a'] and stat['JobName'] and RDAPS['a'] != stat['JobName']: continue
|
|
186
|
+
vals = []
|
|
187
|
+
for k in ckeys:
|
|
188
|
+
vals.append(stat[k][i])
|
|
189
|
+
PgLOG.pglog(' '.join(vals), PgLOG.LOGWRN)
|
|
190
|
+
|
|
191
|
+
#
|
|
192
|
+
# call main() to start program
|
|
193
|
+
#
|
|
194
|
+
if __name__ == "__main__": main()
|