rda-python-miscs 3.0.0__tar.gz → 3.0.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {rda_python_miscs-3.0.0/src/rda_python_miscs.egg-info → rda_python_miscs-3.0.2}/PKG-INFO +14 -9
- {rda_python_miscs-3.0.0 → rda_python_miscs-3.0.2}/README.md +13 -8
- {rda_python_miscs-3.0.0 → rda_python_miscs-3.0.2}/pyproject.toml +15 -15
- rda_python_miscs-3.0.0/src/rda_python_miscs/rdacp.py → rda_python_miscs-3.0.2/src/rda_python_miscs/gdexcp.py +13 -13
- rda_python_miscs-3.0.0/src/rda_python_miscs/rdacp.usg → rda_python_miscs-3.0.2/src/rda_python_miscs/gdexcp.usg +8 -8
- rda_python_miscs-3.0.0/src/rda_python_miscs/rdakill.py → rda_python_miscs-3.0.2/src/rda_python_miscs/gdexkill.py +10 -10
- rda_python_miscs-3.0.0/src/rda_python_miscs/rdakill.usg → rda_python_miscs-3.0.2/src/rda_python_miscs/gdexkill.usg +9 -9
- rda_python_miscs-3.0.0/src/rda_python_miscs/rdamod.py → rda_python_miscs-3.0.2/src/rda_python_miscs/gdexmod.py +16 -16
- rda_python_miscs-3.0.0/src/rda_python_miscs/rdamod.usg → rda_python_miscs-3.0.2/src/rda_python_miscs/gdexmod.usg +13 -13
- rda_python_miscs-3.0.0/src/rda_python_miscs/rdaown.py → rda_python_miscs-3.0.2/src/rda_python_miscs/gdexown.py +19 -19
- rda_python_miscs-3.0.0/src/rda_python_miscs/rdaown.usg → rda_python_miscs-3.0.2/src/rda_python_miscs/gdexown.usg +12 -12
- rda_python_miscs-3.0.0/src/rda_python_miscs/rdaps.py → rda_python_miscs-3.0.2/src/rda_python_miscs/gdexps.py +8 -8
- rda_python_miscs-3.0.0/src/rda_python_miscs/rdaps.usg → rda_python_miscs-3.0.2/src/rda_python_miscs/gdexps.usg +8 -8
- rda_python_miscs-3.0.0/src/rda_python_miscs/rdasub.py → rda_python_miscs-3.0.2/src/rda_python_miscs/gdexsub.py +6 -6
- rda_python_miscs-3.0.0/src/rda_python_miscs/rdasub.usg → rda_python_miscs-3.0.2/src/rda_python_miscs/gdexsub.usg +7 -7
- rda_python_miscs-3.0.0/src/rda_python_miscs/rdazip.py → rda_python_miscs-3.0.2/src/rda_python_miscs/gdexzip.py +8 -8
- rda_python_miscs-3.0.0/src/rda_python_miscs/rdazip.usg → rda_python_miscs-3.0.2/src/rda_python_miscs/gdexzip.usg +7 -7
- {rda_python_miscs-3.0.0 → rda_python_miscs-3.0.2/src/rda_python_miscs.egg-info}/PKG-INFO +14 -9
- {rda_python_miscs-3.0.0 → rda_python_miscs-3.0.2}/src/rda_python_miscs.egg-info/SOURCES.txt +14 -14
- rda_python_miscs-3.0.2/src/rda_python_miscs.egg-info/entry_points.txt +20 -0
- rda_python_miscs-3.0.2/tests/test_miscs.py +16 -0
- rda_python_miscs-3.0.0/src/rda_python_miscs.egg-info/entry_points.txt +0 -20
- rda_python_miscs-3.0.0/tests/test_miscs.py +0 -16
- {rda_python_miscs-3.0.0 → rda_python_miscs-3.0.2}/LICENSE +0 -0
- {rda_python_miscs-3.0.0 → rda_python_miscs-3.0.2}/MANIFEST.in +0 -0
- {rda_python_miscs-3.0.0 → rda_python_miscs-3.0.2}/setup.cfg +0 -0
- {rda_python_miscs-3.0.0 → rda_python_miscs-3.0.2}/src/rda_python_miscs/__init__.py +0 -0
- {rda_python_miscs-3.0.0 → rda_python_miscs-3.0.2}/src/rda_python_miscs/bash_qsub.py +0 -0
- {rda_python_miscs-3.0.0 → rda_python_miscs-3.0.2}/src/rda_python_miscs/bashqsub.py +0 -0
- {rda_python_miscs-3.0.0 → rda_python_miscs-3.0.2}/src/rda_python_miscs/bashqsub.usg +0 -0
- {rda_python_miscs-3.0.0 → rda_python_miscs-3.0.2}/src/rda_python_miscs/gdex_ls.py +0 -0
- {rda_python_miscs-3.0.0 → rda_python_miscs-3.0.2}/src/rda_python_miscs/gdexls.py +0 -0
- {rda_python_miscs-3.0.0 → rda_python_miscs-3.0.2}/src/rda_python_miscs/gdexls.usg +0 -0
- {rda_python_miscs-3.0.0 → rda_python_miscs-3.0.2}/src/rda_python_miscs/gdexls_standalone.py +0 -0
- {rda_python_miscs-3.0.0 → rda_python_miscs-3.0.2}/src/rda_python_miscs/pg_docs.py +0 -0
- {rda_python_miscs-3.0.0 → rda_python_miscs-3.0.2}/src/rda_python_miscs/pg_rst.py +0 -0
- {rda_python_miscs-3.0.0 → rda_python_miscs-3.0.2}/src/rda_python_miscs/pg_rst.usg +0 -0
- {rda_python_miscs-3.0.0 → rda_python_miscs-3.0.2}/src/rda_python_miscs/pg_wget.py +0 -0
- {rda_python_miscs-3.0.0 → rda_python_miscs-3.0.2}/src/rda_python_miscs/pgwget.py +0 -0
- {rda_python_miscs-3.0.0 → rda_python_miscs-3.0.2}/src/rda_python_miscs/pgwget.usg +0 -0
- {rda_python_miscs-3.0.0 → rda_python_miscs-3.0.2}/src/rda_python_miscs/rda_cp.py +0 -0
- {rda_python_miscs-3.0.0 → rda_python_miscs-3.0.2}/src/rda_python_miscs/rda_kill.py +0 -0
- {rda_python_miscs-3.0.0 → rda_python_miscs-3.0.2}/src/rda_python_miscs/rda_mod.py +0 -0
- {rda_python_miscs-3.0.0 → rda_python_miscs-3.0.2}/src/rda_python_miscs/rda_own.py +0 -0
- {rda_python_miscs-3.0.0 → rda_python_miscs-3.0.2}/src/rda_python_miscs/rda_ps.py +0 -0
- {rda_python_miscs-3.0.0 → rda_python_miscs-3.0.2}/src/rda_python_miscs/rda_sub.py +0 -0
- {rda_python_miscs-3.0.0 → rda_python_miscs-3.0.2}/src/rda_python_miscs/rda_zip.py +0 -0
- {rda_python_miscs-3.0.0 → rda_python_miscs-3.0.2}/src/rda_python_miscs/rdals.py +0 -0
- {rda_python_miscs-3.0.0 → rda_python_miscs-3.0.2}/src/rda_python_miscs/rdals.usg +0 -0
- {rda_python_miscs-3.0.0 → rda_python_miscs-3.0.2}/src/rda_python_miscs/rst_templates/index.rst.temp +0 -0
- {rda_python_miscs-3.0.0 → rda_python_miscs-3.0.2}/src/rda_python_miscs/rst_templates/section.rst.temp +0 -0
- {rda_python_miscs-3.0.0 → rda_python_miscs-3.0.2}/src/rda_python_miscs/tcsh_qsub.py +0 -0
- {rda_python_miscs-3.0.0 → rda_python_miscs-3.0.2}/src/rda_python_miscs/tcshqsub.py +0 -0
- {rda_python_miscs-3.0.0 → rda_python_miscs-3.0.2}/src/rda_python_miscs/tcshqsub.usg +0 -0
- {rda_python_miscs-3.0.0 → rda_python_miscs-3.0.2}/src/rda_python_miscs.egg-info/dependency_links.txt +0 -0
- {rda_python_miscs-3.0.0 → rda_python_miscs-3.0.2}/src/rda_python_miscs.egg-info/requires.txt +0 -0
- {rda_python_miscs-3.0.0 → rda_python_miscs-3.0.2}/src/rda_python_miscs.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: rda_python_miscs
|
|
3
|
-
Version: 3.0.
|
|
3
|
+
Version: 3.0.2
|
|
4
4
|
Summary: RDA Python package to hold RDA miscellaneous utility programs
|
|
5
5
|
Author-email: Zaihua Ji <zji@ucar.edu>
|
|
6
6
|
Project-URL: Homepage, https://github.com/NCAR/rda-python-miscs
|
|
@@ -30,21 +30,21 @@ The package provides two categories of programs:
|
|
|
30
30
|
|---------|-------|-------------|
|
|
31
31
|
| `bashqsub` | | Submit a job as a bash batch job on a PBS node via qsub |
|
|
32
32
|
| `tcshqsub` | | Submit a job as a tcsh batch job on a PBS node via qsub |
|
|
33
|
-
| `
|
|
33
|
+
| `gdexsub` | `rdasub` | Submit a command as a nohup background process on the local machine |
|
|
34
34
|
| `pgwget` | | Download remote files by root name pattern, combining downloaded parts into a single file |
|
|
35
35
|
| `gdexls` | | List local files and directories with matching metadata from the GDEX database |
|
|
36
|
-
| `
|
|
37
|
-
| `
|
|
38
|
-
| `
|
|
36
|
+
| `gdexps` | `rdaps` | Show process status for local or PBS batch processes, with filtering by PID, owner, or name |
|
|
37
|
+
| `gdexzip` | `rdazip` | Compress or uncompress files using a supported format |
|
|
38
|
+
| `gdexown` | `rdaown` | Change ownership of files and directories to gdexdata (must be run as root) |
|
|
39
39
|
| `pgrst` | | Convert .usg files to RST and push to gdex-docs-* repos on GitHub for readthedocs.io |
|
|
40
40
|
|
|
41
41
|
**Run as gdexdata via setuid (requires setup below):**
|
|
42
42
|
|
|
43
43
|
| Command | Alias | Connector script | Description |
|
|
44
44
|
|---------|-------|-----------------|-------------|
|
|
45
|
-
| `
|
|
46
|
-
| `
|
|
47
|
-
| `
|
|
45
|
+
| `gdexcp` | `rdacp` | `setuid_gdexcp` / `setuid_rdacp` | Copy files and directories across local, remote, Object Store, or Globus endpoints |
|
|
46
|
+
| `gdexkill` | `rdakill` | `setuid_gdexkill` / `setuid_rdakill` | Kill local processes and their children, or cancel PBS batch jobs |
|
|
47
|
+
| `gdexmod` | `rdamod` | `setuid_gdexmod` / `setuid_rdamod` | Change permission modes for files and directories owned by gdexdata |
|
|
48
48
|
|
|
49
49
|
## Environment setup
|
|
50
50
|
|
|
@@ -103,12 +103,17 @@ pip install rda_python_miscs
|
|
|
103
103
|
|
|
104
104
|
## Setuid Setup
|
|
105
105
|
|
|
106
|
-
The setuid programs (`
|
|
106
|
+
The setuid programs (`gdexcp`, `gdexkill`, `gdexmod` and their `rda*` aliases)
|
|
107
107
|
execute as the common user `PGLOG['COMMONUSER']` (default `gdexdata`) via
|
|
108
108
|
the `rda_python_setuid` mechanism, which is pulled in automatically as a
|
|
109
109
|
dependency. After `pip install` above, choose one of the wiring options
|
|
110
110
|
below.
|
|
111
111
|
|
|
112
|
+
> **Note:** If `rda_python_setuid` is already installed and fully set up in
|
|
113
|
+
> your environment, you can skip the compile step (`-c/--compile`) and the
|
|
114
|
+
> optional `pgstart` step (`-p/--pgstart`). The `-l/--link` step is still
|
|
115
|
+
> required to wire up this package's own setuid programs.
|
|
116
|
+
|
|
112
117
|
### Full setuid install (requires sudo access to COMMONUSER)
|
|
113
118
|
|
|
114
119
|
Run these steps once per environment:
|
|
@@ -13,21 +13,21 @@ The package provides two categories of programs:
|
|
|
13
13
|
|---------|-------|-------------|
|
|
14
14
|
| `bashqsub` | | Submit a job as a bash batch job on a PBS node via qsub |
|
|
15
15
|
| `tcshqsub` | | Submit a job as a tcsh batch job on a PBS node via qsub |
|
|
16
|
-
| `
|
|
16
|
+
| `gdexsub` | `rdasub` | Submit a command as a nohup background process on the local machine |
|
|
17
17
|
| `pgwget` | | Download remote files by root name pattern, combining downloaded parts into a single file |
|
|
18
18
|
| `gdexls` | | List local files and directories with matching metadata from the GDEX database |
|
|
19
|
-
| `
|
|
20
|
-
| `
|
|
21
|
-
| `
|
|
19
|
+
| `gdexps` | `rdaps` | Show process status for local or PBS batch processes, with filtering by PID, owner, or name |
|
|
20
|
+
| `gdexzip` | `rdazip` | Compress or uncompress files using a supported format |
|
|
21
|
+
| `gdexown` | `rdaown` | Change ownership of files and directories to gdexdata (must be run as root) |
|
|
22
22
|
| `pgrst` | | Convert .usg files to RST and push to gdex-docs-* repos on GitHub for readthedocs.io |
|
|
23
23
|
|
|
24
24
|
**Run as gdexdata via setuid (requires setup below):**
|
|
25
25
|
|
|
26
26
|
| Command | Alias | Connector script | Description |
|
|
27
27
|
|---------|-------|-----------------|-------------|
|
|
28
|
-
| `
|
|
29
|
-
| `
|
|
30
|
-
| `
|
|
28
|
+
| `gdexcp` | `rdacp` | `setuid_gdexcp` / `setuid_rdacp` | Copy files and directories across local, remote, Object Store, or Globus endpoints |
|
|
29
|
+
| `gdexkill` | `rdakill` | `setuid_gdexkill` / `setuid_rdakill` | Kill local processes and their children, or cancel PBS batch jobs |
|
|
30
|
+
| `gdexmod` | `rdamod` | `setuid_gdexmod` / `setuid_rdamod` | Change permission modes for files and directories owned by gdexdata |
|
|
31
31
|
|
|
32
32
|
## Environment setup
|
|
33
33
|
|
|
@@ -86,12 +86,17 @@ pip install rda_python_miscs
|
|
|
86
86
|
|
|
87
87
|
## Setuid Setup
|
|
88
88
|
|
|
89
|
-
The setuid programs (`
|
|
89
|
+
The setuid programs (`gdexcp`, `gdexkill`, `gdexmod` and their `rda*` aliases)
|
|
90
90
|
execute as the common user `PGLOG['COMMONUSER']` (default `gdexdata`) via
|
|
91
91
|
the `rda_python_setuid` mechanism, which is pulled in automatically as a
|
|
92
92
|
dependency. After `pip install` above, choose one of the wiring options
|
|
93
93
|
below.
|
|
94
94
|
|
|
95
|
+
> **Note:** If `rda_python_setuid` is already installed and fully set up in
|
|
96
|
+
> your environment, you can skip the compile step (`-c/--compile`) and the
|
|
97
|
+
> optional `pgstart` step (`-p/--pgstart`). The `-l/--link` step is still
|
|
98
|
+
> required to wire up this package's own setuid programs.
|
|
99
|
+
|
|
95
100
|
### Full setuid install (requires sudo access to COMMONUSER)
|
|
96
101
|
|
|
97
102
|
Run these steps once per environment:
|
|
@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
|
|
|
6
6
|
|
|
7
7
|
[project]
|
|
8
8
|
name = "rda_python_miscs"
|
|
9
|
-
version = "3.0.
|
|
9
|
+
version = "3.0.2"
|
|
10
10
|
authors = [
|
|
11
11
|
{ name="Zaihua Ji", email="zji@ucar.edu" },
|
|
12
12
|
]
|
|
@@ -35,20 +35,20 @@ pythonpath = [
|
|
|
35
35
|
[project.scripts]
|
|
36
36
|
"bashqsub" = "rda_python_miscs.bashqsub:main"
|
|
37
37
|
"tcshqsub" = "rda_python_miscs.tcshqsub:main"
|
|
38
|
-
"
|
|
39
|
-
"
|
|
38
|
+
"gdexsub" = "rda_python_miscs.gdexsub:main"
|
|
39
|
+
"rdasub" = "rda_python_miscs.gdexsub:main"
|
|
40
40
|
"pgwget" = "rda_python_miscs.pgwget:main"
|
|
41
41
|
"gdexls" = "rda_python_miscs.gdexls:main"
|
|
42
|
-
"
|
|
43
|
-
"
|
|
44
|
-
"
|
|
45
|
-
"
|
|
46
|
-
"
|
|
47
|
-
"
|
|
48
|
-
"
|
|
49
|
-
"
|
|
50
|
-
"
|
|
51
|
-
"
|
|
52
|
-
"
|
|
53
|
-
"
|
|
42
|
+
"gdexps" = "rda_python_miscs.gdexps:main"
|
|
43
|
+
"rdaps" = "rda_python_miscs.gdexps:main"
|
|
44
|
+
"gdexzip" = "rda_python_miscs.gdexzip:main"
|
|
45
|
+
"rdazip" = "rda_python_miscs.gdexzip:main"
|
|
46
|
+
"gdexown" = "rda_python_miscs.gdexown:main"
|
|
47
|
+
"rdaown" = "rda_python_miscs.gdexown:main"
|
|
48
|
+
"setuid_gdexcp" = "rda_python_miscs.gdexcp:main"
|
|
49
|
+
"setuid_rdacp" = "rda_python_miscs.gdexcp:main"
|
|
50
|
+
"setuid_gdexkill" = "rda_python_miscs.gdexkill:main"
|
|
51
|
+
"setuid_rdakill" = "rda_python_miscs.gdexkill:main"
|
|
52
|
+
"setuid_gdexmod" = "rda_python_miscs.gdexmod:main"
|
|
53
|
+
"setuid_rdamod" = "rda_python_miscs.gdexmod:main"
|
|
54
54
|
"pgrst" = "rda_python_miscs.pg_rst:main"
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
#!/usr/bin/env python3
|
|
2
2
|
##################################################################################
|
|
3
|
-
# Title:
|
|
3
|
+
# Title: gdexcp
|
|
4
4
|
# Author: Zaihua Ji, zji@ucar.edu
|
|
5
5
|
# Date: 10/24/2020
|
|
6
6
|
# 2025-03-10 transferred to package rda_python_miscs from
|
|
7
7
|
# https://github.com/NCAR/rda-utility-programs.git
|
|
8
|
-
# Purpose: copy files locally and remotely by '
|
|
8
|
+
# Purpose: copy files locally and remotely by 'gdexdata'
|
|
9
9
|
# Github: https://github.com/NCAR/rda-python-miscs.git
|
|
10
10
|
##################################################################################
|
|
11
11
|
import re
|
|
@@ -14,16 +14,16 @@ import sys
|
|
|
14
14
|
from os import path as op
|
|
15
15
|
from rda_python_common.pg_file import PgFile
|
|
16
16
|
|
|
17
|
-
class
|
|
18
|
-
"""Copy files and directories locally or between remote hosts via '
|
|
17
|
+
class GdexCp(PgFile):
|
|
18
|
+
"""Copy files and directories locally or between remote hosts via 'gdexdata'.
|
|
19
19
|
|
|
20
20
|
Supports local-to-local, local-to-remote, remote-to-local, and Object Store /
|
|
21
|
-
Globus transfers. Target files are owned by '
|
|
21
|
+
Globus transfers. Target files are owned by 'gdexdata' and created with
|
|
22
22
|
configurable permission modes. Recursive copying is controlled by -r / -R.
|
|
23
23
|
"""
|
|
24
24
|
|
|
25
25
|
def __init__(self):
|
|
26
|
-
"""Initialize
|
|
26
|
+
"""Initialize GdexCp with default copy options and runtime state."""
|
|
27
27
|
super().__init__()
|
|
28
28
|
self.RDACP = {
|
|
29
29
|
'fh': None, # from host name, default to localhost
|
|
@@ -64,8 +64,8 @@ class RdaCp(PgFile):
|
|
|
64
64
|
argv = sys.argv[1:]
|
|
65
65
|
self.set_suid(self.PGLOG['EUID'])
|
|
66
66
|
self.set_help_path(__file__)
|
|
67
|
-
self.PGLOG['LOGFILE'] = "
|
|
68
|
-
self.cmdlog("
|
|
67
|
+
self.PGLOG['LOGFILE'] = "gdexcp.log" # set different log file
|
|
68
|
+
self.cmdlog("gdexcp {} ({})".format(' '.join(argv), self.CINFO['curdir']))
|
|
69
69
|
defopt = option = 'f'
|
|
70
70
|
for arg in argv:
|
|
71
71
|
if re.match(r'-(h|-help)$', arg, re.I):
|
|
@@ -95,7 +95,7 @@ class RdaCp(PgFile):
|
|
|
95
95
|
elif option == 'fh':
|
|
96
96
|
self.CINFO['fhost'] = arg + '-'
|
|
97
97
|
option = defopt
|
|
98
|
-
if dohelp or not self.RDACP['f']: self.show_usage("
|
|
98
|
+
if dohelp or not self.RDACP['f']: self.show_usage("gdexcp")
|
|
99
99
|
|
|
100
100
|
# function to start actions
|
|
101
101
|
def start_actions(self):
|
|
@@ -106,7 +106,7 @@ class RdaCp(PgFile):
|
|
|
106
106
|
when specified, then calls copy_top_list. Logs a summary count on completion.
|
|
107
107
|
"""
|
|
108
108
|
self.dssdb_dbname()
|
|
109
|
-
self.validate_decs_group('
|
|
109
|
+
self.validate_decs_group('gdexcp', self.PGLOG['CURUID'], 1)
|
|
110
110
|
if not self.RDACP['R'] and self.RDACP['r']: self.RDACP['R'] = 1000
|
|
111
111
|
if not self.RDACP['t']:
|
|
112
112
|
self.CINFO['tpath'] = self.RDACP['t'] = "."
|
|
@@ -232,10 +232,10 @@ class RdaCp(PgFile):
|
|
|
232
232
|
|
|
233
233
|
# main function to execute this script
|
|
234
234
|
def main():
|
|
235
|
-
"""Entry point: instantiate
|
|
235
|
+
"""Entry point: instantiate GdexCp, parse arguments, run, and exit."""
|
|
236
236
|
from rda_python_setuid.setup_guide import show_setup_guide
|
|
237
|
-
object =
|
|
238
|
-
show_setup_guide(object, 'rda_python_miscs', ['
|
|
237
|
+
object = GdexCp()
|
|
238
|
+
show_setup_guide(object, 'rda_python_miscs', ['gdexcp', 'gdexkill', 'gdexmod'])
|
|
239
239
|
object.read_parameters()
|
|
240
240
|
object.start_actions()
|
|
241
241
|
object.pgexit(0)
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
|
|
2
2
|
Copy files and directories to a target location. Source and target may each
|
|
3
3
|
reside on the local host, a remote host, an Object Store bucket, or a Globus
|
|
4
|
-
endpoint. Target files are owned by '
|
|
4
|
+
endpoint. Target files are owned by 'gdexdata' and created with configurable
|
|
5
5
|
permission modes.
|
|
6
6
|
|
|
7
|
-
Usage:
|
|
7
|
+
Usage: gdexcp [-f] FromDirectories/Files [-t ToDirectory/FileName] \
|
|
8
8
|
[-fh FromHostName] [-th ToHostName] \
|
|
9
9
|
[-fb FromBucket] [-tb ToBucket] \
|
|
10
10
|
[-fp FromGlobusEndpoint] [-tp ToGlobusEndpoint] \
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
default option, so paths may be given without the -f flag.
|
|
15
15
|
Shell wildcards are supported. Use './' or '*' to copy everything
|
|
16
16
|
in the current directory. Source paths must be readable by user
|
|
17
|
-
'
|
|
17
|
+
'gdexdata'; gdexcp will attempt to fix the mode if they are not;
|
|
18
18
|
|
|
19
19
|
- Option -t, target directory or file name. Defaults to '.' (current
|
|
20
20
|
directory). Multiple source files cannot be copied to a single
|
|
@@ -57,21 +57,21 @@
|
|
|
57
57
|
1. Copy all files and subdirectories under the current directory to a
|
|
58
58
|
remote host:
|
|
59
59
|
|
|
60
|
-
|
|
60
|
+
gdexcp -r -f * -t /PathTo/d277006/ -th castle
|
|
61
61
|
|
|
62
62
|
2. Copy the contents of a specific local directory to a remote location
|
|
63
63
|
(trailing '/' on source omits the directory entry itself):
|
|
64
64
|
|
|
65
|
-
|
|
65
|
+
gdexcp -r -f /PathTo/DirectoryName/ -t /PathTo/d277006/ -th castle
|
|
66
66
|
|
|
67
67
|
Without the trailing '/', DirectoryName itself is also copied:
|
|
68
68
|
|
|
69
|
-
|
|
69
|
+
gdexcp -r -f /PathTo/DirectoryName -t /PathTo/d277006/ -th castle
|
|
70
70
|
|
|
71
71
|
3. Copy a single file to an Object Store bucket:
|
|
72
72
|
|
|
73
|
-
|
|
73
|
+
gdexcp -f /PathTo/myfile.nc -tb my-bucket -t myfile.nc
|
|
74
74
|
|
|
75
75
|
4. Copy files from a remote host to the local current directory:
|
|
76
76
|
|
|
77
|
-
|
|
77
|
+
gdexcp -fh castle -f /PathTo/d277006/myfile.nc
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
#!/usr/bin/env python3
|
|
2
2
|
##################################################################################
|
|
3
|
-
# Title:
|
|
3
|
+
# Title: gdexkill
|
|
4
4
|
# Author: Zaihua Ji, zji@ucar.edu
|
|
5
5
|
# Date: 10/24/2020
|
|
6
6
|
# 2025-03-10 transferred to package rda_python_miscs from
|
|
7
7
|
# https://github.com/NCAR/rda-utility-programs.git
|
|
8
8
|
# Purpose: kill a local or batch process and its child processes for a given
|
|
9
|
-
# running process ID by '
|
|
9
|
+
# running process ID by 'gdexdata'
|
|
10
10
|
# Github: https://github.com/NCAR/rda-python-miscs.git
|
|
11
11
|
##################################################################################
|
|
12
12
|
import re
|
|
@@ -14,7 +14,7 @@ import sys
|
|
|
14
14
|
import time
|
|
15
15
|
from rda_python_common.pg_file import PgFile
|
|
16
16
|
|
|
17
|
-
class
|
|
17
|
+
class GdexKill(PgFile):
|
|
18
18
|
"""Kill local processes or PBS batch jobs by process ID, parent PID, or status.
|
|
19
19
|
|
|
20
20
|
For local processes, sends SIGKILL (-9) to the matched process and all its
|
|
@@ -24,7 +24,7 @@ class RdaKill(PgFile):
|
|
|
24
24
|
"""
|
|
25
25
|
|
|
26
26
|
def __init__(self):
|
|
27
|
-
"""Initialize
|
|
27
|
+
"""Initialize GdexKill with default kill options."""
|
|
28
28
|
super().__init__()
|
|
29
29
|
self.RDAKILL = {
|
|
30
30
|
'a': None, # application name
|
|
@@ -52,8 +52,8 @@ class RdaKill(PgFile):
|
|
|
52
52
|
self.dssdb_dbname()
|
|
53
53
|
self.set_suid(self.PGLOG['EUID'])
|
|
54
54
|
self.set_help_path(__file__)
|
|
55
|
-
self.PGLOG['LOGFILE'] = "
|
|
56
|
-
self.cmdlog("
|
|
55
|
+
self.PGLOG['LOGFILE'] = "gdexkill.log" # set different log file
|
|
56
|
+
self.cmdlog("gdexkill {}".format(' '.join(argv)))
|
|
57
57
|
for arg in argv:
|
|
58
58
|
ms = re.match(r'-([ahpPqsu])$', arg)
|
|
59
59
|
if ms:
|
|
@@ -79,7 +79,7 @@ class RdaKill(PgFile):
|
|
|
79
79
|
optcnt += 1
|
|
80
80
|
else:
|
|
81
81
|
self.pglog(arg + ": pass in value without Option", self.LGEREX)
|
|
82
|
-
if not optcnt: self.show_usage("
|
|
82
|
+
if not optcnt: self.show_usage("gdexkill")
|
|
83
83
|
|
|
84
84
|
# function to start actions
|
|
85
85
|
def start_actions(self):
|
|
@@ -257,10 +257,10 @@ class RdaKill(PgFile):
|
|
|
257
257
|
|
|
258
258
|
# main function to execute this script
|
|
259
259
|
def main():
|
|
260
|
-
"""Entry point: instantiate
|
|
260
|
+
"""Entry point: instantiate GdexKill, parse arguments, run, and exit."""
|
|
261
261
|
from rda_python_setuid.setup_guide import show_setup_guide
|
|
262
|
-
object =
|
|
263
|
-
show_setup_guide(object, 'rda_python_miscs', ['
|
|
262
|
+
object = GdexKill()
|
|
263
|
+
show_setup_guide(object, 'rda_python_miscs', ['gdexcp', 'gdexkill', 'gdexmod'])
|
|
264
264
|
object.read_parameters()
|
|
265
265
|
object.start_actions()
|
|
266
266
|
object.pgexit(0)
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
other filter criteria; or cancel one or more PBS batch jobs by job ID or by
|
|
4
4
|
job status. For PBS operations the -h option must specify the PBS node name.
|
|
5
5
|
|
|
6
|
-
Usage:
|
|
6
|
+
Usage: gdexkill [-h HostName] [-p ProcessID] [-P ParentProcessID] \
|
|
7
7
|
[-s BatchStatus] [-q BatchQueue] [-r] \
|
|
8
8
|
[-u ProcessOwner] [-a ApplicationName]
|
|
9
9
|
|
|
@@ -36,31 +36,31 @@
|
|
|
36
36
|
Without this option all users are included;
|
|
37
37
|
|
|
38
38
|
For local processes, all child processes are also terminated recursively.
|
|
39
|
-
Usage is displayed if
|
|
39
|
+
Usage is displayed if gdexkill is run without any options or arguments.
|
|
40
40
|
|
|
41
41
|
Examples:
|
|
42
42
|
|
|
43
43
|
1. Kill a local process by PID:
|
|
44
44
|
|
|
45
|
-
|
|
46
|
-
|
|
45
|
+
gdexkill -p 13199
|
|
46
|
+
gdexkill 13199
|
|
47
47
|
|
|
48
48
|
2. Kill a local process by PID and verify it matches a specific application:
|
|
49
49
|
|
|
50
|
-
|
|
50
|
+
gdexkill -p 13199 -a dsrqst
|
|
51
51
|
|
|
52
52
|
3. Kill all local processes (and their children) owned by a user:
|
|
53
53
|
|
|
54
|
-
|
|
54
|
+
gdexkill -u zji -a dsrqst
|
|
55
55
|
|
|
56
56
|
4. Kill a PBS batch job by job ID:
|
|
57
57
|
|
|
58
|
-
|
|
58
|
+
gdexkill -h casper-pbs -p 334323
|
|
59
59
|
|
|
60
60
|
5. Kill all running PBS batch jobs for a user in the default queue:
|
|
61
61
|
|
|
62
|
-
|
|
62
|
+
gdexkill -h casper-pbs -s RUN -u zji
|
|
63
63
|
|
|
64
64
|
6. Kill all pending PBS batch jobs in a specific queue:
|
|
65
65
|
|
|
66
|
-
|
|
66
|
+
gdexkill -h casper-pbs -s PEND -q gdex
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
#!/usr/bin/env python3
|
|
2
2
|
##################################################################################
|
|
3
|
-
# Title:
|
|
3
|
+
# Title: gdexmod
|
|
4
4
|
# Author: Zaihua Ji, zji@ucar.edu
|
|
5
5
|
# Date: 10/24/2020
|
|
6
6
|
# 2025-03-10 transferred to package rda_python_miscs from
|
|
7
7
|
# https://github.com/NCAR/rda-utility-programs.git
|
|
8
8
|
# Purpose: change file/directory modes in given one or multiple local directories
|
|
9
|
-
# owned by '
|
|
9
|
+
# owned by 'gdexdata'
|
|
10
10
|
# Github: https://github.com/NCAR/rda-python-miscs.git
|
|
11
11
|
##################################################################################
|
|
12
12
|
import re
|
|
@@ -15,17 +15,17 @@ import sys
|
|
|
15
15
|
from os import path as op
|
|
16
16
|
from rda_python_common.pg_file import PgFile
|
|
17
17
|
|
|
18
|
-
class
|
|
19
|
-
"""Change file and directory permission modes for paths owned by '
|
|
18
|
+
class GdexMod(PgFile):
|
|
19
|
+
"""Change file and directory permission modes for paths owned by 'gdexdata'.
|
|
20
20
|
|
|
21
|
-
Only items owned by '
|
|
21
|
+
Only items owned by 'gdexdata' are changed; items with a different owner are
|
|
22
22
|
logged as errors. Items already at the target mode are silently skipped.
|
|
23
23
|
A leading letter ('D' or 'F') is logged with each changed path to indicate
|
|
24
24
|
its type.
|
|
25
25
|
"""
|
|
26
26
|
|
|
27
27
|
def __init__(self):
|
|
28
|
-
"""Initialize
|
|
28
|
+
"""Initialize GdexMod with default mode-change options and runtime state."""
|
|
29
29
|
super().__init__()
|
|
30
30
|
self.RDAMOD = {
|
|
31
31
|
'd': 0, # 1 to change directory mode
|
|
@@ -55,9 +55,9 @@ class RdaMod(PgFile):
|
|
|
55
55
|
"""
|
|
56
56
|
self.set_suid(self.PGLOG['EUID'])
|
|
57
57
|
self.set_help_path(__file__)
|
|
58
|
-
self.PGLOG['LOGFILE'] = "
|
|
58
|
+
self.PGLOG['LOGFILE'] = "gdexmod.log" # set different log file
|
|
59
59
|
argv = sys.argv[1:]
|
|
60
|
-
self.cmdlog("
|
|
60
|
+
self.cmdlog("gdexmod {} ({})".format(' '.join(argv), self.MINFO['curdir']))
|
|
61
61
|
option = defopt = 'l'
|
|
62
62
|
for arg in argv:
|
|
63
63
|
ms = re.match(r'-(\w)$', arg)
|
|
@@ -80,7 +80,7 @@ class RdaMod(PgFile):
|
|
|
80
80
|
else:
|
|
81
81
|
self.RDAMOD[option] = arg
|
|
82
82
|
option = defopt
|
|
83
|
-
if self.RDAMOD['h'] or not self.MINFO['files']: self.show_usage("
|
|
83
|
+
if self.RDAMOD['h'] or not self.MINFO['files']: self.show_usage("gdexmod")
|
|
84
84
|
|
|
85
85
|
# function to start actions
|
|
86
86
|
def start_actions(self):
|
|
@@ -89,7 +89,7 @@ class RdaMod(PgFile):
|
|
|
89
89
|
if not (self.RDAMOD['d'] or self.RDAMOD['f']):
|
|
90
90
|
self.RDAMOD['d'] = self.RDAMOD['f'] = 1 # both directories and files as default
|
|
91
91
|
if not self.RDAMOD['R'] and self.RDAMOD['r']: self.RDAMOD['R'] = 1000
|
|
92
|
-
self.validate_decs_group('
|
|
92
|
+
self.validate_decs_group('gdexmod', self.PGLOG['CURUID'], 1)
|
|
93
93
|
self.change_top_list(self.MINFO['files'])
|
|
94
94
|
if (self.MINFO['dcnt'] + self.MINFO['fcnt']) > 1:
|
|
95
95
|
msg = ''
|
|
@@ -159,7 +159,7 @@ class RdaMod(PgFile):
|
|
|
159
159
|
"""Change the permission mode of one file or directory.
|
|
160
160
|
|
|
161
161
|
Skips the item if the -f/-d flag for its type is not set, if it is not
|
|
162
|
-
owned by '
|
|
162
|
+
owned by 'gdexdata', or if its current mode already matches the target.
|
|
163
163
|
Logs the old-to-new mode transition on success or an error on owner mismatch.
|
|
164
164
|
Updates MINFO['fcnt'] for files and MINFO['dcnt'] for directories on success.
|
|
165
165
|
|
|
@@ -180,8 +180,8 @@ class RdaMod(PgFile):
|
|
|
180
180
|
if not self.RDAMOD['d']: return 0
|
|
181
181
|
fname = "D" + fname
|
|
182
182
|
mode = self.RDAMOD['D']
|
|
183
|
-
if info['logname'] != "
|
|
184
|
-
return self.pglog("{}: owner {} not
|
|
183
|
+
if info['logname'] != "gdexdata":
|
|
184
|
+
return self.pglog("{}: owner {} not gdexdata".format(fname, info['logname']), self.LOGERR)
|
|
185
185
|
if info['mode'] == mode: return 0 # no need change mode
|
|
186
186
|
if self.set_local_mode(file, info['isfile'], mode, info['mode'], info['logname'], self.LOGWRN):
|
|
187
187
|
if info['isfile']:
|
|
@@ -193,10 +193,10 @@ class RdaMod(PgFile):
|
|
|
193
193
|
|
|
194
194
|
# main function to execute this script
|
|
195
195
|
def main():
|
|
196
|
-
"""Entry point: instantiate
|
|
196
|
+
"""Entry point: instantiate GdexMod, parse arguments, run, and exit."""
|
|
197
197
|
from rda_python_setuid.setup_guide import show_setup_guide
|
|
198
|
-
object =
|
|
199
|
-
show_setup_guide(object, 'rda_python_miscs', ['
|
|
198
|
+
object = GdexMod()
|
|
199
|
+
show_setup_guide(object, 'rda_python_miscs', ['gdexcp', 'gdexkill', 'gdexmod'])
|
|
200
200
|
object.read_parameters()
|
|
201
201
|
object.start_actions()
|
|
202
202
|
object.pgexit(0)
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
|
|
2
|
-
Change permission modes for files and directories owned by '
|
|
3
|
-
items owned by '
|
|
2
|
+
Change permission modes for files and directories owned by 'gdexdata'. Only
|
|
3
|
+
items owned by 'gdexdata' are changed; items with a different owner are logged
|
|
4
4
|
as errors. Items already at the target mode are silently skipped. A leading
|
|
5
5
|
letter is displayed in front of each changed path to indicate its type: 'D'
|
|
6
6
|
for a directory and 'F' for a file.
|
|
7
7
|
|
|
8
|
-
Usage:
|
|
8
|
+
Usage: gdexmod [-d] [-f] [-D DirectoryMode] [-F FileMode] [-h] [-r] \
|
|
9
9
|
[-R RecursiveLevel] Directory/File List
|
|
10
10
|
|
|
11
11
|
- Option -d, change directory modes only. Both directories and files are
|
|
@@ -33,39 +33,39 @@
|
|
|
33
33
|
in the current directory.
|
|
34
34
|
|
|
35
35
|
This utility can be run from any directory. No mode is changed if all
|
|
36
|
-
matched items are already at the target mode or if none are owned by '
|
|
36
|
+
matched items are already at the target mode or if none are owned by 'gdexdata'.
|
|
37
37
|
|
|
38
38
|
Examples:
|
|
39
39
|
|
|
40
40
|
1. Change into the dataset home directory and change modes of all
|
|
41
41
|
immediate contents:
|
|
42
42
|
|
|
43
|
-
cd /PathTo/
|
|
44
|
-
|
|
43
|
+
cd /PathTo/d277006
|
|
44
|
+
gdexmod ./
|
|
45
45
|
|
|
46
46
|
Add -r to recurse into sub-directories:
|
|
47
47
|
|
|
48
|
-
|
|
48
|
+
gdexmod -r ./
|
|
49
49
|
|
|
50
50
|
2. Pass an absolute path with a trailing '/' to process the directory's
|
|
51
51
|
contents (the directory entry itself is not changed):
|
|
52
52
|
|
|
53
|
-
|
|
53
|
+
gdexmod /PathTo/d277006/
|
|
54
54
|
|
|
55
55
|
Without the trailing '/', the directory entry itself is also changed
|
|
56
56
|
unless -r or -R is given:
|
|
57
57
|
|
|
58
|
-
|
|
58
|
+
gdexmod /PathTo/d277006
|
|
59
59
|
|
|
60
60
|
3. Use a relative path from a neighbouring directory:
|
|
61
61
|
|
|
62
|
-
|
|
63
|
-
|
|
62
|
+
gdexmod ../d277006/
|
|
63
|
+
gdexmod ../d277006/*
|
|
64
64
|
|
|
65
65
|
4. Change file modes only, with a non-default target mode:
|
|
66
66
|
|
|
67
|
-
|
|
67
|
+
gdexmod -f -F 660 -r /PathTo/d277006/
|
|
68
68
|
|
|
69
69
|
5. Change directory modes only, recursively:
|
|
70
70
|
|
|
71
|
-
|
|
71
|
+
gdexmod -d -r /PathTo/d277006/
|