rda-python-common 2.1.7__tar.gz → 2.1.8__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rda_python_common-2.1.8/PKG-INFO +305 -0
- rda_python_common-2.1.8/README.md +285 -0
- {rda_python_common-2.1.7 → rda_python_common-2.1.8}/pyproject.toml +3 -2
- rda_python_common-2.1.8/src/rda_python_common/__init__.py +38 -0
- {rda_python_common-2.1.7 → rda_python_common-2.1.8}/src/rda_python_common/pg_dbi.py +7 -7
- {rda_python_common-2.1.7 → rda_python_common-2.1.8}/src/rda_python_common/pg_file.py +2 -2
- {rda_python_common-2.1.7 → rda_python_common-2.1.8}/src/rda_python_common/pg_lock.py +1 -1
- {rda_python_common-2.1.7 → rda_python_common-2.1.8}/src/rda_python_common/pg_log.py +6 -6
- {rda_python_common-2.1.7 → rda_python_common-2.1.8}/src/rda_python_common/pg_opt.py +1 -1
- {rda_python_common-2.1.7 → rda_python_common-2.1.8}/src/rda_python_common/pg_sig.py +1 -1
- {rda_python_common-2.1.7 → rda_python_common-2.1.8}/src/rda_python_common/pg_split.py +3 -2
- {rda_python_common-2.1.7 → rda_python_common-2.1.8}/src/rda_python_common/pg_util.py +24 -23
- rda_python_common-2.1.8/src/rda_python_common/pgpassword.py +127 -0
- rda_python_common-2.1.8/src/rda_python_common.egg-info/PKG-INFO +305 -0
- {rda_python_common-2.1.7 → rda_python_common-2.1.8}/src/rda_python_common.egg-info/requires.txt +1 -0
- rda_python_common-2.1.7/PKG-INFO +0 -21
- rda_python_common-2.1.7/README.md +0 -2
- rda_python_common-2.1.7/src/rda_python_common/__init__.py +0 -0
- rda_python_common-2.1.7/src/rda_python_common/pgpassword.py +0 -86
- rda_python_common-2.1.7/src/rda_python_common.egg-info/PKG-INFO +0 -21
- {rda_python_common-2.1.7 → rda_python_common-2.1.8}/LICENSE +0 -0
- {rda_python_common-2.1.7 → rda_python_common-2.1.8}/setup.cfg +0 -0
- {rda_python_common-2.1.7 → rda_python_common-2.1.8}/src/rda_python_common/PgCMD.py +0 -0
- {rda_python_common-2.1.7 → rda_python_common-2.1.8}/src/rda_python_common/PgDBI.py +0 -0
- {rda_python_common-2.1.7 → rda_python_common-2.1.8}/src/rda_python_common/PgFile.py +0 -0
- {rda_python_common-2.1.7 → rda_python_common-2.1.8}/src/rda_python_common/PgLOG.py +0 -0
- {rda_python_common-2.1.7 → rda_python_common-2.1.8}/src/rda_python_common/PgLock.py +0 -0
- {rda_python_common-2.1.7 → rda_python_common-2.1.8}/src/rda_python_common/PgOPT.py +0 -0
- {rda_python_common-2.1.7 → rda_python_common-2.1.8}/src/rda_python_common/PgSIG.py +0 -0
- {rda_python_common-2.1.7 → rda_python_common-2.1.8}/src/rda_python_common/PgSplit.py +0 -0
- {rda_python_common-2.1.7 → rda_python_common-2.1.8}/src/rda_python_common/PgUtil.py +0 -0
- {rda_python_common-2.1.7 → rda_python_common-2.1.8}/src/rda_python_common/pg_cmd.py +0 -0
- {rda_python_common-2.1.7 → rda_python_common-2.1.8}/src/rda_python_common/pg_password.py +0 -0
- {rda_python_common-2.1.7 → rda_python_common-2.1.8}/src/rda_python_common.egg-info/SOURCES.txt +0 -0
- {rda_python_common-2.1.7 → rda_python_common-2.1.8}/src/rda_python_common.egg-info/dependency_links.txt +0 -0
- {rda_python_common-2.1.7 → rda_python_common-2.1.8}/src/rda_python_common.egg-info/entry_points.txt +0 -0
- {rda_python_common-2.1.7 → rda_python_common-2.1.8}/src/rda_python_common.egg-info/top_level.txt +0 -0
- {rda_python_common-2.1.7 → rda_python_common-2.1.8}/test/test_common.py +0 -0
|
@@ -0,0 +1,305 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: rda_python_common
|
|
3
|
+
Version: 2.1.8
|
|
4
|
+
Summary: RDA Python common library codes shared by other RDA python packages
|
|
5
|
+
Author-email: Zaihua Ji <zji@ucar.edu>
|
|
6
|
+
Project-URL: Homepage, https://github.com/NCAR/rda-python-common
|
|
7
|
+
Classifier: Programming Language :: Python :: 3
|
|
8
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
9
|
+
Classifier: Operating System :: OS Independent
|
|
10
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
11
|
+
Requires-Python: >=3.7
|
|
12
|
+
Description-Content-Type: text/markdown
|
|
13
|
+
License-File: LICENSE
|
|
14
|
+
Requires-Dist: hvac
|
|
15
|
+
Requires-Dist: psycopg2==2.9.10
|
|
16
|
+
Requires-Dist: rda-python-globus
|
|
17
|
+
Requires-Dist: unidecode
|
|
18
|
+
Requires-Dist: hvac
|
|
19
|
+
Dynamic: license-file
|
|
20
|
+
|
|
21
|
+
# rda-python-common
|
|
22
|
+
|
|
23
|
+
Python common library codes to be shared by other RDA python utility programs.
|
|
24
|
+
|
|
25
|
+
## Installing and using in another RDA python repo
|
|
26
|
+
|
|
27
|
+
`rda-python-common` is the foundation that every other `rda-python-*` repo
|
|
28
|
+
builds on. To consume it from a new or existing repo, follow these steps.
|
|
29
|
+
|
|
30
|
+
### 1. Install the package
|
|
31
|
+
|
|
32
|
+
For local development, clone this repo alongside your project and install it
|
|
33
|
+
in editable mode so that changes are picked up without re-installing:
|
|
34
|
+
|
|
35
|
+
```bash
|
|
36
|
+
git clone https://github.com/NCAR/rda-python-common.git
|
|
37
|
+
cd rda-python-common
|
|
38
|
+
pip install -e .
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
For a regular (non-editable) install from a checkout:
|
|
42
|
+
|
|
43
|
+
```bash
|
|
44
|
+
pip install /path/to/rda-python-common
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
For a production install on a system that uses the published distribution:
|
|
48
|
+
|
|
49
|
+
```bash
|
|
50
|
+
pip install rda_python_common
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
The package brings in its own transitive dependencies (`psycopg2-binary`,
|
|
54
|
+
`rda-python-globus`, `unidecode`, `hvac`).
|
|
55
|
+
|
|
56
|
+
### 2. Declare it as a dependency in your project
|
|
57
|
+
|
|
58
|
+
Add `rda_python_common` to the `dependencies` list of your project's
|
|
59
|
+
`pyproject.toml` so that downstream installs pull it in automatically:
|
|
60
|
+
|
|
61
|
+
```toml
|
|
62
|
+
[project]
|
|
63
|
+
name = "rda_python_yourtool"
|
|
64
|
+
version = "0.1.0"
|
|
65
|
+
dependencies = [
|
|
66
|
+
"rda_python_common",
|
|
67
|
+
# ... other deps
|
|
68
|
+
]
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
This is the same pattern used by `rda-python-dsarch`, `rda-python-dsupdt`,
|
|
72
|
+
`rda-python-dsrqst`, `rda-python-dscheck`, `rda-python-metrics`, and
|
|
73
|
+
`rda-python-miscs`.
|
|
74
|
+
|
|
75
|
+
### 3. Import the modules you need
|
|
76
|
+
|
|
77
|
+
Two import styles are supported (see [Usage examples](#usage-examples) below):
|
|
78
|
+
|
|
79
|
+
```python
|
|
80
|
+
# Preferred for new code -- import the class from the lower-case module
|
|
81
|
+
from rda_python_common.pg_log import PgLOG
|
|
82
|
+
from rda_python_common.pg_dbi import PgDBI
|
|
83
|
+
|
|
84
|
+
# Legacy module-style imports remain supported for back-compatibility
|
|
85
|
+
from rda_python_common import PgLOG, PgDBI
|
|
86
|
+
PgLOG.pglog("hello", PgLOG.LOGWRN)
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
### 4. Verify the install
|
|
90
|
+
|
|
91
|
+
```bash
|
|
92
|
+
python -c "import rda_python_common; print(rda_python_common.__version__)"
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
You should see the installed version (currently `2.1.8`). If the import
|
|
96
|
+
fails, double-check that the active Python environment is the one where you
|
|
97
|
+
ran `pip install`.
|
|
98
|
+
|
|
99
|
+
## Modules
|
|
100
|
+
|
|
101
|
+
All shared functionality lives under `src/rda_python_common/` and is organised as
|
|
102
|
+
a single-inheritance class hierarchy. Each module defines exactly one class;
|
|
103
|
+
later classes extend earlier ones, so an application that instantiates the
|
|
104
|
+
top-of-chain class (typically `PgOPT` or `PgCMD`) gets every helper through one
|
|
105
|
+
object.
|
|
106
|
+
|
|
107
|
+
Inheritance tree (top-down; multi-inheritance shown as two arrows
|
|
108
|
+
converging on the same child):
|
|
109
|
+
|
|
110
|
+
```
|
|
111
|
+
PgLOG
|
|
112
|
+
┌────┴────┐
|
|
113
|
+
▼ ▼
|
|
114
|
+
PgUtil PgDBI
|
|
115
|
+
│ │ │ │ │
|
|
116
|
+
│ └────┐ ┌─┘ │ └─► PgPassword
|
|
117
|
+
│ ▼ ▼ │
|
|
118
|
+
│ PgSplit │ (multi-inherits
|
|
119
|
+
│ │ PgUtil + PgDBI)
|
|
120
|
+
│ ▼
|
|
121
|
+
│ PgSIG
|
|
122
|
+
│ │
|
|
123
|
+
│ ┌──────────┘
|
|
124
|
+
▼ ▼
|
|
125
|
+
PgFile (multi-inherits
|
|
126
|
+
│ PgUtil + PgSIG)
|
|
127
|
+
├─► PgOPT
|
|
128
|
+
│
|
|
129
|
+
└─► PgLock
|
|
130
|
+
│
|
|
131
|
+
└─► PgCMD
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
The tree is single inheritance everywhere except at two join points:
|
|
135
|
+
|
|
136
|
+
- **`PgFile(PgUtil, PgSIG)`** — combines date/record utilities (`PgUtil`
|
|
137
|
+
via `PgLOG`) with daemon/signal/DB control (`PgSIG` → `PgDBI` → `PgLOG`),
|
|
138
|
+
so its descendants `PgOPT`, `PgLock`, and `PgCMD` inherit logging, DB,
|
|
139
|
+
util, signal, and file facilities through one MRO.
|
|
140
|
+
- **`PgSplit(PgUtil, PgDBI)`** — combines record-manipulation helpers
|
|
141
|
+
(`PgUtil`) with the `pgadd`/`pgget`/`pgmget`/`pgupdt`/`pgdel` DB
|
|
142
|
+
operations (`PgDBI`) it needs to keep the shared `wfile` table and the
|
|
143
|
+
per-dataset `wfile_<dsid>` partitions in sync.
|
|
144
|
+
|
|
145
|
+
- **`pg_log.py`** — `PgLOG`. Root of the hierarchy. Provides the central
|
|
146
|
+
logging facility (bit-mask `logact` flags such as `MSGLOG`, `WARNLG`,
|
|
147
|
+
`ERRLOG`, `EXITLG`), e-mail dispatch, system-command execution, process
|
|
148
|
+
metadata lookup, and the global `PGLOG` settings dictionary used by every
|
|
149
|
+
other module.
|
|
150
|
+
|
|
151
|
+
- **`pg_util.py`** — `PgUtil(PgLOG)`. Miscellaneous date/time, dataset-ID,
|
|
152
|
+
and column-oriented record-manipulation helpers. Holds the `DATEFMTS`
|
|
153
|
+
regex table, `MONTHS`/`MNS`/`WDAYS`/`WDS` lookup lists, and the `MDAYS`
|
|
154
|
+
days-per-month array used for date arithmetic, formatting, parsing, and
|
|
155
|
+
record sort/search/classification across all RDA tools.
|
|
156
|
+
|
|
157
|
+
- **`pg_file.py`** — `PgFile(PgUtil, PgSIG)`. Unified file-operation layer
|
|
158
|
+
spanning local file systems, remote hosts (rsync/ssh/scp), AWS S3 / object
|
|
159
|
+
store, and Globus endpoints. Used by `rdacp`, `dsarch`, `dsupdt`, and
|
|
160
|
+
related tools whenever data is moved, listed, or stat-ed.
|
|
161
|
+
|
|
162
|
+
- **`pg_lock.py`** — `PgLock(PgFile)`. RDADB record-locking primitives for
|
|
163
|
+
the `dscheck`, `dsrqst`, `dlupdt`, `dcupdt`, `ptrqst`, and `dataset`
|
|
164
|
+
tables. Acquires, refreshes, and releases per-record locks so that
|
|
165
|
+
long-running batch jobs coordinate cleanly.
|
|
166
|
+
|
|
167
|
+
- **`pg_dbi.py`** — `PgDBI(PgLOG)`. PostgreSQL database interface built on
|
|
168
|
+
`psycopg2`. Wraps connection management, batch `INSERT`/`SELECT`/
|
|
169
|
+
`UPDATE`/`DELETE`, transaction control, and credential lookup from
|
|
170
|
+
`.pgpass` or OpenBao. All RDA tools talk to the `rdadb` database through
|
|
171
|
+
this class.
|
|
172
|
+
|
|
173
|
+
- **`pg_sig.py`** — `PgSIG(PgDBI)`. Daemon process control, POSIX signal
|
|
174
|
+
handling, child/background-process management, and PBS/Torque batch-job
|
|
175
|
+
status queries. Provides the `PGSIG` runtime dictionary plus `VUSERS`,
|
|
176
|
+
`CPIDS`, `CBIDS`, and `SDUMP` tables that drive RDA daemon programs.
|
|
177
|
+
|
|
178
|
+
- **`pg_cmd.py`** — `PgCMD(PgLock)`. Manages `dscheck` batch and delayed-
|
|
179
|
+
mode command tracking. Records, updates, and reaps the per-command rows
|
|
180
|
+
that let RDA utilities resume or be monitored across PBS batch jobs.
|
|
181
|
+
|
|
182
|
+
- **`pg_split.py`** — `PgSplit(PgUtil, PgDBI)`. Synchronises `wfile` records
|
|
183
|
+
between the shared `wfile` table and the per-dataset `wfile_<dsid>`
|
|
184
|
+
partition tables. Provides compare/add/update/delete helpers used when
|
|
185
|
+
archiving or reconciling dataset file inventories.
|
|
186
|
+
|
|
187
|
+
- **`pg_opt.py`** — `PgOPT(PgFile)`. Command-line option parsing and
|
|
188
|
+
application configuration framework for RDA tools (`dsarch`, `dsupdt`,
|
|
189
|
+
`dsrqst`, ...). Holds the master `OPTS` definition table, parsed
|
|
190
|
+
`params`, command-line vs. input-file option tracking (`CMDOPTS`/
|
|
191
|
+
`INOPTS`), output formatting, dataset/help/media/storage/backup type
|
|
192
|
+
maps, and the global `PGOPT` settings.
|
|
193
|
+
|
|
194
|
+
- **`pgpassword.py`** — `PgPassword(PgDBI)`. Standalone CLI entry point
|
|
195
|
+
(`pgpassword`) that resolves a PostgreSQL login password from OpenBao
|
|
196
|
+
(`get_baopassword`) or `~/.pgpass` (`get_pgpassword()`) given database/schema/
|
|
197
|
+
host/port/user selectors via `-d`, `-c`, `-h`, `-p`, `-u`, `-l`, `-k`.
|
|
198
|
+
Prints the resolved password to stdout so shell scripts can capture it.
|
|
199
|
+
|
|
200
|
+
## Usage examples
|
|
201
|
+
|
|
202
|
+
Each class lives in its own submodule. Import the class you need, then
|
|
203
|
+
either instantiate it directly or subclass it to add application-specific
|
|
204
|
+
state and methods.
|
|
205
|
+
|
|
206
|
+
### 1. Direct instantiation — use the helpers as-is
|
|
207
|
+
|
|
208
|
+
```python
|
|
209
|
+
# Logging only
|
|
210
|
+
from rda_python_common.pg_log import PgLOG
|
|
211
|
+
|
|
212
|
+
log = PgLOG()
|
|
213
|
+
log.pglog("dsarch started", log.LOGWRN)
|
|
214
|
+
|
|
215
|
+
# Database access (PgDBI inherits PgLOG, so you get logging too)
|
|
216
|
+
from rda_python_common.pg_dbi import PgDBI
|
|
217
|
+
|
|
218
|
+
db = PgDBI()
|
|
219
|
+
rec = db.pgget('dataset', 'dsid, title', "dsid = 'd633000'")
|
|
220
|
+
print(rec)
|
|
221
|
+
```
|
|
222
|
+
|
|
223
|
+
### 2. Subclassing a single common class
|
|
224
|
+
|
|
225
|
+
```python
|
|
226
|
+
# A small utility that needs date/record helpers plus logging.
|
|
227
|
+
from rda_python_common.pg_util import PgUtil
|
|
228
|
+
|
|
229
|
+
class DateReport(PgUtil):
|
|
230
|
+
def __init__(self):
|
|
231
|
+
super().__init__() # initialise PgUtil (and PgLOG)
|
|
232
|
+
self.today = self.curtime() # method inherited from PgUtil
|
|
233
|
+
|
|
234
|
+
def run(self):
|
|
235
|
+
self.pglog(f"report date: {self.today}", self.LOGWRN)
|
|
236
|
+
|
|
237
|
+
DateReport().run()
|
|
238
|
+
```
|
|
239
|
+
|
|
240
|
+
### 3. Subclassing one of the multi-inheriting joins
|
|
241
|
+
|
|
242
|
+
```python
|
|
243
|
+
# A worker that needs file I/O (PgFile) and dscheck command tracking (PgCMD).
|
|
244
|
+
# PgCMD already extends PgFile via PgLock, so a single base is enough.
|
|
245
|
+
from rda_python_common.pg_cmd import PgCMD
|
|
246
|
+
|
|
247
|
+
class Worker(PgCMD):
|
|
248
|
+
def __init__(self):
|
|
249
|
+
super().__init__()
|
|
250
|
+
self.jobs = []
|
|
251
|
+
|
|
252
|
+
def archive_one(self, src, dst):
|
|
253
|
+
# PgFile method, available through the inheritance chain
|
|
254
|
+
self.local_copy_local(src, dst)
|
|
255
|
+
# PgDBI method, available through PgCMD -> PgLock -> PgFile -> PgSIG -> PgDBI
|
|
256
|
+
self.pgupdt('wfile', {'status': 'A'}, f"wfile = '{dst}'")
|
|
257
|
+
|
|
258
|
+
Worker().archive_one('/in/file', '/out/file')
|
|
259
|
+
```
|
|
260
|
+
|
|
261
|
+
### 4. Combining multiple common classes (application action class)
|
|
262
|
+
|
|
263
|
+
This mirrors how RDA tools such as `dsarch` are structured. The leaf class
|
|
264
|
+
multi-inherits several common classes so a single object exposes options,
|
|
265
|
+
command tracking, and wfile splitting.
|
|
266
|
+
|
|
267
|
+
```python
|
|
268
|
+
# Excerpt of the pattern used by rda_python_dsarch/dsarch.py
|
|
269
|
+
from rda_python_common.pg_opt import PgOPT
|
|
270
|
+
from rda_python_common.pg_cmd import PgCMD
|
|
271
|
+
from rda_python_common.pg_split import PgSplit
|
|
272
|
+
|
|
273
|
+
class PgArch(PgOPT, PgCMD, PgSplit):
|
|
274
|
+
"""Shared state + helpers for a CLI archiving tool."""
|
|
275
|
+
def __init__(self):
|
|
276
|
+
super().__init__()
|
|
277
|
+
self.RTPATH = {} # runtime path cache
|
|
278
|
+
self.OPTS = {} # option table (populated by subclass)
|
|
279
|
+
|
|
280
|
+
class DsArch(PgArch):
|
|
281
|
+
def __init__(self):
|
|
282
|
+
super().__init__()
|
|
283
|
+
self.ALLCNT = self.ADDCNT = self.MODCNT = 0
|
|
284
|
+
|
|
285
|
+
def main(self):
|
|
286
|
+
self.read_parameters() # from PgOPT
|
|
287
|
+
self.start_actions() # dispatch
|
|
288
|
+
|
|
289
|
+
if __name__ == "__main__":
|
|
290
|
+
DsArch().main()
|
|
291
|
+
```
|
|
292
|
+
|
|
293
|
+
### 5. Reading a PostgreSQL password from OpenBao or ~/.pgpass
|
|
294
|
+
|
|
295
|
+
```python
|
|
296
|
+
from rda_python_common.pgpassword import PgPassword
|
|
297
|
+
|
|
298
|
+
pw = PgPassword()
|
|
299
|
+
pw.default_scinfo('rdadb', 'dssdb', 'rda-pgdb', 'gdexweb', None, 5432)
|
|
300
|
+
password = pw.get_baopassword() or pw.get_pgpassword()
|
|
301
|
+
```
|
|
302
|
+
|
|
303
|
+
In every case `super().__init__()` cooperates correctly across the
|
|
304
|
+
multi-inheriting joins (`PgFile` and `PgSplit`), so subclasses only need
|
|
305
|
+
to call it once.
|
|
@@ -0,0 +1,285 @@
|
|
|
1
|
+
# rda-python-common
|
|
2
|
+
|
|
3
|
+
Python common library codes to be shared by other RDA python utility programs.
|
|
4
|
+
|
|
5
|
+
## Installing and using in another RDA python repo
|
|
6
|
+
|
|
7
|
+
`rda-python-common` is the foundation that every other `rda-python-*` repo
|
|
8
|
+
builds on. To consume it from a new or existing repo, follow these steps.
|
|
9
|
+
|
|
10
|
+
### 1. Install the package
|
|
11
|
+
|
|
12
|
+
For local development, clone this repo alongside your project and install it
|
|
13
|
+
in editable mode so that changes are picked up without re-installing:
|
|
14
|
+
|
|
15
|
+
```bash
|
|
16
|
+
git clone https://github.com/NCAR/rda-python-common.git
|
|
17
|
+
cd rda-python-common
|
|
18
|
+
pip install -e .
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
For a regular (non-editable) install from a checkout:
|
|
22
|
+
|
|
23
|
+
```bash
|
|
24
|
+
pip install /path/to/rda-python-common
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
For a production install on a system that uses the published distribution:
|
|
28
|
+
|
|
29
|
+
```bash
|
|
30
|
+
pip install rda_python_common
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
The package brings in its own transitive dependencies (`psycopg2-binary`,
|
|
34
|
+
`rda-python-globus`, `unidecode`, `hvac`).
|
|
35
|
+
|
|
36
|
+
### 2. Declare it as a dependency in your project
|
|
37
|
+
|
|
38
|
+
Add `rda_python_common` to the `dependencies` list of your project's
|
|
39
|
+
`pyproject.toml` so that downstream installs pull it in automatically:
|
|
40
|
+
|
|
41
|
+
```toml
|
|
42
|
+
[project]
|
|
43
|
+
name = "rda_python_yourtool"
|
|
44
|
+
version = "0.1.0"
|
|
45
|
+
dependencies = [
|
|
46
|
+
"rda_python_common",
|
|
47
|
+
# ... other deps
|
|
48
|
+
]
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
This is the same pattern used by `rda-python-dsarch`, `rda-python-dsupdt`,
|
|
52
|
+
`rda-python-dsrqst`, `rda-python-dscheck`, `rda-python-metrics`, and
|
|
53
|
+
`rda-python-miscs`.
|
|
54
|
+
|
|
55
|
+
### 3. Import the modules you need
|
|
56
|
+
|
|
57
|
+
Two import styles are supported (see [Usage examples](#usage-examples) below):
|
|
58
|
+
|
|
59
|
+
```python
|
|
60
|
+
# Preferred for new code -- import the class from the lower-case module
|
|
61
|
+
from rda_python_common.pg_log import PgLOG
|
|
62
|
+
from rda_python_common.pg_dbi import PgDBI
|
|
63
|
+
|
|
64
|
+
# Legacy module-style imports remain supported for back-compatibility
|
|
65
|
+
from rda_python_common import PgLOG, PgDBI
|
|
66
|
+
PgLOG.pglog("hello", PgLOG.LOGWRN)
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
### 4. Verify the install
|
|
70
|
+
|
|
71
|
+
```bash
|
|
72
|
+
python -c "import rda_python_common; print(rda_python_common.__version__)"
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
You should see the installed version (currently `2.1.8`). If the import
|
|
76
|
+
fails, double-check that the active Python environment is the one where you
|
|
77
|
+
ran `pip install`.
|
|
78
|
+
|
|
79
|
+
## Modules
|
|
80
|
+
|
|
81
|
+
All shared functionality lives under `src/rda_python_common/` and is organised as
|
|
82
|
+
a single-inheritance class hierarchy. Each module defines exactly one class;
|
|
83
|
+
later classes extend earlier ones, so an application that instantiates the
|
|
84
|
+
top-of-chain class (typically `PgOPT` or `PgCMD`) gets every helper through one
|
|
85
|
+
object.
|
|
86
|
+
|
|
87
|
+
Inheritance tree (top-down; multi-inheritance shown as two arrows
|
|
88
|
+
converging on the same child):
|
|
89
|
+
|
|
90
|
+
```
|
|
91
|
+
PgLOG
|
|
92
|
+
┌────┴────┐
|
|
93
|
+
▼ ▼
|
|
94
|
+
PgUtil PgDBI
|
|
95
|
+
│ │ │ │ │
|
|
96
|
+
│ └────┐ ┌─┘ │ └─► PgPassword
|
|
97
|
+
│ ▼ ▼ │
|
|
98
|
+
│ PgSplit │ (multi-inherits
|
|
99
|
+
│ │ PgUtil + PgDBI)
|
|
100
|
+
│ ▼
|
|
101
|
+
│ PgSIG
|
|
102
|
+
│ │
|
|
103
|
+
│ ┌──────────┘
|
|
104
|
+
▼ ▼
|
|
105
|
+
PgFile (multi-inherits
|
|
106
|
+
│ PgUtil + PgSIG)
|
|
107
|
+
├─► PgOPT
|
|
108
|
+
│
|
|
109
|
+
└─► PgLock
|
|
110
|
+
│
|
|
111
|
+
└─► PgCMD
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
The tree is single inheritance everywhere except at two join points:
|
|
115
|
+
|
|
116
|
+
- **`PgFile(PgUtil, PgSIG)`** — combines date/record utilities (`PgUtil`
|
|
117
|
+
via `PgLOG`) with daemon/signal/DB control (`PgSIG` → `PgDBI` → `PgLOG`),
|
|
118
|
+
so its descendants `PgOPT`, `PgLock`, and `PgCMD` inherit logging, DB,
|
|
119
|
+
util, signal, and file facilities through one MRO.
|
|
120
|
+
- **`PgSplit(PgUtil, PgDBI)`** — combines record-manipulation helpers
|
|
121
|
+
(`PgUtil`) with the `pgadd`/`pgget`/`pgmget`/`pgupdt`/`pgdel` DB
|
|
122
|
+
operations (`PgDBI`) it needs to keep the shared `wfile` table and the
|
|
123
|
+
per-dataset `wfile_<dsid>` partitions in sync.
|
|
124
|
+
|
|
125
|
+
- **`pg_log.py`** — `PgLOG`. Root of the hierarchy. Provides the central
|
|
126
|
+
logging facility (bit-mask `logact` flags such as `MSGLOG`, `WARNLG`,
|
|
127
|
+
`ERRLOG`, `EXITLG`), e-mail dispatch, system-command execution, process
|
|
128
|
+
metadata lookup, and the global `PGLOG` settings dictionary used by every
|
|
129
|
+
other module.
|
|
130
|
+
|
|
131
|
+
- **`pg_util.py`** — `PgUtil(PgLOG)`. Miscellaneous date/time, dataset-ID,
|
|
132
|
+
and column-oriented record-manipulation helpers. Holds the `DATEFMTS`
|
|
133
|
+
regex table, `MONTHS`/`MNS`/`WDAYS`/`WDS` lookup lists, and the `MDAYS`
|
|
134
|
+
days-per-month array used for date arithmetic, formatting, parsing, and
|
|
135
|
+
record sort/search/classification across all RDA tools.
|
|
136
|
+
|
|
137
|
+
- **`pg_file.py`** — `PgFile(PgUtil, PgSIG)`. Unified file-operation layer
|
|
138
|
+
spanning local file systems, remote hosts (rsync/ssh/scp), AWS S3 / object
|
|
139
|
+
store, and Globus endpoints. Used by `rdacp`, `dsarch`, `dsupdt`, and
|
|
140
|
+
related tools whenever data is moved, listed, or stat-ed.
|
|
141
|
+
|
|
142
|
+
- **`pg_lock.py`** — `PgLock(PgFile)`. RDADB record-locking primitives for
|
|
143
|
+
the `dscheck`, `dsrqst`, `dlupdt`, `dcupdt`, `ptrqst`, and `dataset`
|
|
144
|
+
tables. Acquires, refreshes, and releases per-record locks so that
|
|
145
|
+
long-running batch jobs coordinate cleanly.
|
|
146
|
+
|
|
147
|
+
- **`pg_dbi.py`** — `PgDBI(PgLOG)`. PostgreSQL database interface built on
|
|
148
|
+
`psycopg2`. Wraps connection management, batch `INSERT`/`SELECT`/
|
|
149
|
+
`UPDATE`/`DELETE`, transaction control, and credential lookup from
|
|
150
|
+
`.pgpass` or OpenBao. All RDA tools talk to the `rdadb` database through
|
|
151
|
+
this class.
|
|
152
|
+
|
|
153
|
+
- **`pg_sig.py`** — `PgSIG(PgDBI)`. Daemon process control, POSIX signal
|
|
154
|
+
handling, child/background-process management, and PBS/Torque batch-job
|
|
155
|
+
status queries. Provides the `PGSIG` runtime dictionary plus `VUSERS`,
|
|
156
|
+
`CPIDS`, `CBIDS`, and `SDUMP` tables that drive RDA daemon programs.
|
|
157
|
+
|
|
158
|
+
- **`pg_cmd.py`** — `PgCMD(PgLock)`. Manages `dscheck` batch and delayed-
|
|
159
|
+
mode command tracking. Records, updates, and reaps the per-command rows
|
|
160
|
+
that let RDA utilities resume or be monitored across PBS batch jobs.
|
|
161
|
+
|
|
162
|
+
- **`pg_split.py`** — `PgSplit(PgUtil, PgDBI)`. Synchronises `wfile` records
|
|
163
|
+
between the shared `wfile` table and the per-dataset `wfile_<dsid>`
|
|
164
|
+
partition tables. Provides compare/add/update/delete helpers used when
|
|
165
|
+
archiving or reconciling dataset file inventories.
|
|
166
|
+
|
|
167
|
+
- **`pg_opt.py`** — `PgOPT(PgFile)`. Command-line option parsing and
|
|
168
|
+
application configuration framework for RDA tools (`dsarch`, `dsupdt`,
|
|
169
|
+
`dsrqst`, ...). Holds the master `OPTS` definition table, parsed
|
|
170
|
+
`params`, command-line vs. input-file option tracking (`CMDOPTS`/
|
|
171
|
+
`INOPTS`), output formatting, dataset/help/media/storage/backup type
|
|
172
|
+
maps, and the global `PGOPT` settings.
|
|
173
|
+
|
|
174
|
+
- **`pgpassword.py`** — `PgPassword(PgDBI)`. Standalone CLI entry point
|
|
175
|
+
(`pgpassword`) that resolves a PostgreSQL login password from OpenBao
|
|
176
|
+
(`get_baopassword`) or `~/.pgpass` (`get_pgpassword()`) given database/schema/
|
|
177
|
+
host/port/user selectors via `-d`, `-c`, `-h`, `-p`, `-u`, `-l`, `-k`.
|
|
178
|
+
Prints the resolved password to stdout so shell scripts can capture it.
|
|
179
|
+
|
|
180
|
+
## Usage examples
|
|
181
|
+
|
|
182
|
+
Each class lives in its own submodule. Import the class you need, then
|
|
183
|
+
either instantiate it directly or subclass it to add application-specific
|
|
184
|
+
state and methods.
|
|
185
|
+
|
|
186
|
+
### 1. Direct instantiation — use the helpers as-is
|
|
187
|
+
|
|
188
|
+
```python
|
|
189
|
+
# Logging only
|
|
190
|
+
from rda_python_common.pg_log import PgLOG
|
|
191
|
+
|
|
192
|
+
log = PgLOG()
|
|
193
|
+
log.pglog("dsarch started", log.LOGWRN)
|
|
194
|
+
|
|
195
|
+
# Database access (PgDBI inherits PgLOG, so you get logging too)
|
|
196
|
+
from rda_python_common.pg_dbi import PgDBI
|
|
197
|
+
|
|
198
|
+
db = PgDBI()
|
|
199
|
+
rec = db.pgget('dataset', 'dsid, title', "dsid = 'd633000'")
|
|
200
|
+
print(rec)
|
|
201
|
+
```
|
|
202
|
+
|
|
203
|
+
### 2. Subclassing a single common class
|
|
204
|
+
|
|
205
|
+
```python
|
|
206
|
+
# A small utility that needs date/record helpers plus logging.
|
|
207
|
+
from rda_python_common.pg_util import PgUtil
|
|
208
|
+
|
|
209
|
+
class DateReport(PgUtil):
|
|
210
|
+
def __init__(self):
|
|
211
|
+
super().__init__() # initialise PgUtil (and PgLOG)
|
|
212
|
+
self.today = self.curtime() # method inherited from PgUtil
|
|
213
|
+
|
|
214
|
+
def run(self):
|
|
215
|
+
self.pglog(f"report date: {self.today}", self.LOGWRN)
|
|
216
|
+
|
|
217
|
+
DateReport().run()
|
|
218
|
+
```
|
|
219
|
+
|
|
220
|
+
### 3. Subclassing one of the multi-inheriting joins
|
|
221
|
+
|
|
222
|
+
```python
|
|
223
|
+
# A worker that needs file I/O (PgFile) and dscheck command tracking (PgCMD).
|
|
224
|
+
# PgCMD already extends PgFile via PgLock, so a single base is enough.
|
|
225
|
+
from rda_python_common.pg_cmd import PgCMD
|
|
226
|
+
|
|
227
|
+
class Worker(PgCMD):
|
|
228
|
+
def __init__(self):
|
|
229
|
+
super().__init__()
|
|
230
|
+
self.jobs = []
|
|
231
|
+
|
|
232
|
+
def archive_one(self, src, dst):
|
|
233
|
+
# PgFile method, available through the inheritance chain
|
|
234
|
+
self.local_copy_local(src, dst)
|
|
235
|
+
# PgDBI method, available through PgCMD -> PgLock -> PgFile -> PgSIG -> PgDBI
|
|
236
|
+
self.pgupdt('wfile', {'status': 'A'}, f"wfile = '{dst}'")
|
|
237
|
+
|
|
238
|
+
Worker().archive_one('/in/file', '/out/file')
|
|
239
|
+
```
|
|
240
|
+
|
|
241
|
+
### 4. Combining multiple common classes (application action class)
|
|
242
|
+
|
|
243
|
+
This mirrors how RDA tools such as `dsarch` are structured. The leaf class
|
|
244
|
+
multi-inherits several common classes so a single object exposes options,
|
|
245
|
+
command tracking, and wfile splitting.
|
|
246
|
+
|
|
247
|
+
```python
|
|
248
|
+
# Excerpt of the pattern used by rda_python_dsarch/dsarch.py
|
|
249
|
+
from rda_python_common.pg_opt import PgOPT
|
|
250
|
+
from rda_python_common.pg_cmd import PgCMD
|
|
251
|
+
from rda_python_common.pg_split import PgSplit
|
|
252
|
+
|
|
253
|
+
class PgArch(PgOPT, PgCMD, PgSplit):
|
|
254
|
+
"""Shared state + helpers for a CLI archiving tool."""
|
|
255
|
+
def __init__(self):
|
|
256
|
+
super().__init__()
|
|
257
|
+
self.RTPATH = {} # runtime path cache
|
|
258
|
+
self.OPTS = {} # option table (populated by subclass)
|
|
259
|
+
|
|
260
|
+
class DsArch(PgArch):
|
|
261
|
+
def __init__(self):
|
|
262
|
+
super().__init__()
|
|
263
|
+
self.ALLCNT = self.ADDCNT = self.MODCNT = 0
|
|
264
|
+
|
|
265
|
+
def main(self):
|
|
266
|
+
self.read_parameters() # from PgOPT
|
|
267
|
+
self.start_actions() # dispatch
|
|
268
|
+
|
|
269
|
+
if __name__ == "__main__":
|
|
270
|
+
DsArch().main()
|
|
271
|
+
```
|
|
272
|
+
|
|
273
|
+
### 5. Reading a PostgreSQL password from OpenBao or ~/.pgpass
|
|
274
|
+
|
|
275
|
+
```python
|
|
276
|
+
from rda_python_common.pgpassword import PgPassword
|
|
277
|
+
|
|
278
|
+
pw = PgPassword()
|
|
279
|
+
pw.default_scinfo('rdadb', 'dssdb', 'rda-pgdb', 'gdexweb', None, 5432)
|
|
280
|
+
password = pw.get_baopassword() or pw.get_pgpassword()
|
|
281
|
+
```
|
|
282
|
+
|
|
283
|
+
In every case `super().__init__()` cooperates correctly across the
|
|
284
|
+
multi-inheriting joins (`PgFile` and `PgSplit`), so subclasses only need
|
|
285
|
+
to call it once.
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "rda_python_common"
|
|
7
|
-
version = "2.1.
|
|
7
|
+
version = "2.1.8"
|
|
8
8
|
authors = [
|
|
9
9
|
{ name="Zaihua Ji", email="zji@ucar.edu" },
|
|
10
10
|
]
|
|
@@ -21,7 +21,8 @@ dependencies = [
|
|
|
21
21
|
"hvac",
|
|
22
22
|
"psycopg2==2.9.10",
|
|
23
23
|
"rda-python-globus",
|
|
24
|
-
"unidecode"
|
|
24
|
+
"unidecode",
|
|
25
|
+
"hvac"
|
|
25
26
|
]
|
|
26
27
|
|
|
27
28
|
[project.urls]
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"""rda_python_common: shared utility package for RDA Python tools.
|
|
2
|
+
|
|
3
|
+
This package exposes two parallel APIs:
|
|
4
|
+
|
|
5
|
+
1. Legacy module-based API (back-compat). Import the capitalized submodules
|
|
6
|
+
and call their module-level functions, e.g.::
|
|
7
|
+
|
|
8
|
+
from rda_python_common import PgLOG
|
|
9
|
+
PgLOG.pglog("message", PgLOG.LOGWRN)
|
|
10
|
+
|
|
11
|
+
2. Class-based API (preferred for new code). Import the class from the
|
|
12
|
+
lower-case module and either instantiate or subclass it, e.g.::
|
|
13
|
+
|
|
14
|
+
from rda_python_common.pg_log import PgLOG
|
|
15
|
+
log = PgLOG()
|
|
16
|
+
log.pglog("message", log.LOGWRN)
|
|
17
|
+
|
|
18
|
+
The legacy submodules are eagerly imported below so that
|
|
19
|
+
``from rda_python_common import PgLOG`` continues to return the module
|
|
20
|
+
object that existing callers expect.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from . import PgLOG, PgUtil, PgDBI, PgFile, PgLock, PgCMD, PgSIG, PgOPT, PgSplit
|
|
24
|
+
|
|
25
|
+
__version__ = "2.1.8"
|
|
26
|
+
|
|
27
|
+
__all__ = [
|
|
28
|
+
"PgLOG",
|
|
29
|
+
"PgUtil",
|
|
30
|
+
"PgDBI",
|
|
31
|
+
"PgFile",
|
|
32
|
+
"PgLock",
|
|
33
|
+
"PgCMD",
|
|
34
|
+
"PgSIG",
|
|
35
|
+
"PgOPT",
|
|
36
|
+
"PgSplit",
|
|
37
|
+
"__version__",
|
|
38
|
+
]
|
|
@@ -62,12 +62,12 @@ class PgDBI(PgLOG):
|
|
|
62
62
|
super().__init__() # initialize parent class
|
|
63
63
|
|
|
64
64
|
# PostgreSQL specified query timestamp format
|
|
65
|
-
self.fmtyr = lambda fn
|
|
66
|
-
self.fmtqt = lambda fn
|
|
67
|
-
self.fmtmn = lambda fn
|
|
68
|
-
self.fmtdt = lambda fn
|
|
69
|
-
self.fmtym = lambda fn
|
|
70
|
-
self.fmthr = lambda fn
|
|
65
|
+
self.fmtyr = lambda fn: "extract(year from {})::int".format(fn)
|
|
66
|
+
self.fmtqt = lambda fn: "extract(quarter from {})::int".format(fn)
|
|
67
|
+
self.fmtmn = lambda fn: "extract(month from {})::int".format(fn)
|
|
68
|
+
self.fmtdt = lambda fn: "date({})".format(fn)
|
|
69
|
+
self.fmtym = lambda fn: "to_char({}, 'yyyy-mm')".format(fn)
|
|
70
|
+
self.fmthr = lambda fn: "extract(hour from {})::int".format(fn)
|
|
71
71
|
|
|
72
72
|
self.pgdb = None # reference to a connected database object
|
|
73
73
|
self.curtran = 0 # 0 - no transaction, 1 - in transaction
|
|
@@ -577,7 +577,7 @@ class PgDBI(PgLOG):
|
|
|
577
577
|
self.qelog(dberror, 0, "Retry Connecting", ary, pgcnt, self.LOGWRN)
|
|
578
578
|
self.pgconnect(1, pgcnt + 1)
|
|
579
579
|
return (self.FAILURE if not self.pgdb else self.SUCCESS)
|
|
580
|
-
elif
|
|
580
|
+
elif pgcode.startswith('55'): # try to lock again
|
|
581
581
|
self.qelog(dberror, 10, "Retry Locking", ary, pgcnt, self.LOGWRN)
|
|
582
582
|
return self.SUCCESS
|
|
583
583
|
elif pgcode == '25P02': # try to add table
|
|
@@ -2595,7 +2595,7 @@ class PgFile(PgUtil, PgSIG):
|
|
|
2595
2595
|
if opt&17:
|
|
2596
2596
|
dy = int(items[6])
|
|
2597
2597
|
mn = self.get_month(items[5])
|
|
2598
|
-
if
|
|
2598
|
+
if items[7].isdigit():
|
|
2599
2599
|
yr = int(items[7])
|
|
2600
2600
|
mtime = "00:00:00"
|
|
2601
2601
|
else:
|
|
@@ -2972,7 +2972,7 @@ class PgFile(PgUtil, PgSIG):
|
|
|
2972
2972
|
if dir is None:
|
|
2973
2973
|
if isinstance(val, int):
|
|
2974
2974
|
self.DIRLVLS = val
|
|
2975
|
-
elif
|
|
2975
|
+
elif val.isdigit():
|
|
2976
2976
|
self.DIRLVLS = int(val)
|
|
2977
2977
|
elif dir and not re.match(r'^(\.|\./|/)$', dir) and dir not in self.DELDIRS:
|
|
2978
2978
|
self.DELDIRS[dir] = val
|