rda-python-common 2.1.7__tar.gz → 2.1.8__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. rda_python_common-2.1.8/PKG-INFO +305 -0
  2. rda_python_common-2.1.8/README.md +285 -0
  3. {rda_python_common-2.1.7 → rda_python_common-2.1.8}/pyproject.toml +3 -2
  4. rda_python_common-2.1.8/src/rda_python_common/__init__.py +38 -0
  5. {rda_python_common-2.1.7 → rda_python_common-2.1.8}/src/rda_python_common/pg_dbi.py +7 -7
  6. {rda_python_common-2.1.7 → rda_python_common-2.1.8}/src/rda_python_common/pg_file.py +2 -2
  7. {rda_python_common-2.1.7 → rda_python_common-2.1.8}/src/rda_python_common/pg_lock.py +1 -1
  8. {rda_python_common-2.1.7 → rda_python_common-2.1.8}/src/rda_python_common/pg_log.py +6 -6
  9. {rda_python_common-2.1.7 → rda_python_common-2.1.8}/src/rda_python_common/pg_opt.py +1 -1
  10. {rda_python_common-2.1.7 → rda_python_common-2.1.8}/src/rda_python_common/pg_sig.py +1 -1
  11. {rda_python_common-2.1.7 → rda_python_common-2.1.8}/src/rda_python_common/pg_split.py +3 -2
  12. {rda_python_common-2.1.7 → rda_python_common-2.1.8}/src/rda_python_common/pg_util.py +24 -23
  13. rda_python_common-2.1.8/src/rda_python_common/pgpassword.py +127 -0
  14. rda_python_common-2.1.8/src/rda_python_common.egg-info/PKG-INFO +305 -0
  15. {rda_python_common-2.1.7 → rda_python_common-2.1.8}/src/rda_python_common.egg-info/requires.txt +1 -0
  16. rda_python_common-2.1.7/PKG-INFO +0 -21
  17. rda_python_common-2.1.7/README.md +0 -2
  18. rda_python_common-2.1.7/src/rda_python_common/__init__.py +0 -0
  19. rda_python_common-2.1.7/src/rda_python_common/pgpassword.py +0 -86
  20. rda_python_common-2.1.7/src/rda_python_common.egg-info/PKG-INFO +0 -21
  21. {rda_python_common-2.1.7 → rda_python_common-2.1.8}/LICENSE +0 -0
  22. {rda_python_common-2.1.7 → rda_python_common-2.1.8}/setup.cfg +0 -0
  23. {rda_python_common-2.1.7 → rda_python_common-2.1.8}/src/rda_python_common/PgCMD.py +0 -0
  24. {rda_python_common-2.1.7 → rda_python_common-2.1.8}/src/rda_python_common/PgDBI.py +0 -0
  25. {rda_python_common-2.1.7 → rda_python_common-2.1.8}/src/rda_python_common/PgFile.py +0 -0
  26. {rda_python_common-2.1.7 → rda_python_common-2.1.8}/src/rda_python_common/PgLOG.py +0 -0
  27. {rda_python_common-2.1.7 → rda_python_common-2.1.8}/src/rda_python_common/PgLock.py +0 -0
  28. {rda_python_common-2.1.7 → rda_python_common-2.1.8}/src/rda_python_common/PgOPT.py +0 -0
  29. {rda_python_common-2.1.7 → rda_python_common-2.1.8}/src/rda_python_common/PgSIG.py +0 -0
  30. {rda_python_common-2.1.7 → rda_python_common-2.1.8}/src/rda_python_common/PgSplit.py +0 -0
  31. {rda_python_common-2.1.7 → rda_python_common-2.1.8}/src/rda_python_common/PgUtil.py +0 -0
  32. {rda_python_common-2.1.7 → rda_python_common-2.1.8}/src/rda_python_common/pg_cmd.py +0 -0
  33. {rda_python_common-2.1.7 → rda_python_common-2.1.8}/src/rda_python_common/pg_password.py +0 -0
  34. {rda_python_common-2.1.7 → rda_python_common-2.1.8}/src/rda_python_common.egg-info/SOURCES.txt +0 -0
  35. {rda_python_common-2.1.7 → rda_python_common-2.1.8}/src/rda_python_common.egg-info/dependency_links.txt +0 -0
  36. {rda_python_common-2.1.7 → rda_python_common-2.1.8}/src/rda_python_common.egg-info/entry_points.txt +0 -0
  37. {rda_python_common-2.1.7 → rda_python_common-2.1.8}/src/rda_python_common.egg-info/top_level.txt +0 -0
  38. {rda_python_common-2.1.7 → rda_python_common-2.1.8}/test/test_common.py +0 -0
@@ -0,0 +1,305 @@
1
+ Metadata-Version: 2.4
2
+ Name: rda_python_common
3
+ Version: 2.1.8
4
+ Summary: RDA Python common library codes shared by other RDA python packages
5
+ Author-email: Zaihua Ji <zji@ucar.edu>
6
+ Project-URL: Homepage, https://github.com/NCAR/rda-python-common
7
+ Classifier: Programming Language :: Python :: 3
8
+ Classifier: License :: OSI Approved :: MIT License
9
+ Classifier: Operating System :: OS Independent
10
+ Classifier: Development Status :: 5 - Production/Stable
11
+ Requires-Python: >=3.7
12
+ Description-Content-Type: text/markdown
13
+ License-File: LICENSE
14
+ Requires-Dist: hvac
15
+ Requires-Dist: psycopg2==2.9.10
16
+ Requires-Dist: rda-python-globus
17
+ Requires-Dist: unidecode
18
+ Requires-Dist: hvac
19
+ Dynamic: license-file
20
+
21
+ # rda-python-common
22
+
23
+ Python common library codes to be shared by other RDA python utility programs.
24
+
25
+ ## Installing and using in another RDA python repo
26
+
27
+ `rda-python-common` is the foundation that every other `rda-python-*` repo
28
+ builds on. To consume it from a new or existing repo, follow these steps.
29
+
30
+ ### 1. Install the package
31
+
32
+ For local development, clone this repo alongside your project and install it
33
+ in editable mode so that changes are picked up without re-installing:
34
+
35
+ ```bash
36
+ git clone https://github.com/NCAR/rda-python-common.git
37
+ cd rda-python-common
38
+ pip install -e .
39
+ ```
40
+
41
+ For a regular (non-editable) install from a checkout:
42
+
43
+ ```bash
44
+ pip install /path/to/rda-python-common
45
+ ```
46
+
47
+ For a production install on a system that uses the published distribution:
48
+
49
+ ```bash
50
+ pip install rda_python_common
51
+ ```
52
+
53
+ The package brings in its own transitive dependencies (`psycopg2-binary`,
54
+ `rda-python-globus`, `unidecode`, `hvac`).
55
+
56
+ ### 2. Declare it as a dependency in your project
57
+
58
+ Add `rda_python_common` to the `dependencies` list of your project's
59
+ `pyproject.toml` so that downstream installs pull it in automatically:
60
+
61
+ ```toml
62
+ [project]
63
+ name = "rda_python_yourtool"
64
+ version = "0.1.0"
65
+ dependencies = [
66
+ "rda_python_common",
67
+ # ... other deps
68
+ ]
69
+ ```
70
+
71
+ This is the same pattern used by `rda-python-dsarch`, `rda-python-dsupdt`,
72
+ `rda-python-dsrqst`, `rda-python-dscheck`, `rda-python-metrics`, and
73
+ `rda-python-miscs`.
74
+
75
+ ### 3. Import the modules you need
76
+
77
+ Two import styles are supported (see [Usage examples](#usage-examples) below):
78
+
79
+ ```python
80
+ # Preferred for new code -- import the class from the lower-case module
81
+ from rda_python_common.pg_log import PgLOG
82
+ from rda_python_common.pg_dbi import PgDBI
83
+
84
+ # Legacy module-style imports remain supported for back-compatibility
85
+ from rda_python_common import PgLOG, PgDBI
86
+ PgLOG.pglog("hello", PgLOG.LOGWRN)
87
+ ```
88
+
89
+ ### 4. Verify the install
90
+
91
+ ```bash
92
+ python -c "import rda_python_common; print(rda_python_common.__version__)"
93
+ ```
94
+
95
+ You should see the installed version (currently `2.1.8`). If the import
96
+ fails, double-check that the active Python environment is the one where you
97
+ ran `pip install`.
98
+
99
+ ## Modules
100
+
101
+ All shared functionality lives under `src/rda_python_common/` and is organised as
102
+ a single-inheritance class hierarchy. Each module defines exactly one class;
103
+ later classes extend earlier ones, so an application that instantiates the
104
+ top-of-chain class (typically `PgOPT` or `PgCMD`) gets every helper through one
105
+ object.
106
+
107
+ Inheritance tree (top-down; multi-inheritance shown as two arrows
108
+ converging on the same child):
109
+
110
+ ```
111
+ PgLOG
112
+ ┌────┴────┐
113
+ ▼ ▼
114
+ PgUtil PgDBI
115
+ │ │ │ │ │
116
+ │ └────┐ ┌─┘ │ └─► PgPassword
117
+ │ ▼ ▼ │
118
+ │ PgSplit │ (multi-inherits
119
+ │ │ PgUtil + PgDBI)
120
+ │ ▼
121
+ │ PgSIG
122
+ │ │
123
+ │ ┌──────────┘
124
+ ▼ ▼
125
+ PgFile (multi-inherits
126
+ │ PgUtil + PgSIG)
127
+ ├─► PgOPT
128
+
129
+ └─► PgLock
130
+
131
+ └─► PgCMD
132
+ ```
133
+
134
+ The tree is single inheritance everywhere except at two join points:
135
+
136
+ - **`PgFile(PgUtil, PgSIG)`** — combines date/record utilities (`PgUtil`
137
+ via `PgLOG`) with daemon/signal/DB control (`PgSIG` → `PgDBI` → `PgLOG`),
138
+ so its descendants `PgOPT`, `PgLock`, and `PgCMD` inherit logging, DB,
139
+ util, signal, and file facilities through one MRO.
140
+ - **`PgSplit(PgUtil, PgDBI)`** — combines record-manipulation helpers
141
+ (`PgUtil`) with the `pgadd`/`pgget`/`pgmget`/`pgupdt`/`pgdel` DB
142
+ operations (`PgDBI`) it needs to keep the shared `wfile` table and the
143
+ per-dataset `wfile_<dsid>` partitions in sync.
144
+
145
+ - **`pg_log.py`** — `PgLOG`. Root of the hierarchy. Provides the central
146
+ logging facility (bit-mask `logact` flags such as `MSGLOG`, `WARNLG`,
147
+ `ERRLOG`, `EXITLG`), e-mail dispatch, system-command execution, process
148
+ metadata lookup, and the global `PGLOG` settings dictionary used by every
149
+ other module.
150
+
151
+ - **`pg_util.py`** — `PgUtil(PgLOG)`. Miscellaneous date/time, dataset-ID,
152
+ and column-oriented record-manipulation helpers. Holds the `DATEFMTS`
153
+ regex table, `MONTHS`/`MNS`/`WDAYS`/`WDS` lookup lists, and the `MDAYS`
154
+ days-per-month array used for date arithmetic, formatting, parsing, and
155
+ record sort/search/classification across all RDA tools.
156
+
157
+ - **`pg_file.py`** — `PgFile(PgUtil, PgSIG)`. Unified file-operation layer
158
+ spanning local file systems, remote hosts (rsync/ssh/scp), AWS S3 / object
159
+ store, and Globus endpoints. Used by `rdacp`, `dsarch`, `dsupdt`, and
160
+ related tools whenever data is moved, listed, or stat-ed.
161
+
162
+ - **`pg_lock.py`** — `PgLock(PgFile)`. RDADB record-locking primitives for
163
+ the `dscheck`, `dsrqst`, `dlupdt`, `dcupdt`, `ptrqst`, and `dataset`
164
+ tables. Acquires, refreshes, and releases per-record locks so that
165
+ long-running batch jobs coordinate cleanly.
166
+
167
+ - **`pg_dbi.py`** — `PgDBI(PgLOG)`. PostgreSQL database interface built on
168
+ `psycopg2`. Wraps connection management, batch `INSERT`/`SELECT`/
169
+ `UPDATE`/`DELETE`, transaction control, and credential lookup from
170
+ `.pgpass` or OpenBao. All RDA tools talk to the `rdadb` database through
171
+ this class.
172
+
173
+ - **`pg_sig.py`** — `PgSIG(PgDBI)`. Daemon process control, POSIX signal
174
+ handling, child/background-process management, and PBS/Torque batch-job
175
+ status queries. Provides the `PGSIG` runtime dictionary plus `VUSERS`,
176
+ `CPIDS`, `CBIDS`, and `SDUMP` tables that drive RDA daemon programs.
177
+
178
+ - **`pg_cmd.py`** — `PgCMD(PgLock)`. Manages `dscheck` batch and delayed-
179
+ mode command tracking. Records, updates, and reaps the per-command rows
180
+ that let RDA utilities resume or be monitored across PBS batch jobs.
181
+
182
+ - **`pg_split.py`** — `PgSplit(PgUtil, PgDBI)`. Synchronises `wfile` records
183
+ between the shared `wfile` table and the per-dataset `wfile_<dsid>`
184
+ partition tables. Provides compare/add/update/delete helpers used when
185
+ archiving or reconciling dataset file inventories.
186
+
187
+ - **`pg_opt.py`** — `PgOPT(PgFile)`. Command-line option parsing and
188
+ application configuration framework for RDA tools (`dsarch`, `dsupdt`,
189
+ `dsrqst`, ...). Holds the master `OPTS` definition table, parsed
190
+ `params`, command-line vs. input-file option tracking (`CMDOPTS`/
191
+ `INOPTS`), output formatting, dataset/help/media/storage/backup type
192
+ maps, and the global `PGOPT` settings.
193
+
194
+ - **`pgpassword.py`** — `PgPassword(PgDBI)`. Standalone CLI entry point
195
+ (`pgpassword`) that resolves a PostgreSQL login password from OpenBao
196
+ (`get_baopassword`) or `~/.pgpass` (`get_pgpassword()`) given database/schema/
197
+ host/port/user selectors via `-d`, `-c`, `-h`, `-p`, `-u`, `-l`, `-k`.
198
+ Prints the resolved password to stdout so shell scripts can capture it.
199
+
200
+ ## Usage examples
201
+
202
+ Each class lives in its own submodule. Import the class you need, then
203
+ either instantiate it directly or subclass it to add application-specific
204
+ state and methods.
205
+
206
+ ### 1. Direct instantiation — use the helpers as-is
207
+
208
+ ```python
209
+ # Logging only
210
+ from rda_python_common.pg_log import PgLOG
211
+
212
+ log = PgLOG()
213
+ log.pglog("dsarch started", log.LOGWRN)
214
+
215
+ # Database access (PgDBI inherits PgLOG, so you get logging too)
216
+ from rda_python_common.pg_dbi import PgDBI
217
+
218
+ db = PgDBI()
219
+ rec = db.pgget('dataset', 'dsid, title', "dsid = 'd633000'")
220
+ print(rec)
221
+ ```
222
+
223
+ ### 2. Subclassing a single common class
224
+
225
+ ```python
226
+ # A small utility that needs date/record helpers plus logging.
227
+ from rda_python_common.pg_util import PgUtil
228
+
229
+ class DateReport(PgUtil):
230
+ def __init__(self):
231
+ super().__init__() # initialise PgUtil (and PgLOG)
232
+ self.today = self.curtime() # method inherited from PgUtil
233
+
234
+ def run(self):
235
+ self.pglog(f"report date: {self.today}", self.LOGWRN)
236
+
237
+ DateReport().run()
238
+ ```
239
+
240
+ ### 3. Subclassing one of the multi-inheriting joins
241
+
242
+ ```python
243
+ # A worker that needs file I/O (PgFile) and dscheck command tracking (PgCMD).
244
+ # PgCMD already extends PgFile via PgLock, so a single base is enough.
245
+ from rda_python_common.pg_cmd import PgCMD
246
+
247
+ class Worker(PgCMD):
248
+ def __init__(self):
249
+ super().__init__()
250
+ self.jobs = []
251
+
252
+ def archive_one(self, src, dst):
253
+ # PgFile method, available through the inheritance chain
254
+ self.local_copy_local(src, dst)
255
+ # PgDBI method, available through PgCMD -> PgLock -> PgFile -> PgSIG -> PgDBI
256
+ self.pgupdt('wfile', {'status': 'A'}, f"wfile = '{dst}'")
257
+
258
+ Worker().archive_one('/in/file', '/out/file')
259
+ ```
260
+
261
+ ### 4. Combining multiple common classes (application action class)
262
+
263
+ This mirrors how RDA tools such as `dsarch` are structured. The leaf class
264
+ multi-inherits several common classes so a single object exposes options,
265
+ command tracking, and wfile splitting.
266
+
267
+ ```python
268
+ # Excerpt of the pattern used by rda_python_dsarch/dsarch.py
269
+ from rda_python_common.pg_opt import PgOPT
270
+ from rda_python_common.pg_cmd import PgCMD
271
+ from rda_python_common.pg_split import PgSplit
272
+
273
+ class PgArch(PgOPT, PgCMD, PgSplit):
274
+ """Shared state + helpers for a CLI archiving tool."""
275
+ def __init__(self):
276
+ super().__init__()
277
+ self.RTPATH = {} # runtime path cache
278
+ self.OPTS = {} # option table (populated by subclass)
279
+
280
+ class DsArch(PgArch):
281
+ def __init__(self):
282
+ super().__init__()
283
+ self.ALLCNT = self.ADDCNT = self.MODCNT = 0
284
+
285
+ def main(self):
286
+ self.read_parameters() # from PgOPT
287
+ self.start_actions() # dispatch
288
+
289
+ if __name__ == "__main__":
290
+ DsArch().main()
291
+ ```
292
+
293
+ ### 5. Reading a PostgreSQL password from OpenBao or ~/.pgpass
294
+
295
+ ```python
296
+ from rda_python_common.pgpassword import PgPassword
297
+
298
+ pw = PgPassword()
299
+ pw.default_scinfo('rdadb', 'dssdb', 'rda-pgdb', 'gdexweb', None, 5432)
300
+ password = pw.get_baopassword() or pw.get_pgpassword()
301
+ ```
302
+
303
+ In every case `super().__init__()` cooperates correctly across the
304
+ multi-inheriting joins (`PgFile` and `PgSplit`), so subclasses only need
305
+ to call it once.
@@ -0,0 +1,285 @@
1
+ # rda-python-common
2
+
3
+ Python common library codes to be shared by other RDA python utility programs.
4
+
5
+ ## Installing and using in another RDA python repo
6
+
7
+ `rda-python-common` is the foundation that every other `rda-python-*` repo
8
+ builds on. To consume it from a new or existing repo, follow these steps.
9
+
10
+ ### 1. Install the package
11
+
12
+ For local development, clone this repo alongside your project and install it
13
+ in editable mode so that changes are picked up without re-installing:
14
+
15
+ ```bash
16
+ git clone https://github.com/NCAR/rda-python-common.git
17
+ cd rda-python-common
18
+ pip install -e .
19
+ ```
20
+
21
+ For a regular (non-editable) install from a checkout:
22
+
23
+ ```bash
24
+ pip install /path/to/rda-python-common
25
+ ```
26
+
27
+ For a production install on a system that uses the published distribution:
28
+
29
+ ```bash
30
+ pip install rda_python_common
31
+ ```
32
+
33
+ The package brings in its own transitive dependencies (`psycopg2-binary`,
34
+ `rda-python-globus`, `unidecode`, `hvac`).
35
+
36
+ ### 2. Declare it as a dependency in your project
37
+
38
+ Add `rda_python_common` to the `dependencies` list of your project's
39
+ `pyproject.toml` so that downstream installs pull it in automatically:
40
+
41
+ ```toml
42
+ [project]
43
+ name = "rda_python_yourtool"
44
+ version = "0.1.0"
45
+ dependencies = [
46
+ "rda_python_common",
47
+ # ... other deps
48
+ ]
49
+ ```
50
+
51
+ This is the same pattern used by `rda-python-dsarch`, `rda-python-dsupdt`,
52
+ `rda-python-dsrqst`, `rda-python-dscheck`, `rda-python-metrics`, and
53
+ `rda-python-miscs`.
54
+
55
+ ### 3. Import the modules you need
56
+
57
+ Two import styles are supported (see [Usage examples](#usage-examples) below):
58
+
59
+ ```python
60
+ # Preferred for new code -- import the class from the lower-case module
61
+ from rda_python_common.pg_log import PgLOG
62
+ from rda_python_common.pg_dbi import PgDBI
63
+
64
+ # Legacy module-style imports remain supported for back-compatibility
65
+ from rda_python_common import PgLOG, PgDBI
66
+ PgLOG.pglog("hello", PgLOG.LOGWRN)
67
+ ```
68
+
69
+ ### 4. Verify the install
70
+
71
+ ```bash
72
+ python -c "import rda_python_common; print(rda_python_common.__version__)"
73
+ ```
74
+
75
+ You should see the installed version (currently `2.1.8`). If the import
76
+ fails, double-check that the active Python environment is the one where you
77
+ ran `pip install`.
78
+
79
+ ## Modules
80
+
81
+ All shared functionality lives under `src/rda_python_common/` and is organised as
82
+ a single-inheritance class hierarchy. Each module defines exactly one class;
83
+ later classes extend earlier ones, so an application that instantiates the
84
+ top-of-chain class (typically `PgOPT` or `PgCMD`) gets every helper through one
85
+ object.
86
+
87
+ Inheritance tree (top-down; multi-inheritance shown as two arrows
88
+ converging on the same child):
89
+
90
+ ```
91
+ PgLOG
92
+ ┌────┴────┐
93
+ ▼ ▼
94
+ PgUtil PgDBI
95
+ │ │ │ │ │
96
+ │ └────┐ ┌─┘ │ └─► PgPassword
97
+ │ ▼ ▼ │
98
+ │ PgSplit │ (multi-inherits
99
+ │ │ PgUtil + PgDBI)
100
+ │ ▼
101
+ │ PgSIG
102
+ │ │
103
+ │ ┌──────────┘
104
+ ▼ ▼
105
+ PgFile (multi-inherits
106
+ │ PgUtil + PgSIG)
107
+ ├─► PgOPT
108
+
109
+ └─► PgLock
110
+
111
+ └─► PgCMD
112
+ ```
113
+
114
+ The tree is single inheritance everywhere except at two join points:
115
+
116
+ - **`PgFile(PgUtil, PgSIG)`** — combines date/record utilities (`PgUtil`
117
+ via `PgLOG`) with daemon/signal/DB control (`PgSIG` → `PgDBI` → `PgLOG`),
118
+ so its descendants `PgOPT`, `PgLock`, and `PgCMD` inherit logging, DB,
119
+ util, signal, and file facilities through one MRO.
120
+ - **`PgSplit(PgUtil, PgDBI)`** — combines record-manipulation helpers
121
+ (`PgUtil`) with the `pgadd`/`pgget`/`pgmget`/`pgupdt`/`pgdel` DB
122
+ operations (`PgDBI`) it needs to keep the shared `wfile` table and the
123
+ per-dataset `wfile_<dsid>` partitions in sync.
124
+
125
+ - **`pg_log.py`** — `PgLOG`. Root of the hierarchy. Provides the central
126
+ logging facility (bit-mask `logact` flags such as `MSGLOG`, `WARNLG`,
127
+ `ERRLOG`, `EXITLG`), e-mail dispatch, system-command execution, process
128
+ metadata lookup, and the global `PGLOG` settings dictionary used by every
129
+ other module.
130
+
131
+ - **`pg_util.py`** — `PgUtil(PgLOG)`. Miscellaneous date/time, dataset-ID,
132
+ and column-oriented record-manipulation helpers. Holds the `DATEFMTS`
133
+ regex table, `MONTHS`/`MNS`/`WDAYS`/`WDS` lookup lists, and the `MDAYS`
134
+ days-per-month array used for date arithmetic, formatting, parsing, and
135
+ record sort/search/classification across all RDA tools.
136
+
137
+ - **`pg_file.py`** — `PgFile(PgUtil, PgSIG)`. Unified file-operation layer
138
+ spanning local file systems, remote hosts (rsync/ssh/scp), AWS S3 / object
139
+ store, and Globus endpoints. Used by `rdacp`, `dsarch`, `dsupdt`, and
140
+ related tools whenever data is moved, listed, or stat-ed.
141
+
142
+ - **`pg_lock.py`** — `PgLock(PgFile)`. RDADB record-locking primitives for
143
+ the `dscheck`, `dsrqst`, `dlupdt`, `dcupdt`, `ptrqst`, and `dataset`
144
+ tables. Acquires, refreshes, and releases per-record locks so that
145
+ long-running batch jobs coordinate cleanly.
146
+
147
+ - **`pg_dbi.py`** — `PgDBI(PgLOG)`. PostgreSQL database interface built on
148
+ `psycopg2`. Wraps connection management, batch `INSERT`/`SELECT`/
149
+ `UPDATE`/`DELETE`, transaction control, and credential lookup from
150
+ `.pgpass` or OpenBao. All RDA tools talk to the `rdadb` database through
151
+ this class.
152
+
153
+ - **`pg_sig.py`** — `PgSIG(PgDBI)`. Daemon process control, POSIX signal
154
+ handling, child/background-process management, and PBS/Torque batch-job
155
+ status queries. Provides the `PGSIG` runtime dictionary plus `VUSERS`,
156
+ `CPIDS`, `CBIDS`, and `SDUMP` tables that drive RDA daemon programs.
157
+
158
+ - **`pg_cmd.py`** — `PgCMD(PgLock)`. Manages `dscheck` batch and delayed-
159
+ mode command tracking. Records, updates, and reaps the per-command rows
160
+ that let RDA utilities resume or be monitored across PBS batch jobs.
161
+
162
+ - **`pg_split.py`** — `PgSplit(PgUtil, PgDBI)`. Synchronises `wfile` records
163
+ between the shared `wfile` table and the per-dataset `wfile_<dsid>`
164
+ partition tables. Provides compare/add/update/delete helpers used when
165
+ archiving or reconciling dataset file inventories.
166
+
167
+ - **`pg_opt.py`** — `PgOPT(PgFile)`. Command-line option parsing and
168
+ application configuration framework for RDA tools (`dsarch`, `dsupdt`,
169
+ `dsrqst`, ...). Holds the master `OPTS` definition table, parsed
170
+ `params`, command-line vs. input-file option tracking (`CMDOPTS`/
171
+ `INOPTS`), output formatting, dataset/help/media/storage/backup type
172
+ maps, and the global `PGOPT` settings.
173
+
174
+ - **`pgpassword.py`** — `PgPassword(PgDBI)`. Standalone CLI entry point
175
+ (`pgpassword`) that resolves a PostgreSQL login password from OpenBao
176
+ (`get_baopassword`) or `~/.pgpass` (`get_pgpassword()`) given database/schema/
177
+ host/port/user selectors via `-d`, `-c`, `-h`, `-p`, `-u`, `-l`, `-k`.
178
+ Prints the resolved password to stdout so shell scripts can capture it.
179
+
180
+ ## Usage examples
181
+
182
+ Each class lives in its own submodule. Import the class you need, then
183
+ either instantiate it directly or subclass it to add application-specific
184
+ state and methods.
185
+
186
+ ### 1. Direct instantiation — use the helpers as-is
187
+
188
+ ```python
189
+ # Logging only
190
+ from rda_python_common.pg_log import PgLOG
191
+
192
+ log = PgLOG()
193
+ log.pglog("dsarch started", log.LOGWRN)
194
+
195
+ # Database access (PgDBI inherits PgLOG, so you get logging too)
196
+ from rda_python_common.pg_dbi import PgDBI
197
+
198
+ db = PgDBI()
199
+ rec = db.pgget('dataset', 'dsid, title', "dsid = 'd633000'")
200
+ print(rec)
201
+ ```
202
+
203
+ ### 2. Subclassing a single common class
204
+
205
+ ```python
206
+ # A small utility that needs date/record helpers plus logging.
207
+ from rda_python_common.pg_util import PgUtil
208
+
209
+ class DateReport(PgUtil):
210
+ def __init__(self):
211
+ super().__init__() # initialise PgUtil (and PgLOG)
212
+ self.today = self.curtime() # method inherited from PgUtil
213
+
214
+ def run(self):
215
+ self.pglog(f"report date: {self.today}", self.LOGWRN)
216
+
217
+ DateReport().run()
218
+ ```
219
+
220
+ ### 3. Subclassing one of the multi-inheriting joins
221
+
222
+ ```python
223
+ # A worker that needs file I/O (PgFile) and dscheck command tracking (PgCMD).
224
+ # PgCMD already extends PgFile via PgLock, so a single base is enough.
225
+ from rda_python_common.pg_cmd import PgCMD
226
+
227
+ class Worker(PgCMD):
228
+ def __init__(self):
229
+ super().__init__()
230
+ self.jobs = []
231
+
232
+ def archive_one(self, src, dst):
233
+ # PgFile method, available through the inheritance chain
234
+ self.local_copy_local(src, dst)
235
+ # PgDBI method, available through PgCMD -> PgLock -> PgFile -> PgSIG -> PgDBI
236
+ self.pgupdt('wfile', {'status': 'A'}, f"wfile = '{dst}'")
237
+
238
+ Worker().archive_one('/in/file', '/out/file')
239
+ ```
240
+
241
+ ### 4. Combining multiple common classes (application action class)
242
+
243
+ This mirrors how RDA tools such as `dsarch` are structured. The leaf class
244
+ multi-inherits several common classes so a single object exposes options,
245
+ command tracking, and wfile splitting.
246
+
247
+ ```python
248
+ # Excerpt of the pattern used by rda_python_dsarch/dsarch.py
249
+ from rda_python_common.pg_opt import PgOPT
250
+ from rda_python_common.pg_cmd import PgCMD
251
+ from rda_python_common.pg_split import PgSplit
252
+
253
+ class PgArch(PgOPT, PgCMD, PgSplit):
254
+ """Shared state + helpers for a CLI archiving tool."""
255
+ def __init__(self):
256
+ super().__init__()
257
+ self.RTPATH = {} # runtime path cache
258
+ self.OPTS = {} # option table (populated by subclass)
259
+
260
+ class DsArch(PgArch):
261
+ def __init__(self):
262
+ super().__init__()
263
+ self.ALLCNT = self.ADDCNT = self.MODCNT = 0
264
+
265
+ def main(self):
266
+ self.read_parameters() # from PgOPT
267
+ self.start_actions() # dispatch
268
+
269
+ if __name__ == "__main__":
270
+ DsArch().main()
271
+ ```
272
+
273
+ ### 5. Reading a PostgreSQL password from OpenBao or ~/.pgpass
274
+
275
+ ```python
276
+ from rda_python_common.pgpassword import PgPassword
277
+
278
+ pw = PgPassword()
279
+ pw.default_scinfo('rdadb', 'dssdb', 'rda-pgdb', 'gdexweb', None, 5432)
280
+ password = pw.get_baopassword() or pw.get_pgpassword()
281
+ ```
282
+
283
+ In every case `super().__init__()` cooperates correctly across the
284
+ multi-inheriting joins (`PgFile` and `PgSplit`), so subclasses only need
285
+ to call it once.
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "rda_python_common"
7
- version = "2.1.7"
7
+ version = "2.1.8"
8
8
  authors = [
9
9
  { name="Zaihua Ji", email="zji@ucar.edu" },
10
10
  ]
@@ -21,7 +21,8 @@ dependencies = [
21
21
  "hvac",
22
22
  "psycopg2==2.9.10",
23
23
  "rda-python-globus",
24
- "unidecode"
24
+ "unidecode",
25
+ "hvac"
25
26
  ]
26
27
 
27
28
  [project.urls]
@@ -0,0 +1,38 @@
1
+ """rda_python_common: shared utility package for RDA Python tools.
2
+
3
+ This package exposes two parallel APIs:
4
+
5
+ 1. Legacy module-based API (back-compat). Import the capitalized submodules
6
+ and call their module-level functions, e.g.::
7
+
8
+ from rda_python_common import PgLOG
9
+ PgLOG.pglog("message", PgLOG.LOGWRN)
10
+
11
+ 2. Class-based API (preferred for new code). Import the class from the
12
+ lower-case module and either instantiate or subclass it, e.g.::
13
+
14
+ from rda_python_common.pg_log import PgLOG
15
+ log = PgLOG()
16
+ log.pglog("message", log.LOGWRN)
17
+
18
+ The legacy submodules are eagerly imported below so that
19
+ ``from rda_python_common import PgLOG`` continues to return the module
20
+ object that existing callers expect.
21
+ """
22
+
23
+ from . import PgLOG, PgUtil, PgDBI, PgFile, PgLock, PgCMD, PgSIG, PgOPT, PgSplit
24
+
25
+ __version__ = "2.1.8"
26
+
27
+ __all__ = [
28
+ "PgLOG",
29
+ "PgUtil",
30
+ "PgDBI",
31
+ "PgFile",
32
+ "PgLock",
33
+ "PgCMD",
34
+ "PgSIG",
35
+ "PgOPT",
36
+ "PgSplit",
37
+ "__version__",
38
+ ]
@@ -62,12 +62,12 @@ class PgDBI(PgLOG):
62
62
  super().__init__() # initialize parent class
63
63
 
64
64
  # PostgreSQL specified query timestamp format
65
- self.fmtyr = lambda fn=self: "extract(year from {})::int".format(fn)
66
- self.fmtqt = lambda fn=self: "extract(quarter from {})::int".format(fn)
67
- self.fmtmn = lambda fn=self: "extract(month from {})::int".format(fn)
68
- self.fmtdt = lambda fn=self: "date({})".format(fn)
69
- self.fmtym = lambda fn=self: "to_char({}, 'yyyy-mm')".format(fn)
70
- self.fmthr = lambda fn=self: "extract(hour from {})::int".format(fn)
65
+ self.fmtyr = lambda fn: "extract(year from {})::int".format(fn)
66
+ self.fmtqt = lambda fn: "extract(quarter from {})::int".format(fn)
67
+ self.fmtmn = lambda fn: "extract(month from {})::int".format(fn)
68
+ self.fmtdt = lambda fn: "date({})".format(fn)
69
+ self.fmtym = lambda fn: "to_char({}, 'yyyy-mm')".format(fn)
70
+ self.fmthr = lambda fn: "extract(hour from {})::int".format(fn)
71
71
 
72
72
  self.pgdb = None # reference to a connected database object
73
73
  self.curtran = 0 # 0 - no transaction, 1 - in transaction
@@ -577,7 +577,7 @@ class PgDBI(PgLOG):
577
577
  self.qelog(dberror, 0, "Retry Connecting", ary, pgcnt, self.LOGWRN)
578
578
  self.pgconnect(1, pgcnt + 1)
579
579
  return (self.FAILURE if not self.pgdb else self.SUCCESS)
580
- elif re.match(r'^55', pgcode): # try to lock again
580
+ elif pgcode.startswith('55'): # try to lock again
581
581
  self.qelog(dberror, 10, "Retry Locking", ary, pgcnt, self.LOGWRN)
582
582
  return self.SUCCESS
583
583
  elif pgcode == '25P02': # try to add table
@@ -2595,7 +2595,7 @@ class PgFile(PgUtil, PgSIG):
2595
2595
  if opt&17:
2596
2596
  dy = int(items[6])
2597
2597
  mn = self.get_month(items[5])
2598
- if re.match(r'^\d+$', items[7]):
2598
+ if items[7].isdigit():
2599
2599
  yr = int(items[7])
2600
2600
  mtime = "00:00:00"
2601
2601
  else:
@@ -2972,7 +2972,7 @@ class PgFile(PgUtil, PgSIG):
2972
2972
  if dir is None:
2973
2973
  if isinstance(val, int):
2974
2974
  self.DIRLVLS = val
2975
- elif re.match(r'^\d+$', val):
2975
+ elif val.isdigit():
2976
2976
  self.DIRLVLS = int(val)
2977
2977
  elif dir and not re.match(r'^(\.|\./|/)$', dir) and dir not in self.DELDIRS:
2978
2978
  self.DELDIRS[dir] = val