rda-python-dbms 2.0.9__py3-none-any.whl → 3.0.1__py3-none-any.whl

rda_python_dbms/pgchksum.py

@@ -22,6 +22,7 @@ class PgChksum(PgCMD, PgSplit):
       super().__init__()  # initialize parent class
       self.PGSUM = {
          'd' : [],   # delay mode option, value array
+         'e' : 0,    # 1 for emailing dataset owners the list of their missing files
          's' : [],   # specialist login name array for email notice, default to none
          'f' : [],   # file name array, default to none
          'i' : [],   # dataset id array, default for all
@@ -46,6 +47,7 @@ class PgChksum(PgCMD, PgSplit):
          'begin' : 0,
          'end' : 0
       }
+      self.MISSING = {}   # dsid => list of missing file names with their location
       self.pgname = "pgchksum"
       self.pgcmd = None
 
@@ -61,7 +63,7 @@ class PgChksum(PgCMD, PgSplit):
                self.PGLOG['BCKGRND'] = 1   # processing in backgroup mode
                option = None
             elif option in self.PGSUM:
-               if option in ['m', 'n', 'x']:
+               if option in ['e', 'm', 'n', 'x']:
                   self.PGSUM[option] = 1
                   option = None
             else:
@@ -80,12 +82,13 @@ class PgChksum(PgCMD, PgSplit):
          else:
             self.pglog(arg + ": value passed without leading Option", self.LGEREX)
       if not self.PGSUM['t']:
-         print("Usage: pgchksum -t (W|S|Q) [-b] [-m] [-n] [-x] [-c FileCount] \\")
+         print("Usage: pgchksum -t (W|S|Q) [-b] [-e] [-m] [-n] [-x] [-c FileCount] \\")
          print("                [-d [Hostnames] [RerunCount]] [-f FileNames] [-i DatasetIDs] \\")
          print("                [-k SkipCount] [-o r|s] [-s Specialists]")
          print("  -b - background process and no screen output")
          print("  -c - File count to re-evaluate checksum; for all gathered files if not specified")
          print("  -d - delayed mode background/batch processes on RDA machines/DAV PBS nodes")
+         print("  -e - email the primary owner of each dataset a reminder listing its missing files")
          print("  -f - Re-evaluate the specify File names only")
          print("  -i - Dataset IDs the files belong; for all files if not specified")
          print("  -k - skip the given file count to re-evaluate checksum")
@@ -100,6 +103,8 @@ class PgChksum(PgCMD, PgSplit):
          self.pgexit(0)
       elif self.PGSUM['t'] not in 'QSW':
          self.pglog(self.PGSUM['t'] + ": Unknown File Type, must be one of 'QSW'", self.LGEREX)
+      if self.PGSUM['e'] and self.PGSUM['n']:
+         self.pglog("Option -e (email owners) cannot work with -n (no email)", self.LGEREX)
        # set different log file
       self.PGLOG['LOGFILE'] = self.pgname + '.log'
       self.set_suid(self.PGLOG['EUID'])
@@ -151,6 +156,7 @@ class PgChksum(PgCMD, PgSplit):
             self.evaluate_backfile_checksum(fcnt, pgrecs, cnts)
          self.dump_progress(fcnt, fcnt, cnts, self.PVALS['emlsum'])
          self.PVALS['end'] = int(time.time())
+         if self.PGSUM['e'] and self.MISSING: self.send_owner_emails()
          if not self.PGSUM['n']: self.send_check_email(cnts['cnt'], fcnt)
       if self.PGLOG['DSCHECK']:
          if self.PGLOG['ERRMSG']:
@@ -173,6 +179,43 @@ class PgChksum(PgCMD, PgSplit):
       else:
          self.pglog(subject, self.LOGWRN|self.SNDEML)
    
+   # email the primary owner(s) of each dataset the list of its missing files
+   def send_owner_emails(self):
+      for dsid in sorted(self.MISSING):
+         files = self.MISSING[dsid]
+         fcnt = len(files)
+         s = 's' if fcnt > 1 else ''
+         tjoin = "dsowner LEFT JOIN dssgrp ON specialist = logname"
+         pgrecs = self.pgmget(tjoin, "specialist, fstname",
+                              "dsid = '{}' AND priority = 1 ORDER BY specialist".format(dsid), self.PVALS['emlerr'])
+         if not pgrecs or not pgrecs['specialist']:
+            self.pglog("{}: No primary owner found in dsowner to email {} missing {} file{}".format(dsid, fcnt, self.PVALS['TYPE'], s), self.PVALS['emllog'])
+            continue
+         owners = []
+         names = []
+         for i in range(len(pgrecs['specialist'])):
+            owners.append(pgrecs['specialist'][i])
+            names.append(pgrecs['fstname'][i] if pgrecs['fstname'][i] else pgrecs['specialist'][i])
+         subject = "PGCHKSUM: {} {} File{} Missing for {}".format(fcnt, self.PVALS['TYPE'], s, dsid)
+         msg = "Dear {},\n\n".format(' and '.join(names))
+         msg += ("Checking {} files of dataset {}, {} found the following {} file{} recorded in RDADB " +
+                 "but missing from the archive location{}. Please restore the file{}, or remove/update " +
+                 "the file record{} via dsarch if no longer needed:\n\n").format(
+                 self.PVALS['TYPE'], dsid, self.pgname, fcnt, s, s, s, s)
+         for i in range(fcnt):
+            msg += "{}. {}\n".format(i+1, files[i])
+         msg += "\n{} on {}\n".format(self.pgcmd, self.PGLOG['HOSTNAME'])
+         receiver = ', '.join(o if o.find('@') > -1 else o + "@ucar.edu" for o in owners)
+         if self.send_python_email(subject, receiver, msg, None, '', self.PVALS['emlerr']):
+            self.pglog("{}: emailed {} about {} missing {} file{}".format(dsid, receiver, fcnt, self.PVALS['TYPE'], s), self.PVALS['emllog'])
+
+   # record a missing file for the dataset owner email
+   def add_missing_file(self, fname, pgrec, location):
+      if not self.PGSUM['e']: return
+      dsid = pgrec['dsid']
+      if dsid not in self.MISSING: self.MISSING[dsid] = []
+      self.MISSING[dsid].append("{} (Type {}, {})".format(fname, pgrec['type'], location))
+
    # get the checksum filelist for wfile
    def get_checksum_wfilelist(self):
       self.PVALS['TYPE'] = "Web"
@@ -197,7 +240,7 @@ class PgChksum(PgCMD, PgSplit):
          if self.PGLOG['DSCHECK']: self.set_dscheck_attribute("dsid", dsid, self.PVALS['emlerr'])
          if self.PGSUM['m']: self.PVALS['CND'] += "(checksum IS NULL OR checksum < ' ') "
       elif dscnt > 1:
-         self.PVALS['CND'] = "dsid IN ({}) ".format(','.join(self.PGSUM['i']))
+         self.PVALS['CND'] = "dsid IN ('{}') ".format("','".join(self.PGSUM['i']))
       if self.PGSUM['o']: self.PVALS['CND'] += "ORDER BY {} ".format('wid' if self.PGSUM['o'] == 's' else "RANDOM()")
       if fcnt: self.PVALS['CND'] += "LIMIT {} ".format(fcnt)
       if self.PGSUM['k']:
@@ -206,6 +249,7 @@ class PgChksum(PgCMD, PgSplit):
       if dscnt == 1:
          wfrecs = self.pgmget_wfile(dsid, flds, self.PVALS['CND'])
          cnt = len(wfrecs['wid']) if wfrecs else 0
+         if cnt > 0: wfrecs['dsid'] = [dsid]*cnt   # wfile_{dsid} has no dsid column
       else:
          pgrecs = self.pgmget('wfile', 'wid, dsid', self.PVALS['CND'])
          cnt = len(pgrecs['wid']) if pgrecs else 0
@@ -245,7 +289,7 @@ class PgChksum(PgCMD, PgSplit):
          self.PVALS['CND'] = "dsid = '{}'".format(dsid)
          if self.PGLOG['DSCHECK']: self.set_dscheck_attribute("dsid", dsid, self.PVALS['emlerr'])
       elif cnt > 1:
-         self.PVALS['CND'] = "dsid IN ({})".format(','.join(self.PGSUM['i']))
+         self.PVALS['CND'] = "dsid IN ('{}')".format("','".join(self.PGSUM['i']))
       if self.PGSUM['m']:
          if self.PVALS['CND']: self.PVALS['CND'] += " AND "
          self.PVALS['CND'] += "(checksum IS NULL OR checksum < ' ')"
@@ -279,7 +323,7 @@ class PgChksum(PgCMD, PgSplit):
          self.PVALS['CND'] = "dsid = '{}' AND ".format(dsid)
          if self.PGLOG['DSCHECK']: self.set_dscheck_attribute("dsid", dsid, self.PVALS['emlerr'])
       elif cnt > 1:
-         self.PVALS['CND'] = "dsid IN ({}) AND ".format(','.join(self.PGSUM['i']))
+         self.PVALS['CND'] = "dsid IN ('{}') AND ".format("','".join(self.PGSUM['i']))
       if self.PGSUM['m']:
          self.PVALS['CND'] += "(checksum IS NULL OR checksum < ' ') AND "
       self.PVALS['CND'] += "status = 'A'"
@@ -376,6 +420,7 @@ class PgChksum(PgCMD, PgSplit):
          self.evaluate_file_status('DSK', fname, pgrec, info, cnts)
       else:
          self.pglog("{}: Type {} {} file is missing on Disk".format(fname, pgrec['type'], self.PVALS['TYPE']), self.PVALS['emllog'])
+         self.add_missing_file(fname, pgrec, 'on Disk')
          cnts['ecnt'] += 1
    
    # evaluate an object file
@@ -385,6 +430,7 @@ class PgChksum(PgCMD, PgSplit):
          self.evaluate_file_status('OBS', "{}-{}".format(bucket, fname), pgrec, info, cnts)
       else:
          self.pglog("{}: Type {} {} file is missing in Object Store".format(fname, pgrec['type'], self.PVALS['TYPE']), self.PVALS['emllog'])
+         self.add_missing_file("{}-{}".format(bucket, fname), pgrec, 'in Object Store')
          cnts['ecnt'] += 1
    
    # evaluate a quasar file
@@ -394,6 +440,7 @@ class PgChksum(PgCMD, PgSplit):
          self.evaluate_file_status('QSR', "{}-{}".format(endpoint, fname), pgrec, info, cnts)
       else:
          self.pglog("{}: Type {} {} file is missing on Quasar".format(fname, pgrec['type'], self.PVALS['TYPE']), self.PVALS['emllog'])
+         self.add_missing_file("{}-{}".format(endpoint, fname), pgrec, 'on Quasar')
          cnts['ecnt'] += 1
    
    # evaluate a file status

rda_python_dbms-3.0.1.dist-info/METADATA

@@ -0,0 +1,100 @@
+Metadata-Version: 2.4
+Name: rda_python_dbms
+Version: 3.0.1
+Summary: RDA Python Package to manage PostreSQL databases
+Author-email: Zaihua Ji <zji@ucar.edu>
+Project-URL: Homepage, https://github.com/NCAR/rda-python-dbms
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Development Status :: 5 - Production/Stable
+Requires-Python: >=3.7
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: rda_python_common
+Dynamic: license-file
+
+# rda_python_dbms
+
+RDA Python package to manage RDA PostgreSQL databases.
+
+## Programs
+
+The package installs the following command-line utilities, all running as the
+current user:
+
+- `pgddl` — create, drop or alter tables, indexes and keys from the table
+  definitions in `<tablename>.json` files
+- `pgseq` — reset the sequences of all (or the given) tables in one or more
+  schemas
+- `pgbackup` — dump the configured `rdadb` schemas on `rda-pgdb-02` into
+  `/data/pgbackup`, replacing existing backups and emailing a summary
+- `pgdbdump` — dump a PostgreSQL database to a local directory
+- `pgdbrestore` — restore a PostgreSQL database from a local dump directory
+- `pgchksum` — re-check archived files against their RDADB records
+  (checksums, sizes and timestamps for Web and Saved files; sizes and
+  timestamps only for Quasar backup files), optionally fix the records
+  (`-x`), and email each dataset's primary owner a list of its missing
+  files (`-e`)
+- `pgcmptables` — compare the columns, indexes and data of identical tables
+  on two database servers
+- `pgschema` — copy the tables of a schema into the same schema in another
+  database on the same server
+
+Run a program without its required options to print its usage. Do not use
+`-h` for help: it is the host name option in several programs. `pgbackup`
+has no usage text and starts the backup as soon as it runs.
+
+## Environment setup
+
+Create a Python environment first; package installs in the next section run
+inside whichever environment you activate here.
+
+### Option A — Python venv (DECS machines)
+
+```bash
+python3 -m venv $ENVHOME          # e.g. /glade/u/home/gdexdata/gdexmsenv
+source $ENVHOME/bin/activate
+```
+
+### Option B — Conda (DAV/Casper)
+
+```bash
+conda create --prefix $ENVHOME python=3.12   # e.g. /glade/work/gdexdata/conda-envs/pg-gdex
+conda activate $ENVHOME
+```
+
+## Installing rda-python-dbms
+
+Pick whichever install mode fits your workflow.  All variants pull in the
+transitive dependency (`rda_python_common`) automatically.
+
+For local development, clone this repo alongside your project and install it
+in editable mode so that changes are picked up without re-installing:
+
+```bash
+git clone https://github.com/NCAR/rda-python-dbms.git
+cd rda-python-dbms
+pip install -e .
+```
+
+To test a specific branch (e.g. an in-progress feature or fix branch), pass
+`-b/--branch` to `git clone`:
+
+```bash
+git clone -b <branch-name> https://github.com/NCAR/rda-python-dbms.git
+cd rda-python-dbms
+pip install -e .
+```
+
+For a regular (non-editable) install from a checkout:
+
+```bash
+pip install /path/to/rda-python-dbms
+```
+
+For a production install on a system that uses the published distribution:
+
+```bash
+pip install rda_python_dbms
+```

{rda_python_dbms-2.0.9.dist-info → rda_python_dbms-3.0.1.dist-info}/RECORD

@@ -10,7 +10,7 @@ rda_python_dbms/pg_ddllib.py,sha256=-zawllfUCe7QZNvza-u5TAYKP4yilC9K6TetHhoGenU,
 rda_python_dbms/pg_schema.py,sha256=Vsf5fKENYIOgMKE3yS0TkXhg-aeHZ2nYzpvltFAcdFY,5520
 rda_python_dbms/pg_seq.py,sha256=wNxqsyIjcTplJErPirZ-VkZ8HS4mgQUwPz8Xwd12kYA,4914
 rda_python_dbms/pgbackup.py,sha256=IuP3nQEEVR86YlSZuGyqsdz1-O9rgET76FnWB7iL3xk,4249
-rda_python_dbms/pgchksum.py,sha256=8jfBncptxp57Gr2t66Kk9MFt5Z_aXfozOGZPaohKGwM,21802
+rda_python_dbms/pgchksum.py,sha256=ePHdh2MQ3q8102Nm7znXPg8ToI-ohGnyg3f-ipIYWu8,24921
 rda_python_dbms/pgcmptables.py,sha256=LQ4KK4JuQs2z3nBdINWYgyiGu2MF9RZjS5_cAktbs6k,47123
 rda_python_dbms/pgdbdump.py,sha256=8d-1z16Y2DyJnQqex2wFu5svChKgncPnzdQ-iUftHg8,3831
 rda_python_dbms/pgdbrestore.py,sha256=d12hl1r6usK1oGIFAPVWUB_KkIH04PPlugxVSnLOX1g,3110
@@ -954,9 +954,9 @@ rda_python_dbms/tables/obsua/wind_5v2_0.json,sha256=NehMFE-GMNlD5Bh_fiacfSOmZWkr
 rda_python_dbms/tables/obsua/wind_60v2_0.json,sha256=jOyQgD5ph7YBMuQfvYz1Ln6X_-_Vdhl36HBexOHZ1dA,1067
 rda_python_dbms/tables/obsua/wind_8v2_0.json,sha256=5wtCThmFeOyiAGH_Qv3_jrXjhjo9b87ehlpI9QQY2SM,1064
 rda_python_dbms/tables/obsua/wind_9v2_0.json,sha256=7OKFUicOn15LL_WuF1VE_L8DjxsHmRP4uBnopuMIeTg,1064
-rda_python_dbms-2.0.9.dist-info/licenses/LICENSE,sha256=Uky6fiiVw7cakFVRtVpzg-mvqOQ-WcA_H8lHqhPRg58,1101
-rda_python_dbms-2.0.9.dist-info/METADATA,sha256=8jhKTRIo91_CXYgIdyMBwXbCZQ1VOpS-1ghgC6NxVbI,617
-rda_python_dbms-2.0.9.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
-rda_python_dbms-2.0.9.dist-info/entry_points.txt,sha256=iti6iZS2OEeBqO5aHlWeMFcbuwN_2assidP0qeSkQ6E,346
-rda_python_dbms-2.0.9.dist-info/top_level.txt,sha256=4T1ShNUMyiqmwDIuDL7pDOm_S-6zWWW3Vy6OdAzYA4c,16
-rda_python_dbms-2.0.9.dist-info/RECORD,,
+rda_python_dbms-3.0.1.dist-info/licenses/LICENSE,sha256=Uky6fiiVw7cakFVRtVpzg-mvqOQ-WcA_H8lHqhPRg58,1101
+rda_python_dbms-3.0.1.dist-info/METADATA,sha256=iqXoJ9_l_fP-i2prZLDDLZnXx9Cvko9VL2OCEXz1kAI,3293
+rda_python_dbms-3.0.1.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
+rda_python_dbms-3.0.1.dist-info/entry_points.txt,sha256=iti6iZS2OEeBqO5aHlWeMFcbuwN_2assidP0qeSkQ6E,346
+rda_python_dbms-3.0.1.dist-info/top_level.txt,sha256=4T1ShNUMyiqmwDIuDL7pDOm_S-6zWWW3Vy6OdAzYA4c,16
+rda_python_dbms-3.0.1.dist-info/RECORD,,

{rda_python_dbms-2.0.9.dist-info → rda_python_dbms-3.0.1.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (82.0.1)
+Generator: setuptools (84.2.1)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

rda_python_dbms-2.0.9.dist-info/METADATA

@@ -1,17 +0,0 @@
-Metadata-Version: 2.4
-Name: rda_python_dbms
-Version: 2.0.9
-Summary: RDA Python Package to manage PostreSQL databases
-Author-email: Zaihua Ji <zji@ucar.edu>
-Project-URL: Homepage, https://github.com/NCAR/rda-python-dbms
-Classifier: Programming Language :: Python :: 3
-Classifier: License :: OSI Approved :: MIT License
-Classifier: Operating System :: OS Independent
-Classifier: Development Status :: 5 - Production/Stable
-Requires-Python: >=3.7
-Description-Content-Type: text/markdown
-License-File: LICENSE
-Requires-Dist: rda_python_common
-Dynamic: license-file
-
-RDA python Package to manage RDA PostgreSQL databases.