rda-python-dbms 2.0.8__py3-none-any.whl → 3.0.0__py3-none-any.whl

rda_python_dbms/pgchksum.py

@@ -22,6 +22,7 @@ class PgChksum(PgCMD, PgSplit):
       super().__init__()  # initialize parent class
       self.PGSUM = {
          'd' : [],   # delay mode option, value array
+         'e' : 0,    # 1 for emailing dataset owners the list of their missing files
          's' : [],   # specialist login name array for email notice, default to none
          'f' : [],   # file name array, default to none
          'i' : [],   # dataset id array, default for all
@@ -46,6 +47,7 @@ class PgChksum(PgCMD, PgSplit):
          'begin' : 0,
          'end' : 0
       }
+      self.MISSING = {}   # dsid => list of missing file names with their location
       self.pgname = "pgchksum"
       self.pgcmd = None
 
@@ -61,7 +63,7 @@ class PgChksum(PgCMD, PgSplit):
                self.PGLOG['BCKGRND'] = 1   # processing in backgroup mode
                option = None
             elif option in self.PGSUM:
-               if option in ['m', 'n', 'x']:
+               if option in ['e', 'm', 'n', 'x']:
                   self.PGSUM[option] = 1
                   option = None
             else:
@@ -80,12 +82,13 @@ class PgChksum(PgCMD, PgSplit):
          else:
             self.pglog(arg + ": value passed without leading Option", self.LGEREX)
       if not self.PGSUM['t']:
-         print("Usage: pgchksum -t (W|S|Q) [-b] [-m] [-n] [-x] [-c FileCount] \\")
+         print("Usage: pgchksum -t (W|S|Q) [-b] [-e] [-m] [-n] [-x] [-c FileCount] \\")
          print("                [-d [Hostnames] [RerunCount]] [-f FileNames] [-i DatasetIDs] \\")
          print("                [-k SkipCount] [-o r|s] [-s Specialists]")
          print("  -b - background process and no screen output")
          print("  -c - File count to re-evaluate checksum; for all gathered files if not specified")
          print("  -d - delayed mode background/batch processes on RDA machines/DAV PBS nodes")
+         print("  -e - email the primary owner of each dataset a reminder listing its missing files")
          print("  -f - Re-evaluate the specify File names only")
          print("  -i - Dataset IDs the files belong; for all files if not specified")
          print("  -k - skip the given file count to re-evaluate checksum")
@@ -100,6 +103,8 @@ class PgChksum(PgCMD, PgSplit):
          self.pgexit(0)
       elif self.PGSUM['t'] not in 'QSW':
          self.pglog(self.PGSUM['t'] + ": Unknown File Type, must be one of 'QSW'", self.LGEREX)
+      if self.PGSUM['e'] and self.PGSUM['n']:
+         self.pglog("Option -e (email owners) cannot work with -n (no email)", self.LGEREX)
        # set different log file
       self.PGLOG['LOGFILE'] = self.pgname + '.log'
       self.set_suid(self.PGLOG['EUID'])
@@ -151,6 +156,7 @@ class PgChksum(PgCMD, PgSplit):
             self.evaluate_backfile_checksum(fcnt, pgrecs, cnts)
          self.dump_progress(fcnt, fcnt, cnts, self.PVALS['emlsum'])
          self.PVALS['end'] = int(time.time())
+         if self.PGSUM['e'] and self.MISSING: self.send_owner_emails()
          if not self.PGSUM['n']: self.send_check_email(cnts['cnt'], fcnt)
       if self.PGLOG['DSCHECK']:
          if self.PGLOG['ERRMSG']:
@@ -173,6 +179,43 @@ class PgChksum(PgCMD, PgSplit):
       else:
          self.pglog(subject, self.LOGWRN|self.SNDEML)
    
+   # email the primary owner(s) of each dataset the list of its missing files
+   def send_owner_emails(self):
+      for dsid in sorted(self.MISSING):
+         files = self.MISSING[dsid]
+         fcnt = len(files)
+         s = 's' if fcnt > 1 else ''
+         tjoin = "dsowner LEFT JOIN dssgrp ON specialist = logname"
+         pgrecs = self.pgmget(tjoin, "specialist, fstname",
+                              "dsid = '{}' AND priority = 1 ORDER BY specialist".format(dsid), self.PVALS['emlerr'])
+         if not pgrecs or not pgrecs['specialist']:
+            self.pglog("{}: No primary owner found in dsowner to email {} missing {} file{}".format(dsid, fcnt, self.PVALS['TYPE'], s), self.PVALS['emllog'])
+            continue
+         owners = []
+         names = []
+         for i in range(len(pgrecs['specialist'])):
+            owners.append(pgrecs['specialist'][i])
+            names.append(pgrecs['fstname'][i] if pgrecs['fstname'][i] else pgrecs['specialist'][i])
+         subject = "PGCHKSUM: {} {} File{} Missing for {}".format(fcnt, self.PVALS['TYPE'], s, dsid)
+         msg = "Dear {},\n\n".format(' and '.join(names))
+         msg += ("Checking {} files of dataset {}, {} found the following {} file{} recorded in RDADB " +
+                 "but missing from the archive location{}. Please restore the file{}, or remove/update " +
+                 "the file record{} via dsarch if no longer needed:\n\n").format(
+                 self.PVALS['TYPE'], dsid, self.pgname, fcnt, s, s, s, s)
+         for i in range(fcnt):
+            msg += "{}. {}\n".format(i+1, files[i])
+         msg += "\n{} on {}\n".format(self.pgcmd, self.PGLOG['HOSTNAME'])
+         receiver = ', '.join(o if o.find('@') > -1 else o + "@ucar.edu" for o in owners)
+         if self.send_python_email(subject, receiver, msg, None, '', self.PVALS['emlerr']):
+            self.pglog("{}: emailed {} about {} missing {} file{}".format(dsid, receiver, fcnt, self.PVALS['TYPE'], s), self.PVALS['emllog'])
+
+   # record a missing file for the dataset owner email
+   def add_missing_file(self, fname, pgrec, location):
+      if not self.PGSUM['e']: return
+      dsid = pgrec['dsid']
+      if dsid not in self.MISSING: self.MISSING[dsid] = []
+      self.MISSING[dsid].append("{} (Type {}, {})".format(fname, pgrec['type'], location))
+
    # get the checksum filelist for wfile
    def get_checksum_wfilelist(self):
       self.PVALS['TYPE'] = "Web"
@@ -197,7 +240,7 @@ class PgChksum(PgCMD, PgSplit):
          if self.PGLOG['DSCHECK']: self.set_dscheck_attribute("dsid", dsid, self.PVALS['emlerr'])
          if self.PGSUM['m']: self.PVALS['CND'] += "(checksum IS NULL OR checksum < ' ') "
       elif dscnt > 1:
-         self.PVALS['CND'] = "dsid IN ({}) ".format(','.join(self.PGSUM['i']))
+         self.PVALS['CND'] = "dsid IN ('{}') ".format("','".join(self.PGSUM['i']))
       if self.PGSUM['o']: self.PVALS['CND'] += "ORDER BY {} ".format('wid' if self.PGSUM['o'] == 's' else "RANDOM()")
       if fcnt: self.PVALS['CND'] += "LIMIT {} ".format(fcnt)
       if self.PGSUM['k']:
@@ -206,6 +249,7 @@ class PgChksum(PgCMD, PgSplit):
       if dscnt == 1:
          wfrecs = self.pgmget_wfile(dsid, flds, self.PVALS['CND'])
          cnt = len(wfrecs['wid']) if wfrecs else 0
+         if cnt > 0: wfrecs['dsid'] = [dsid]*cnt   # wfile_{dsid} has no dsid column
       else:
          pgrecs = self.pgmget('wfile', 'wid, dsid', self.PVALS['CND'])
          cnt = len(pgrecs['wid']) if pgrecs else 0
@@ -245,7 +289,7 @@ class PgChksum(PgCMD, PgSplit):
          self.PVALS['CND'] = "dsid = '{}'".format(dsid)
          if self.PGLOG['DSCHECK']: self.set_dscheck_attribute("dsid", dsid, self.PVALS['emlerr'])
       elif cnt > 1:
-         self.PVALS['CND'] = "dsid IN ({})".format(','.join(self.PGSUM['i']))
+         self.PVALS['CND'] = "dsid IN ('{}')".format("','".join(self.PGSUM['i']))
       if self.PGSUM['m']:
          if self.PVALS['CND']: self.PVALS['CND'] += " AND "
          self.PVALS['CND'] += "(checksum IS NULL OR checksum < ' ')"
@@ -279,7 +323,7 @@ class PgChksum(PgCMD, PgSplit):
          self.PVALS['CND'] = "dsid = '{}' AND ".format(dsid)
          if self.PGLOG['DSCHECK']: self.set_dscheck_attribute("dsid", dsid, self.PVALS['emlerr'])
       elif cnt > 1:
-         self.PVALS['CND'] = "dsid IN ({}) AND ".format(','.join(self.PGSUM['i']))
+         self.PVALS['CND'] = "dsid IN ('{}') AND ".format("','".join(self.PGSUM['i']))
       if self.PGSUM['m']:
          self.PVALS['CND'] += "(checksum IS NULL OR checksum < ' ') AND "
       self.PVALS['CND'] += "status = 'A'"
@@ -376,6 +420,7 @@ class PgChksum(PgCMD, PgSplit):
          self.evaluate_file_status('DSK', fname, pgrec, info, cnts)
       else:
          self.pglog("{}: Type {} {} file is missing on Disk".format(fname, pgrec['type'], self.PVALS['TYPE']), self.PVALS['emllog'])
+         self.add_missing_file(fname, pgrec, 'on Disk')
          cnts['ecnt'] += 1
    
    # evaluate an object file
@@ -385,6 +430,7 @@ class PgChksum(PgCMD, PgSplit):
          self.evaluate_file_status('OBS', "{}-{}".format(bucket, fname), pgrec, info, cnts)
       else:
          self.pglog("{}: Type {} {} file is missing in Object Store".format(fname, pgrec['type'], self.PVALS['TYPE']), self.PVALS['emllog'])
+         self.add_missing_file("{}-{}".format(bucket, fname), pgrec, 'in Object Store')
          cnts['ecnt'] += 1
    
    # evaluate a quasar file
@@ -394,6 +440,7 @@ class PgChksum(PgCMD, PgSplit):
          self.evaluate_file_status('QSR', "{}-{}".format(endpoint, fname), pgrec, info, cnts)
       else:
          self.pglog("{}: Type {} {} file is missing on Quasar".format(fname, pgrec['type'], self.PVALS['TYPE']), self.PVALS['emllog'])
+         self.add_missing_file("{}-{}".format(endpoint, fname), pgrec, 'on Quasar')
          cnts['ecnt'] += 1
    
    # evaluate a file status

rda_python_dbms/tables/dssdb/dsgroup.json

@@ -30,7 +30,7 @@
     },
     {
       "name": "grpid",
-      "type": "varchar(40)",
+      "type": "varchar(128)",
       "comment": "group id to identify a group"
     },
     {

{rda_python_dbms-2.0.8.dist-info → rda_python_dbms-3.0.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_dbms
-Version: 2.0.8
+Version: 3.0.0
 Summary: RDA Python Package to manage PostreSQL databases
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-dbms

{rda_python_dbms-2.0.8.dist-info → rda_python_dbms-3.0.0.dist-info}/RECORD

@@ -10,7 +10,7 @@ rda_python_dbms/pg_ddllib.py,sha256=-zawllfUCe7QZNvza-u5TAYKP4yilC9K6TetHhoGenU,
 rda_python_dbms/pg_schema.py,sha256=Vsf5fKENYIOgMKE3yS0TkXhg-aeHZ2nYzpvltFAcdFY,5520
 rda_python_dbms/pg_seq.py,sha256=wNxqsyIjcTplJErPirZ-VkZ8HS4mgQUwPz8Xwd12kYA,4914
 rda_python_dbms/pgbackup.py,sha256=IuP3nQEEVR86YlSZuGyqsdz1-O9rgET76FnWB7iL3xk,4249
-rda_python_dbms/pgchksum.py,sha256=8jfBncptxp57Gr2t66Kk9MFt5Z_aXfozOGZPaohKGwM,21802
+rda_python_dbms/pgchksum.py,sha256=ePHdh2MQ3q8102Nm7znXPg8ToI-ohGnyg3f-ipIYWu8,24921
 rda_python_dbms/pgcmptables.py,sha256=LQ4KK4JuQs2z3nBdINWYgyiGu2MF9RZjS5_cAktbs6k,47123
 rda_python_dbms/pgdbdump.py,sha256=8d-1z16Y2DyJnQqex2wFu5svChKgncPnzdQ-iUftHg8,3831
 rda_python_dbms/pgdbrestore.py,sha256=d12hl1r6usK1oGIFAPVWUB_KkIH04PPlugxVSnLOX1g,3110
@@ -61,7 +61,7 @@ rda_python_dbms/tables/dssdb/dscart.json,sha256=ggVt5SVi1EbNc9bEA7dCRRF4l7pOZIHm
 rda_python_dbms/tables/dssdb/dscheck.json,sha256=ZL6VNerF2PGcP0yPFUXhq7dzJDc5g5xX64eURAghGNY,6426
 rda_python_dbms/tables/dssdb/dschkhist.json,sha256=UeTjTsIpBGGscPNajL1tl7uflQWbYe3ILkyBcDBb9Mc,2880
 rda_python_dbms/tables/dssdb/dsdaemon.json,sha256=Ua0TktcZ5RPEQbwEblnNjWRBjDp-xcuunsaCAIaXsx0,1859
-rda_python_dbms/tables/dssdb/dsgroup.json,sha256=nIYLZhK-ru8zmC6fxqqzl9XhPjf3xL9kxUt9dwpFGNs,5388
+rda_python_dbms/tables/dssdb/dsgroup.json,sha256=aZ7RtRWnRcIruQesdsbUglHxvb8tROfNoWMNP0Bfzdk,5389
 rda_python_dbms/tables/dssdb/dshist.json,sha256=EEpCLLwS0vfV9OwMWgrRmvHP81gMybG8NOTxtDeHR1Y,1396
 rda_python_dbms/tables/dssdb/dslimit.json,sha256=hdpRAC4Bg0ZUCwVVDCCHVwYJcDLotXFB2e30G7ByzKg,758
 rda_python_dbms/tables/dssdb/dsokeys.json,sha256=4c_AsQgxBApawhnkKTzXflxL-603J19z8poimxe_sXw,746
@@ -954,9 +954,9 @@ rda_python_dbms/tables/obsua/wind_5v2_0.json,sha256=NehMFE-GMNlD5Bh_fiacfSOmZWkr
 rda_python_dbms/tables/obsua/wind_60v2_0.json,sha256=jOyQgD5ph7YBMuQfvYz1Ln6X_-_Vdhl36HBexOHZ1dA,1067
 rda_python_dbms/tables/obsua/wind_8v2_0.json,sha256=5wtCThmFeOyiAGH_Qv3_jrXjhjo9b87ehlpI9QQY2SM,1064
 rda_python_dbms/tables/obsua/wind_9v2_0.json,sha256=7OKFUicOn15LL_WuF1VE_L8DjxsHmRP4uBnopuMIeTg,1064
-rda_python_dbms-2.0.8.dist-info/licenses/LICENSE,sha256=Uky6fiiVw7cakFVRtVpzg-mvqOQ-WcA_H8lHqhPRg58,1101
-rda_python_dbms-2.0.8.dist-info/METADATA,sha256=yGW3Ewn-Bd1Nu5TcvSLrB9vj5-aUQ6Hm55XeL2eibAg,617
-rda_python_dbms-2.0.8.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
-rda_python_dbms-2.0.8.dist-info/entry_points.txt,sha256=iti6iZS2OEeBqO5aHlWeMFcbuwN_2assidP0qeSkQ6E,346
-rda_python_dbms-2.0.8.dist-info/top_level.txt,sha256=4T1ShNUMyiqmwDIuDL7pDOm_S-6zWWW3Vy6OdAzYA4c,16
-rda_python_dbms-2.0.8.dist-info/RECORD,,
+rda_python_dbms-3.0.0.dist-info/licenses/LICENSE,sha256=Uky6fiiVw7cakFVRtVpzg-mvqOQ-WcA_H8lHqhPRg58,1101
+rda_python_dbms-3.0.0.dist-info/METADATA,sha256=B9DrL7R_MmsAlJg1nRTq0jUBKjhPMFfITFGqlX5kqIE,617
+rda_python_dbms-3.0.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
+rda_python_dbms-3.0.0.dist-info/entry_points.txt,sha256=iti6iZS2OEeBqO5aHlWeMFcbuwN_2assidP0qeSkQ6E,346
+rda_python_dbms-3.0.0.dist-info/top_level.txt,sha256=4T1ShNUMyiqmwDIuDL7pDOm_S-6zWWW3Vy6OdAzYA4c,16
+rda_python_dbms-3.0.0.dist-info/RECORD,,

{rda_python_dbms-2.0.8.dist-info → rda_python_dbms-3.0.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (82.0.1)
+Generator: setuptools (84.2.1)
 Root-Is-Purelib: true
 Tag: py3-none-any