rda-python-common 1.0.6__tar.gz → 1.0.8__tar.gz

{rda_python_common-1.0.6 → rda_python_common-1.0.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rda_python_common
-Version: 1.0.6
+Version: 1.0.8
 Summary: RDA Python common library codes shared by other RDA python packages
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-common

{rda_python_common-1.0.6 → rda_python_common-1.0.8}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "rda_python_common"
-version = "1.0.6"
+version = "1.0.8"
 
 authors = [
   { name="Zaihua Ji", email="zji@ucar.edu" },

{rda_python_common-1.0.6 → rda_python_common-1.0.8}/src/rda_python_common/PgDBI.py

@@ -2226,6 +2226,8 @@ def get_pgpass_password():
    pwname = DBPASS.get((PGDBI['DBSHOST'], dbport, PGDBI['DBNAME'], PGDBI['LNNAME']))
    if not pwname: pwname = DBPASS.get((PGDBI['DBHOST'], dbport, PGDBI['DBNAME'], PGDBI['LNNAME']))
 
+   return pwname
+
 #
 # Reads the .pgpass file and returns a dictionary of credentials.
 #

{rda_python_common-1.0.6 → rda_python_common-1.0.8}/src/rda_python_common/PgFile.py

@@ -118,6 +118,8 @@ ENDPOINTS = {
    'rda-quasar-drdata' : "NCAR RDA Quasar DRDATA"
 }
 
+BFILES = {}  # cache backup file names and dates for each bid
+
 #
 # reset the up limit for a specified error type
 #
@@ -3012,3 +3014,52 @@ def check_storage_dflags(dflags, dscheck = None, logact = 0):
       if cidx: PgDBI.pgexec("UPDATE dscheck SET dflags = '' WHERE cindex = {}".format(cidx), logact)
 
    return msgary
+
+#
+# check a RDA file is backed up or not for given file record;
+# clear the cached bfile records if frec is None.
+# return 0 if not yet, 1 if backed up, or -1 if backed up but modified
+#
+def file_backup_status(frec, chgdays = 1, logact = 0):
+
+   if frec is None:
+      BFILES.clear()
+      return 0
+
+   bid = frec['bid']
+   if not bid: return 0
+
+   fields = 'bfile, dsid, date_modified'
+   if chgdays > 0: fields += ', note'
+   if bid not in BFILES: BFILES[bid] = PgDBI.pgget('bfile', fields, 'bid = {}'.format(bid), logact)
+   brec = BFILES[bid]
+   if not brec: return 0
+
+   if 'sfile' in frec:
+      fname = frec['sfile']
+      ftype = 'Saved'
+   else:
+      fname = frec['wfile']
+      ftype = 'Web'
+   ret = 1
+   fdate = frec['date_modified']
+   bdate = brec['date_modified']
+   if chgdays > 0 and PgUtil.diffdate(fdate, bdate) >= chgdays:
+      ret = -1
+      if brec['note']:
+         mp = r'{}<:>{}<:>(\d+)<:>(\w+)<:>'.format(fname, frec['type']) 
+         ms = re.search(mp, brec['note'])
+         if ms:
+            fsize = int(ms.group(1))
+            cksum = ms.group(2)
+            if cksum and cksum == frec['checksum'] or not cksum and fsize == frec['data_size']:
+               ret = 1
+
+   if logact:
+      if ret == 1:
+         msg = "{}-{}: {} file backed up to /{}/{} by {}".format(frec['dsid'], fname, ftype, brec['dsid'], brec['bfile'], bdate)
+      else:
+         msg = "{}-{}: {} file changed on {}".format(frec['dsid'], fname, ftype, fdate)
+      PgLOG.pglog(msg, logact)
+
+   return ret

{rda_python_common-1.0.6 → rda_python_common-1.0.8}/src/rda_python_common.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rda_python_common
-Version: 1.0.6
+Version: 1.0.8
 Summary: RDA Python common library codes shared by other RDA python packages
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-common

{rda_python_common-1.0.6 → rda_python_common-1.0.8}/src/rda_python_common.egg-info/SOURCES.txt

@@ -4,11 +4,9 @@ pyproject.toml
 src/rda_python_common/PgCMD.py
 src/rda_python_common/PgDBI.py
 src/rda_python_common/PgFile.py
-src/rda_python_common/PgGLBS.py
 src/rda_python_common/PgLOG.py
 src/rda_python_common/PgLock.py
 src/rda_python_common/PgOPT.py
-src/rda_python_common/PgPGS.py
 src/rda_python_common/PgSIG.py
 src/rda_python_common/PgSplit.py
 src/rda_python_common/PgUtil.py

rda_python_common-1.0.6/src/rda_python_common/PgGLBS.py

@@ -1,200 +0,0 @@
-#
-###############################################################################
-#
-#     Title : PgGLBS.py
-#    Author : Thomas Cram, tcram@ucar.edu
-#      Date : 12/10/2014
-#             10/10/2020, Zaihua Ji, zji@ucar.edu:
-#             converted from perl package to python module
-#             2025-01-10, Zaihua Ji, zji@ucar.edu:
-#             transferred to package rda_python_common from
-#             https://github.com/NCAR/rda-shared-libraries.git
-#   Purpose : python library module for Globus functions and utilities
-#
-#    Github : https://github.com/NCAR/rda-python-common.git
-# 
-###############################################################################
-#
-import os
-import re
-from . import PgLOG
-from . import PgUtil
-from MyGlobus import MyGlobus, MyEndpoints, GLOBUS_REQUEST_DOMAIN
-from . import PgDBI
-
-try:
-   from urllib.parse import urlencode
-except:
-   from urllib import urlencode
-
-BFILES = {}  # cache backup file names and dates for each bid
-
-#
-# Remove the Globus share rule ID for a dsrqst share 
-#
-def remove_globus_rid(ridx, dsid):
-
-   if not ridx: return PgLOG.pglog("[remove_globus_rid] Request index is not defined", PgLOG.LOGWRN)
-   if not dsid: return PgLOG.pglog("[remove_globus_rid] Dataset ID is not defined", PgLOG.LOGWRN)
-
-   cmd = "dsglobus"
-   args = "'-rp -ri {}'".format(ridx)
-   action = "RP"
-   host = "PBS"
-   workdir = "/glade/u/home/tcram"
-   opts = "'-l walltime=15:00'"
-   spec = "tcram"
-   check_cmd = "dscheck ac -cm {} -av {} -ds {} -an {} -hn {} -wd {} -sn {} -qs {} -md".format(cmd, args, dsid, action, host, workdir, spec, opts)
-
-   PgLOG.pgsystem(check_cmd)
-
-#
-# Submit a Globus transfer of the request output on behalf of the user
-#
-def submit_globus_transfer(ridx):
-
-   # call dsglobus to submit transfer
-   cmd = "dsglobus -st -ri {}".format(ridx)
-   return PgLOG.pgsystem(cmd, PgLOG.LOGWRN, 16)
-
-#
-# check a RDA file is backed up or not for given file record;
-# clear the cached bfile records if frec is None.
-# return 0 if not yet, 1 if backed up, or -1 if backed up but modified
-#
-def file_backup_status(frec, chgdays = 1, logact = 0):
-
-   if frec is None:
-      BFILES.clear()
-      return 0
-
-   bid = frec['bid']
-   if not bid: return 0
-
-   fields = 'bfile, dsid, date_modified'
-   if chgdays > 0: fields += ', note'
-   if bid not in BFILES: BFILES[bid] = PgDBI.pgget('bfile', fields, 'bid = {}'.format(bid), logact)
-   brec = BFILES[bid]
-   if not brec: return 0
-
-   if 'sfile' in frec:
-      fname = frec['sfile']
-      ftype = 'Saved'
-   else:
-      fname = frec['wfile']
-      ftype = 'Web'
-   ret = 1
-   fdate = frec['date_modified']
-   bdate = brec['date_modified']
-   if chgdays > 0 and PgUtil.diffdate(fdate, bdate) >= chgdays:
-      ret = -1
-      if brec['note']:
-         mp = r'{}<:>{}<:>(\d+)<:>(\w+)<:>'.format(fname, frec['type']) 
-         ms = re.search(mp, brec['note'])
-         if ms:
-            fsize = int(ms.group(1))
-            cksum = ms.group(2)
-            if cksum and cksum == frec['checksum'] or not cksum and fsize == frec['data_size']:
-               ret = 1
-
-   if logact:
-      if ret == 1:
-         msg = "{}-{}: {} file backed up to /{}/{} by {}".format(frec['dsid'], fname, ftype, brec['dsid'], brec['bfile'], bdate)
-      else:
-         msg = "{}-{}: {} file changed on {}".format(frec['dsid'], fname, ftype, fdate)
-      PgLOG.pglog(msg, logact)
-
-   return ret
-
-#=========================================================================================
-def get_request_file_url(rfile, rpath=None, logact=0):
-   """ Returns the URL for a request file 
-       Input arguments:
-          rfile = request file
-	  dsid = dataset ID (dsnnn.n)
-	  rpath = path to request file, relatvie to RDA data base path (e.g. '/dsrqst/<rqstid>/'
-   """
-   domain = GLOBUS_REQUEST_DOMAIN
-
-   if not rpath:
-      try:
-         cond = "wfile='{}'".format(rfile)
-         wfrqst = PgDBI.pgget('wfrqst', 'rindex', cond, logact)
-      except:
-         msg = "[get_request_file_url] Problem getting rindex for request file {}".format(rfile)
-         PgLOG.pglog(msg)
-      if not wfrqst:
-         raise TypeError("Request file {} not found in table 'wfrqst'".format(rfile))
-      rpath = get_request_path(wfrqst['rindex'], logact=0)
-
-   if (rpath.find('/',0,1) != -1):
-      rpath = rpath.replace('/','',1)
- 
-   url = os.path.join(domain, rpath, rfile)
-   return url
-
-#=========================================================================================
-def get_request_path(rindex, logact=0):
-   """ Returns relative path to request file 
-       Example: '/dsrqst/<rqstid>/'
-   """
-   try:
-      fields = 'rqstid, location'
-      cond = 'rindex={}'.format(rindex)
-      rqst_info = PgDBI.pgget('dsrqst', fields, cond, logact)
-   except:
-      msg = "[get_request_path] Problem getting info for request index {}".format(rindex)
-      PgLOG.pglog(msg)
-   if not rqst_info:
-      raise TypeError("Request index {} not found in RDADB".format(rindex))	
-
-   if rqst_info['location']:
-      base_path = MyGlobus['data_request_endpoint_base']
-      loc = rqst_info['location']
-      loc = loc.rstrip("/")
-      if (loc.find(base_path) != -1):
-         path_len = len(base_path)
-         path = "/{0}/".format(loc[path_len:])
-      else:
-         path = "/"
-   else:
-      path = "/dsrqst/{0}/".format(rqst_info['rqstid'])
-
-   return path
-
-#=========================================================================================
-def get_guest_collection_url(dsid=None, locflag=None, rindex=None, logact=0):
-   """ Returns the URL for the guest collection endpoint in the Globus File Manager.
-       Either dataset ID (dsid) or request index (rindex) is required.  If neither
-       dsid or rindex are provided, the default URL returned is the top level URL for 
-       the 'NCAR RDA Dataset Archive' guest collection.
-	
-       Optional argument locflag = location flag of dataset ('G' = glade, 'O' = stratus, 
-       'B' = both glade and stratus, 'C' = CGD data under /glade/campaign/cgd/cesm)
-   """
-
-   if rindex:
-      origin_id = MyEndpoints['rda#data_request']
-      origin_path = get_request_path(rindex, logact=logact)
-   elif dsid:
-      if not locflag:
-         cond = "dsid='{}'".format(dsid)
-         pgloc = PgDBI.pgget('dataset', 'locflag', cond, logact)
-         locflag = pgloc['locflag']
-      if locflag == 'C':
-         origin_id = MyEndpoints['rda-cgd']
-         origin_path = "/"
-      elif locflag == 'O' or locflag == 'B':
-         origin_id = MyEndpoints['rda-stratus']
-         origin_path = "/{}/".format(dsid)
-      else:
-         origin_id = MyEndpoints['rda#datashare']
-         origin_path = "/{}/".format(dsid)
-   else:
-      origin_id = MyEndpoints['rda#datashare']
-      origin_path = "/"
-	
-   params = {'origin_id': origin_id, 'origin_path': origin_path}
-   url = '{0}?{1}'.format(MyGlobus['globus_share_url'], urlencode(params))
-
-   return url

rda_python_common-1.0.6/src/rda_python_common/PgPGS.py

@@ -1,153 +0,0 @@
-#
-###############################################################################
-#
-#     Title : PgPGS.py  -- PostgreSQL Interface for CDP DataBase Per psql
-#    Author : Zaihua Ji,  zji@ucar.edu
-#      Date : 08/31/2020
-#             2025-01-10 transferred to package rda_python_common from
-#             https://github.com/NCAR/rda-shared-libraries.git
-#   Purpose : python library module to handle sql scripts to retrieve info
-#             from cdp database per psql
-#
-#    Github : https://github.com/NCAR/rda-python-common.git
-#
-###############################################################################
-#
-import os
-import re
-from . import PgLOG
-
-PGPGS = {}
-PGPGS["PGSSERV"] = PgLOG.get_environment("PGSSERV", '-h vetsdbprod -p 5432 -U acadmin access_control');
-PGPGS["SQLPATH"] = PgLOG.get_environment("SQLPATH", PgLOG.PGLOG['DSSHOME']+ "/dssdb/sql");
-
-#
-# local function: create sql file
-#
-def pgs_sql_file(tablenames, fields, condition = None):
-
-   sqlfile = "{}/pgs{}.sql".format(PGPGS['SQLPATH'], os.getpid())
-
-   sqlstr = "SELECT {}\nFROM {}".format(fields, tablenames)
-   if condition:
-      if re.match(r'^\s*(ORDER|GROUP|HAVING)\s', condition, re.I):
-         sqlstr += "\n{}".format(condition)
-      else:
-         sqlstr += "\nWHERE {}".format(condition)
-   sqlstr += ";\n"
-   try:
-      SQL = open(sqlfile, 'w')
-      SQL.write(sqlstr)
-      SQL.close()
-   except Exception as e:
-      PgLOG.pglog("Error Open '{}': {}".format(sqlfile, str(e)), PgLOG.LGWNEX)
-
-   if PgLOG.PGLOG['DBGLEVEL']: PgLOG.pgdbg(1000, sqlstr)
-
-   return sqlfile
-
-#
-# tablenames: comma deliminated string of table names
-#     fields: fieldnames for query pgscle database,
-#  condition: querry conditions for where clause)
-#   Return: one record from tablename, a hash reference with keys as field names
-#           and values as field values upon success, FAILURE otherwise 
-#
-def pgsget(tablenames, fields, condition = None, logact = 0):
-
-   sqlfile = pgs_sql_file(tablenames, fields, condition)
-   sqlout = PgLOG.pgsystem("psql {} < {}".format(PGPGS['PGSSERV'], sqlfile), logact, 273+1024)   # 1+16+256
-
-   colcnt = 0
-   record = {}
-   if sqlout:
-      for line in re.split(r'\n', sqlout):
-         vals = re.split(r'\s*\|\s+', line)
-         if colcnt:    # gather data
-            record = dict(zip(fields, vals))
-            break
-         else:    # gather field names
-            flds = vals
-            colcnt = len(flds)
-   elif PgLOG.PGLOG['SYSERR']:    # error happens
-      PgLOG.pglog(PgLOG.PGLOG['SYSERR'], logact|PgLOG.ERRLOG)
-
-   if PgLOG.PGLOG['DBGLEVEL']:
-      if record:
-         PgLOG.pgdbg(1000, "pgsget: 1 record retrieved from {}:\n{}".format(tablenames, str(record)))
-      else:
-         PgLOG.pgdbg(1000, "pgsget: 0 record retrieved from " + tablenames)
-
-   os.remove(sqlfile)
-
-   return record
-
-#
-# tablenames: comma deliminated string of tables
-#     fields: fieldnames for query pgscle database,
-#  condition: querry conditions for where clause)
-# Return: mutiple records from tablenames, a dict with field names as keys and lists
-#         of retrieved values. All arrays are same size. FAILURE if not success
-#
-def pgsmget(tablenames, fields, condition = None, logact = 0):
-
-   sqlfile = pgs_sql_file(tablenames, fields, condition)
-   sqlout = PgLOG.pgsystem("psql {} < {}".format(PGPGS['PGSSERV'], sqlfile), logact, 273+1024)   # 1+16+256
-
-   rowcnt = colcnt = 0
-   records = {}
-   vals = []
-   if sqlout:
-      for line in re.split(r'\n', sqlout):
-         row = re.split(r'\s*\|\s+', line)
-         if colcnt:    # gather data
-            vals.append(row)
-            rowcnt += 1
-         else:    # gather field names
-            flds = row
-            colcnt = len(flds)
-      if rowcnt > 0:
-         records = dict(zip(flds, list(zip(*vals))))
-   elif PgLOG.PGLOG['SYSERR']:    # error happens
-      PgLOG.pglog(PgLOG.PGLOG['SYSERR'], logact|PgLOG.ERRLOG)
-
-   if PgLOG.PGLOG['DBGLEVEL']:
-      PgLOG.pgdbg(1000, "pgsmget: {} record(s) retrieved from {}".format(rowcnt, tablenames))
-   
-   os.remove(sqlfile)    # remove sqlfile when successful
-
-   return records
-
-#
-#    email: cdp user email address,
-#   userid: cdp user ID,
-# username: cdp user name
-#   Return: one record from CDP PostGreSQL database; PGLOG.FAILURE otherwise
-#
-def get_cdp_user(email, userid = 0, username = None, logact = 0):
- 
-   if userid:
-       condition = "id = {}".format(userid)
-   elif email:
-       condition = "email = '{}'".format(email)
-   elif username:
-      condition = "username = '{}'".format(username)
-   else:
-      return PgLOG.FAILURE
-
-   fields = ("id as cdpid, firstname as fstname, middlename as midinit, " +
-            "lastname as lstname, email, username as cdpname, " +
-            "organization as org_name, organization_type as org_type, country")
-   return  pgsget('users', fields, condition, logact)
-
-#
-#   name: field name
-#  value: field value
-# Return: converted value from upcases to lower case
-#
-def convert_pgs_case(name, value):
-
-   if name == "username" or name == "email":
-      return value.lower()
-   else:
-      return value    # no change