rda-python-common 3.0.9__tar.gz → 3.0.11__tar.gz

{rda_python_common-3.0.9/src/rda_python_common.egg-info → rda_python_common-3.0.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_common
-Version: 3.0.9
+Version: 3.0.11
 Summary: RDA Python common library codes shared by other RDA python packages
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-common

{rda_python_common-3.0.9 → rda_python_common-3.0.11}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "rda_python_common"
-version = "3.0.9"
+version = "3.0.11"
 authors = [
   { name="Zaihua Ji", email="zji@ucar.edu" },
 ]

{rda_python_common-3.0.9 → rda_python_common-3.0.11}/src/rda_python_common/pg_dbi.py

@@ -85,7 +85,7 @@ class PgDBI(PgLOG):
       curtran (int): Transaction counter: 0 = idle, >0 = inside a transaction.
       NMISSES (list): Cached list of scientist IDs (userno) not found in the DB.
       LMISSES (list): Cached list of login names not found in the DB.
-      USRWARN (int): Set to 1 once the 'run filluser' reminder has been logged.
+      USRWARN (int): Set to 1 once the incomplete-user reminder has been logged.
       TABLES (dict): Cache of table-field default info keyed by table name.
       SEQUENCES (dict): Cache of sequence field names keyed by table name.
       SPECIALIST (dict): Cache of specialist records keyed by dataset ID.
@@ -1702,19 +1702,17 @@ class PgDBI(PgLOG):
       return (self.SUCCESS if ret else self.FAILURE)
 
    def dssgrp_user_info(self, userno, logname = None):
-      """Build a partial dssdb.user record from dssdb.dssgrp.
+      """Build a dssdb.user record from dssdb.dssgrp.
 
-      Fallback for when the UCAR People API is unreachable, so a usable user record
-      can still be added for a DECS group member. Fields absent from dssgrp are set
-      to the values implied by DECS membership; the record is marked stat_flag 'M'
-      so that 'filluser -i' completes it once the People API is available again.
+      dssdb.dssgrp is the only source of specialist information; fields absent from
+      it are set to the values implied by DECS group membership.
 
       Args:
          userno (int): Scientist number; pass 0 to look up by logname instead.
          logname (str | None): UCAR login name; used when userno is 0.
 
       Returns:
-         dict | None: Partial mapping of column_name → value suitable for
+         dict | None: Mapping of column_name → value suitable for
                       pgadd('dssdb.user'), or None when not in dssgrp.
       """
       cond = "userno = {}".format(userno) if userno else "logname = '{}'".format(logname)
@@ -1726,15 +1724,16 @@ class PgDBI(PgLOG):
       pgrec['org_name'] = 'NCAR'
       pgrec['country'] = 'UNITED.STATES'
       pgrec['email'] = pgrec['ucaremail'] = email
-      pgrec['stat_flag'] = 'M'
+      pgrec['stat_flag'] = 'A'
       return pgrec
 
    def add_missed_user(self, userno, logname = None):
       """Add a dssdb.user record for a user missing from the table.
 
-      Tries the UCAR People API first, then dssgrp, and finally falls back to a bare
-      stub. A reminder to run filluser is logged once per process whenever the added
-      record is incomplete (stat_flag 'M').
+      Builds the record from dssdb.dssgrp for a DECS group member. For anyone not
+      in dssgrp, adds a record filled with whatever can be derived from the login
+      name, using 'UNKNOWN' for the fields that cannot be filled in; such a record
+      is marked stat_flag 'M' and logs a reminder once per process.
 
       Args:
          userno (int): Scientist number; pass 0 to add by logname instead.
@@ -1743,46 +1742,41 @@ class PgDBI(PgLOG):
       Returns:
          int: The new user.uid, or 0 if the insert fails.
       """
-      pgrec = self.ucar_user_info(userno, logname)
-      if not pgrec: pgrec = self.dssgrp_user_info(userno, logname)
+      pgrec = self.dssgrp_user_info(userno, logname)
       if not pgrec:
-         pgrec = {'userno': userno} if userno else {'logname': logname}
-         pgrec['stat_flag'] = 'M'
-      if pgrec['stat_flag'] == 'M': self.incomplete_user_warning()
+         pgrec = {'stat_flag': 'M', 'lstname': "UNKNOWN", 'fstname': "UNKNOWN"}
+         if userno: pgrec['userno'] = userno
+         if logname:
+            pgrec['logname'] = logname
+            pgrec['email'] = pgrec['ucaremail'] = logname + '@ucar.edu'
+            pgrec['org_type'] = 'NCAR'   # a UCAR login name outside of the DECS group
+            pgrec['org_name'] = 'NCAR'
+            pgrec['country'] = 'UNITED.STATES'
+         else:
+            pgrec['org_type'] = "OTHER"
+            pgrec['org_name'] = pgrec['country'] = "UNKNOWN"
+         self.incomplete_user_warning(userno, logname)
       return self.pgadd("dssdb.user", pgrec, (self.PGDBI['EXITLG']|self.AUTOID))
 
-   def incomplete_user_warning(self):
-      """Log a one-time reminder to run filluser for incomplete dssdb.user records."""
-      if self.USRWARN: return
-      self.USRWARN = 1
-      self.pglog("Incomplete record in dssdb.user; run 'filluser -i' later to complete it", self.LGWNEM)
-
-   def reopen_user_record(self, pgrec):
-      """Reactivate a closed dssdb.user record and return its uid.
-
-      Clears until_date and marks the record incomplete (stat_flag 'M') so that
-      'filluser -i' refreshes it. Records that are not closed are returned as is.
-      Only call this for a present-day lookup; reopening on a historical lookup
-      would wrongly reactivate a user who has since left.
+   def incomplete_user_warning(self, userno, logname = None):
+      """Log a one-time reminder that an incomplete dssdb.user record was added.
 
       Args:
-         pgrec (dict): Record holding at least 'uid' and 'until_date'.
-
-      Returns:
-         int: The uid of the record.
+         userno (int): Scientist number of the added user; 0 if unknown.
+         logname (str | None): UCAR login name of the added user; None if unknown.
       """
-      if pgrec['until_date']:
-         record = {'until_date': None, 'stat_flag': 'M'}
-         self.pgupdt("dssdb.user", record, "uid = {}".format(pgrec['uid']), self.PGDBI['ERRLOG'])
-         self.pglog("Closed user.uid = {} is reopened".format(pgrec['uid']), self.LGWNEM)
-      return pgrec['uid']
+      if self.USRWARN: return
+      self.USRWARN = 1
+      who = logname if logname else userno
+      self.pglog("{}: Not in dssdb.dssgrp; incomplete record added to dssdb.user".format(who), self.LGWNEM)
 
    def check_user_uid(self, userno, date = None):
       """Return the user.uid for a scientist ID, adding a record if missing.
 
       Looks up the active user record for userno on the given date. If not found,
-      logs a warning, attempts a date-range-independent lookup, and finally fetches
-      UCAR person info to insert a new user record.
+      logs a warning and retries without the date range, returning the uid of a
+      closed record as is; a closed record is never reactivated. Only when no
+      record exists at all is a new one added from dssdb.dssgrp.
 
       Args:
          userno (int | str): UCAR scientist number.
@@ -1794,7 +1788,6 @@ class PgDBI(PgLOG):
       """
       if not userno: return 0
       if type(userno) is str: userno = int(userno)
-      istoday = (date is None)
       if date is None:
          datecond = "until_date IS NULL"
          date = 'today'
@@ -1806,8 +1799,8 @@ class PgDBI(PgLOG):
          self.pglog("{}: Scientist ID NOT on file for {}".format(userno, date), self.LGWNEM)
          self.NMISSES.append(userno)
       # check again if a user is on file with different date range
-      pgrec = self.pgget("dssdb.user", "uid, until_date", "userno = {}".format(userno), self.PGDBI['ERRLOG'])
-      if pgrec: return self.reopen_user_record(pgrec) if istoday else pgrec['uid']
+      pgrec = self.pgget("dssdb.user", "uid", "userno = {}".format(userno), self.PGDBI['ERRLOG'])
+      if pgrec: return pgrec['uid']
       uid = self.add_missed_user(userno)
       if uid: self.pglog("{}: Scientist ID Added as user.uid = {}".format(userno, uid), self.LGWNEM)
       return uid
@@ -1815,7 +1808,8 @@ class PgDBI(PgLOG):
    def get_user_uid(self, logname, date = None):
       """Return the user.uid for a UCAR login name, adding a record if missing.
 
-      Similar to check_user_uid() but looks up by logname instead of userno.
+      Similar to check_user_uid() but looks up by logname instead of userno; a
+      closed record is likewise returned as is and never reactivated.
 
       Args:
          logname (str): UCAR login name.
@@ -1825,7 +1819,6 @@ class PgDBI(PgLOG):
          int: user.uid on success, 0 if logname is falsy or insert fails.
       """
       if not logname: return 0
-      istoday = (not date)
       if not date:
          date = 'today'
          datecond = "until_date IS NULL"
@@ -1837,92 +1830,12 @@ class PgDBI(PgLOG):
          self.pglog("{}: UCAR Login Name NOT on file for {}".format(logname, date), self.LGWNEM)
          self.LMISSES.append(logname)
       # check again if a user is on file with different date range
-      pgrec = self.pgget("dssdb.user", "uid, until_date", "logname = '{}'".format(logname), self.PGDBI['ERRLOG'])
-      if pgrec: return self.reopen_user_record(pgrec) if istoday else pgrec['uid']
+      pgrec = self.pgget("dssdb.user", "uid", "logname = '{}'".format(logname), self.PGDBI['ERRLOG'])
+      if pgrec: return pgrec['uid']
       uid = self.add_missed_user(0, logname)
       if uid: self.pglog("{}: UCAR Login Name Added as user.uid = {}".format(logname, uid), self.LGWNEM)
       return uid
 
-   def ucar_user_info(self, userno, logname = None):
-      """Fetch UCAR person info for a scientist ID or login name via pgperson/pgusername.
-
-      Runs the pgperson command-line tool and parses its key<=>value output.
-      Maps UCAR API fields to database column names, normalises country code,
-      organisation type, and employment dates.
-
-      Args:
-         userno (int): Scientist number; pass 0 to look up by logname instead.
-         logname (str | None): UCAR login name; used when userno is 0.
-
-      Returns:
-         dict | None: Mapping of column_name → value suitable for pgadd('dssdb.user'),
-                      or None when pgperson returns no output.
-      """
-      matches = {
-         'upid': "upid",
-         'uid': "userno",
-         'username': "logname",
-         'lastName': "lstname",
-         'firstName': "fstname",
-         'active': "stat_flag",
-         'internalOrg': "division",
-         'externalOrg': "org_name",
-         'country': "country",
-         'forwardEmail': "email",
-         'email': "ucaremail",
-         'phone': "phoneno"
-      }
-      # pgperson/pgusername are installed by rda_python_metrics; skip quietly when the
-      # commands are not on this host so that callers can fall back to dssgrp
-      if not self.valid_command("pgperson"): return None
-      buf = self.pgsystem("pgperson " + ("-uid {}".format(userno) if userno else "-username {}".format(logname)), self.LOGWRN, 20)
-      if not buf: return None
-      pgrec = {}
-      for line in buf.split('\n'):
-         ms = re.match(r'^(.+)<=>(.*)$', line)
-         if ms:
-            (key, val) = ms.groups()
-            if key in matches:
-               if key == 'upid' and 'upid' in pgrec: break  # get one record only
-               pgrec[matches[key]] = val   
-      if not pgrec: return None
-      # the People API omits keys it has no value for, so read them defensively
-      if userno:
-         pgrec['userno'] = userno
-      elif pgrec.get('userno'):
-         pgrec['userno'] = userno = int(pgrec['userno'])
-      if pgrec.get('upid'): pgrec['upid'] = int(pgrec['upid'])
-      if pgrec.get('stat_flag'): pgrec['stat_flag'] = 'A' if pgrec['stat_flag'] == "True" else 'C'
-      email = pgrec.get('email')
-      if email and re.search(r'(@|\.)ucar\.edu$', email, re.I):
-         email = pgrec['email'] = pgrec.get('ucaremail')
-         pgrec['org_name'] = 'NCAR'
-      country = pgrec.get('country')
-      if email or country: pgrec['country'] = self.set_country_code(email, country)
-      val = "NCAR" if pgrec.get('division') else None
-      pgrec['org_type'] = self.get_org_type(val, email)
-      logname = pgrec.get('logname')
-      if not logname or not self.valid_command("pgusername"): return pgrec
-      buf = self.pgsystem("pgusername {}".format(logname), self.LOGWRN, 20)
-      if not buf: return pgrec
-      for line in buf.split('\n'):
-         ms = re.match(r'^(.+)<=>(.*)$', line)
-         if ms:
-            (key, val) = ms.groups()
-            if key == 'startDate':
-               m = re.match(r'^(\d+-\d+-\d+)\s', val)
-               if m:
-                  pgrec['start_date'] = m.group(1)
-               else:
-                  pgrec['start_date'] = val
-            if key == 'endDate':
-               m = re.match(r'^(\d+-\d+-\d+)\s', val)
-               if m:
-                  pgrec['until_date'] = m.group(1)
-               else:
-                  pgrec['until_date'] = val
-      return pgrec
-
    def set_country_code(self, email, country = None):
       """Normalise a country name or derive it from an email domain.
 

{rda_python_common-3.0.9 → rda_python_common-3.0.11}/src/rda_python_common/pg_file.py

@@ -233,9 +233,9 @@ class PgFile(PgUtil, PgSIG):
          if self.pgcmp(thost, self.OHOST, 1) == 0:
             return self.local_copy_object(tofile, fromfile, None, None, logact)
          elif self.pgcmp(thost, self.BHOST, 1) == 0:
-            return self.local_copy_backup(tofile, fromfile, self.QPOINTS['B'], logact)
+            return self.wait_copy_backup(tofile, fromfile, self.QPOINTS['B'], logact)
          elif self.pgcmp(thost, self.DHOST, 1) == 0:
-            return self.local_copy_backup(tofile, fromfile, self.QPOINTS['D'], logact)
+            return self.wait_copy_backup(tofile, fromfile, self.QPOINTS['D'], logact)
          else:
             return self.local_copy_remote(tofile, fromfile, tohost, logact)
       elif self.pgcmp(thost, self.LHOST, 1) == 0:
@@ -664,6 +664,25 @@ class PgFile(PgUtil, PgSIG):
       if not endpoint: endpoint = self.PGLOG['BACKUPEP']
       return self.endpoint_copy_endpoint(tofile, fromfile, endpoint, 'gdex-glade', logact)
 
+   # Copy a local file to Quasar backup tape system and wait until the Globus task is done
+   # for callers that cannot handle a self.FINISH return
+   def wait_copy_backup(self, tofile, fromfile, endpoint = None, logact = 0):
+      """Copy a local GLADE file to the Quasar backup endpoint and wait for completion.
+
+      Args:
+         tofile (str): Destination path on the backup endpoint (leading '/dsNNN.N/').
+         fromfile (str): Source local path (leading '/data/' or '/decsdata/').
+         endpoint (str | None): Globus endpoint name; defaults to PGLOG['BACKUPEP'].
+         logact (int): Logging action flags; default 0.
+
+      Returns:
+         int: self.SUCCESS or self.FAILURE.
+      """
+      if not endpoint: endpoint = self.PGLOG['BACKUPEP']
+      ret = self.local_copy_backup(tofile, fromfile, endpoint, logact)
+      if ret == self.FINISH: ret = self.check_globus_finished(tofile, endpoint, logact)
+      return ret
+
    # Copy a  Quasar backup file to local Globus endpoint
    #   tofile - target file name, leading with /data/ or /decsdata/
    # fromfile - source file name, leading with /dsnnn.n/
@@ -671,6 +690,9 @@ class PgFile(PgUtil, PgSIG):
    def backup_copy_local(self, tofile, fromfile, endpoint = None, logact = 0):
       """Copy a file from the Quasar backup endpoint to a local GLADE path via Globus.
 
+      Waits for the Globus task to finish before returning, so the local file is
+      complete for the caller.
+
       Args:
          tofile (str): Destination local path (leading '/data/' or '/decsdata/').
          fromfile (str): Source backup path (leading '/dsNNN.N/').
@@ -678,10 +700,12 @@ class PgFile(PgUtil, PgSIG):
          logact (int): Logging action flags; default 0.
 
       Returns:
-         int: self.SUCCESS, self.FAILURE, or self.FINISH.
+         int: self.SUCCESS or self.FAILURE.
       """
       if not endpoint: endpoint = self.PGLOG['BACKUPEP']
-      return self.endpoint_copy_endpoint(tofile, fromfile, 'gdex-glade', endpoint, logact)
+      ret = self.endpoint_copy_endpoint(tofile, fromfile, 'gdex-glade', endpoint, logact)
+      if ret == self.FINISH: ret = self.check_globus_finished(tofile, 'gdex-glade', logact)
+      return ret
 
    # Copy a remote file to local
    #   tofile - target file name

{rda_python_common-3.0.9 → rda_python_common-3.0.11/src/rda_python_common.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_common
-Version: 3.0.9
+Version: 3.0.11
 Summary: RDA Python common library codes shared by other RDA python packages
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-common