rda-python-icoads 1.0.13__tar.gz → 1.0.17__tar.gz

{rda_python_icoads-1.0.13/src/rda_python_icoads.egg-info → rda_python_icoads-1.0.17}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_icoads
-Version: 1.0.13
+Version: 1.0.17
 Summary: RDA python package to manage RDA ICOADS datasets
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-icoads

{rda_python_icoads-1.0.13 → rda_python_icoads-1.0.17}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "rda_python_icoads"
-version = "1.0.13"
+version = "1.0.17"
 authors = [
   { name="Zaihua Ji", email="zji@ucar.edu" },
 ]

{rda_python_icoads-1.0.13 → rda_python_icoads-1.0.17}/src/rda_python_icoads/PgIMMA.py

@@ -343,8 +343,8 @@ MUNIQUE = {
    'iivad'    : ['arci', 'cdi']
 }
 
-IVADSC = 'ivaddb1'
-CNTLSC = 'cntldb1'
+IVADSC = 'ivaddb'
+CNTLSC = 'cntldb'
 
 MULTI_NAMES = []
 ATTI2NAME = {}
@@ -454,7 +454,7 @@ def get_imma_records(cdate, line, records):
          aname = ATTI2NAME[line[offset:offset+2]]
       imma = IMMAS[aname]
       pgrec = get_one_attm(imma[3], offset, line)
-#48      if aname == 'iivad': pgrec['cdi'] = PgUtil.adddate('2014-01-01', 0, 0, I36(pgrec['cdi']), 'YYYMMDD')
+#48      if aname == 'iivad': pgrec['cdi'] = PgUtil.adddate('2014-01-01', 0, 0, I36(pgrec['cdi']), 'YYYYMMDD')
       if aname not in records: records[aname] = initialize_attm_records(imma[3])
       if CURIUID:
          append_one_attm(cdate, imma[0], imma[3], pgrec, records[aname])
@@ -1115,27 +1115,33 @@ def get_imma_date(line):
 def get_itidx_date(uid):
 
    global CURIUID, CURIIDX, CURTIDX
+   if CURRN3 < 0: PgLOG.pglog(f"{uid}: Provide a RN3 (>= 0) to proceed", PgLOG.LGEREX)
    uidx = uid[0:2].lower()
    suid = uid[2:6]
    table = f"{CNTLSC}.itidx_{uidx}"
-   cond = f"suid = '{suid}' AND rn3 = {CURRN3}"
-   pgrec = PgDBI.pgget(table, "*", cond, PgLOG.LGEREX)
-   if not pgrec:
-      msg = "{}-{}: suid-rn3 not in {}".format(suid, CURRN3, table)
-      if CURRN3 < 0:
-         PgLOG.pglog(msg + "\nProvide a RN3 (>= 0) to proceed", PgLOG.LGEREX)
-      else:
-         return PgLOG.pglog(msg + ", SKIP it", PgLOG.WARNLG)
-
+   cond = f"suid = '{suid}'"
+   pgrecs = PgDBI.pgmget(table, "*", cond, PgLOG.LGEREX)
+   ucnt = len(pgrecs['iidx']) if pgrecs else 0
+   if ucnt == 0: return PgLOG.pglog(f"{uid}: not in table {table}, SKIP it", PgLOG.WARNLG)
+
+   uidx = -1
+   for i in range(ucnt):
+      if pgrecs['rn3'][i] == CURRN3:
+         uidx = i
+         break
+   if uidx == -1: return PgLOG.pglog(f"{uid}: not in table {table} for rn3({CURRN3}), SKIP it", PgLOG.WARNLG)
+
+   iidx = pgrecs['iidx'][uidx]
+   tidx = pgrecs['tidx'][uidx]
    if CHKEXIST:    # check
-      table = f"{IVADSC}.{ATTMNAME}_{pgrec['tidx']}"
-      cnd = f"iidx = {pgrec['iidx']}"
+      table = f"{IVADSC}.{ATTMNAME}_{tidx}"
+      cnd = f"iidx = {iidx}"
       if PgDBI.pgget(table, "", cnd): return None
 
    CURIUID = uid
-   CURIIDX = pgrec['iidx']
-   CURTIDX = pgrec['tidx']
-   return pgrec['date']
+   CURIIDX = iidx
+   CURTIDX = tidx
+   return pgrecs['date'][uidx]
 
 #
 # get record date for given year, month and day

{rda_python_icoads-1.0.13 → rda_python_icoads-1.0.17}/src/rda_python_icoads/cleanicoads.py

@@ -107,9 +107,10 @@ def clean_imma_data():
       else:
          clean_imma_data_for_tidx(tidx, cnd)
 
-   cnt = PgDBI.pgdel(table, PVALS['dcnd'], PgLOG.LGEREX)
-   s = 's' if cnt > 1 else ''
-   PgLOG.pglog("{}: {} record{} deleted for {}".format(table, cnt, s, PVALS['dcnd']), PgLOG.LOGWRN)
+   if not PVALS['aname']:
+      cnt = PgDBI.pgdel(table, PVALS['dcnd'], PgLOG.LGEREX)
+      s = 's' if cnt > 1 else ''
+      PgLOG.pglog("{}: {} record{} deleted for {}".format(table, cnt, s, PVALS['dcnd']), PgLOG.LOGWRN)
 
 #
 # clean up imma data for table index

{rda_python_icoads-1.0.13 → rda_python_icoads-1.0.17}/src/rda_python_icoads/fillicoads.py

@@ -19,12 +19,15 @@ import re
 from os import path as op
 from rda_python_common import PgLOG
 from rda_python_common import PgDBI
+from rda_python_common import PgUtil
 from . import PgIMMA
 
 PVALS = {
    'uatti' : '',
    'names' : None,
-   'files' : []
+   'files' : [],
+   'dates' : [],
+   'dtlen' : 0
 }
 
 #
@@ -38,25 +41,29 @@ def main():
 
    option = None   
    for arg in argv:
-      if arg == "-b":
-         PgLOG.PGLOG['BCKGRND'] = 1
-      elif arg == "-a":
-         PVALS['uatti'] = "98"
-      elif arg == "-u":
-         leaduid = 1
-      elif arg == "-e":
-         chkexist = 1
-      elif arg == "-f":
-         option = 'f'
-      elif arg == "-r":
-         option = 'r'
-      elif arg == "-i":
-         addinventory = 1
-      elif re.match(r'^-', arg):
-         PgLOG.pglog(arg + ": Invalid Option", PgLOG.LGWNEX)
+      if re.match(r'-\w', arg):
+         option = None
+         if arg[1] == "b":
+            PgLOG.PGLOG['BCKGRND'] = 1
+         elif arg[1] == "a":
+            PVALS['uatti'] = "98"
+         elif arg[1] == "u":
+            leaduid = 1
+         elif arg[1] == "e":
+            chkexist = 1
+         elif arg[1] == "i":
+            addinventory = 1
+         elif arg[1] in "fpr":
+            option = arg[1]
+         else:
+            PgLOG.pglog(arg + ": Invalid Option", PgLOG.LGWNEX)
       elif option == 'f':
          get_imma_filelist(arg)
          option = None
+      elif option == 'p':
+         PVALS['dates'].append(PgUtil.format_date(arg))
+         PVALS['dtlen'] += 1
+         if PVALS['dtlen'] == 2: option = None
       elif option == 'r':
          rn3 = int(arg)
          option = None
@@ -64,11 +71,12 @@ def main():
          PVALS['files'].append(arg)
 
    if not PVALS['files']:
-      print("Usage: fillicoads [-a] [-e] [-f InputFile] [-i] [-r RN3] [-u] FileList")
+      print("Usage: fillicoads [-a] [-e] [-f InputFile] [-i] [-p BDate [EDate]] [-r RN3] [-u] FileList")
       print("   At least one file name needs to fill icoads data into Postgres Server")
       print("   Option -a: add all attms, including multi-line ones, such as IVAD and REANQC")
       print("   Option -f: provide a filename holding a list of IMMA1 files")
       print("   Option -i: add daily counting records into inventory table")
+      print("   Option -p: provide a period for filling data")
       print("   Option -r: the Third digit of IMMA release number")
       print("   Option -u: standalone attachment records with leading 6-character UID")
       print("   Option -e: check existing record before adding attm")
@@ -121,10 +129,10 @@ def process_imma_file(fname, addinventory):
 
    # get the first valid date and do initialization
    line = IMMA.readline()
+   PgIMMA.identify_attm_name(line)  # check and record standalone attm name
    while line:
-      PgIMMA.identify_attm_name(line)  # check and record standalone attm name
       idate = cdate = PgIMMA.get_imma_date(line)
-      if cdate:
+      if cdate and (PVALS['dtlen'] == 0 or PgUtil.diffdate(cdate, PVALS['dates'][0]) >= 0):
          PgIMMA.init_indices_for_date(cdate, iname)
          records = PgIMMA.get_imma_records(cdate, line, records)
          break
@@ -142,6 +150,7 @@ def process_imma_file(fname, addinventory):
                for i in range(PgIMMA.TABLECOUNT): acounts[i] += acnts[i]
                records = {}
                cdate = idate
+               if PVALS['dtlen'] == 2 and PgUtil.diffdate(cdate, PVALS['dates'][1]) > 0: break
                PgIMMA.init_indices_for_date(cdate, iname)
             records = PgIMMA.get_imma_records(idate, line, records)
       line = IMMA.readline()

{rda_python_icoads-1.0.13 → rda_python_icoads-1.0.17}/src/rda_python_icoads/fillitable.py

@@ -143,6 +143,7 @@ def add_field_records():
                acnt += 1
             elif stat == 2:
                ucnt += 1
+         line = IVAR.readline()
 
       IVAR.close()
       PgLOG.pglog("{}/{} of {} values added/updated into table {}".format(acnt, ucnt, vcnt, vtable), PgLOG.LOGWRN)
@@ -152,7 +153,7 @@ def add_field_records():
 #
 def add_field_value(var, vtable, key, desc):
 
-   cnd = "{} = {}".foramt(var, key)
+   cnd = "{} = {}".format(var, key)
 
    pgrec = PgDBI.pgget(vtable, "*", cnd)
 

{rda_python_icoads-1.0.13 → rda_python_icoads-1.0.17}/src/rda_python_icoads/imma1_subset.py

@@ -607,9 +607,9 @@ def join_attm_fields(aname, record):
 def subset_table_index(fname, tidx, bdate, edate):
 
    atables = {}
-   PgDBI.set_scname(dbname = 'ivaddb', scname = PgIMMA.IVADSC, lnname = 'ivaddb', dbhost = PgLOG.PGLOG['PMISCHOST'])
+   PgDBI.set_scname(dbname = 'ivaddb', scname = 'ivaddb', lnname = 'ivaddb', dbhost = PgLOG.PGLOG['PMISCHOST'])
 
-   tname = f"{PgIMMA.CTNLSC}.iattm"
+   tname = "cntldb.iattm"
    for aname in PVALS['anames']:
       cnd = f"tidx = {tidx} AND attm = '{aname}'"
       atables[aname] = PgDBI.pgget(tname, "", cnd, PgLOG.LGEREX)

{rda_python_icoads-1.0.13 → rda_python_icoads-1.0.17/src/rda_python_icoads.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_icoads
-Version: 1.0.13
+Version: 1.0.17
 Summary: RDA python package to manage RDA ICOADS datasets
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-icoads