rda-python-icoads 1.0.7__tar.gz → 1.0.9__tar.gz

rda_python_icoads-1.0.9/MANIFEST.in

@@ -0,0 +1,6 @@
+include src/rda_python_icoads/README_R3.0_Subset.html
+include src/rda_python_icoads/rdimma1_csv.f
+include src/rda_python_icoads/msg
+include src/rda_python_icoads/R3.0-stat_doc.pdf
+include src/rda_python_icoads/msg3.0_subset_readme.txt
+include src/rda_python_icoads/msgsubset.f

{rda_python_icoads-1.0.7 → rda_python_icoads-1.0.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rda_python_icoads
-Version: 1.0.7
+Version: 1.0.9
 Summary: RDA python package to manage RDA ICOADS datasets
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-icoads

{rda_python_icoads-1.0.7 → rda_python_icoads-1.0.9}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "rda_python_icoads"
-version = "1.0.7"
+version = "1.0.9"
 authors = [
   { name="Zaihua Ji", email="zji@ucar.edu" },
 ]
@@ -30,10 +30,27 @@ include-package-data = true
 where = ["src"]
 
 [tool.setuptools.package-data]
-"rda_python_icoads" = ["README_R3.0_Subset.html", "rdimma1_csv.f"]
+"rda_python_icoads" = [
+  "README_R3.0_Subset.html",
+  "rdimma1_csv.f",
+  "msg",
+  "R3.0-stat_doc.pdf",
+  "msg3.0_subset_readme.txt",
+  "msgsubset.f",
+]
 
 [project.urls]
 "Homepage" = "https://github.com/NCAR/rda-python-icoads"
 
 [project.scripts]
-imma1_subset = "rda_python_icoads.imma1_subset:main"
+"imma1_subset" = "rda_python_icoads.imma1_subset:main"
+"msg3_subset" = "rda_python_icoads.msg3_subset:main"
+"checkicoads" = "rda_python_icoads.checkicoads:main"
+"cleanicoads" = "rda_python_icoads.cleanicoads:main"
+"counticoads" = "rda_python_icoads.counticoads:main"
+"fillicoads" = "rda_python_icoads.fillicoads:main"
+"fillinventory" = "rda_python_icoads.fillinventory:main"
+"fillitable" = "rda_python_icoads.fillitable:main"
+"fillmonth" = "rda_python_icoads.fillmonth:main"
+"msg_download" = "rda_python_icoads.msg_download:main"
+"writeicoads" = "rda_python_icoads.writeicoads:main"

rda_python_icoads-1.0.9/src/rda_python_icoads/checkicoads.py

@@ -0,0 +1,222 @@
+#!/usr/bin/env python3
+#
+##################################################################################
+#
+#     Title : checkicoads
+#    Author : Zaihua Ji, zji@ucar.edu
+#      Date : 12/30/2020
+#             2025-03-03 transferred to package rda_python_icoads from
+#             https://github.com/NCAR/rda-icoads.git
+#   Purpose : check and compare ICOADS data files and IVADDB records
+#
+#    Github : https://github.com/NCAR/rda-python-icoads.git
+#
+##################################################################################
+
+import sys
+import os
+import re
+from os import path as op
+from rda_python_common import PgLOG
+from rda_python_common import PgDBI
+from rda_python_common import PgUtil
+from rda_python_common import PgOPT
+from rda_python_common import PgSIG
+from . import PgIMMA
+
+PVALS = {
+   'bdate' : None,
+   'edate' : None,
+   'bmdate' : [],
+   'emdate' : [],
+   'fname' : [],
+   'flag' : [],   # 1 - file exists, 2 - db records exist, 3 - both
+   'mproc' : 10,
+   'fpattern' : "IMMA1_R3.0.0_<YYYY-MM>",
+   'readall' : 0
+}
+
+#
+# main function to run dsarch
+#
+def main():
+
+   option = ''
+   argv = sys.argv[1:]
+   
+   for arg in argv:
+      if arg == "-b":
+         PgLOG.PGLOG['BCKGRND'] = 1
+      elif arg == "-a":
+         PVALS['readall'] = 1
+      elif arg == "-f":
+         option = 'f'
+      elif arg == "-m":
+         option = 'm'
+      elif re.match(r'^-', arg):
+         PgLOG.pglog(arg + ": Invalid Option", PgLOG.LGWNEX)
+      elif option:
+         if option == 'f':
+            PVALS['fpattern'] = arg
+         elif option == 'm':
+            PVALS['mproc'] = arg
+         option = ''
+      elif not PVALS['bdate']:
+         PVALS['bdate'] = arg
+      elif not PVALS['edate']:
+         PVALS['edate'] = arg
+      else:
+         PgLOG.pglog(arg + ": Invalid parameter", PgLOG.LGWNEX)
+   
+   PgLOG.PGLOG['LOGFILE'] = "icoads.log"
+   PgDBI.ivaddb_dbname()
+   
+   if not (PVALS['bdate'] and PVALS['edate']):
+      pgrec = PgDBI.pgget("cntldb.inventory", "min(date) bdate, max(date) edate", '', PgLOG.LGEREX)
+      print("Usage: checkicoads [-a] [-m mproc] [-f FilePattern] BeginDate EndDate")
+      print("   Default FilePattern is " + PVALS['fpattern'])
+      print("   Option -a - read all attms, including multi-line ones, such as IVAD and REANQC")
+      print("   Option -m - start up to given number of processes, one for each month (Default to 10)")
+      print("   Set BeginDate and EndDate between '{}' and '{}'".format(pgrec['bdate'], pgrec['edate']))
+      sys.exit(0)
+
+   if PgUtil.diffdate(PVALS['bdate'], PVALS['edate']) > 0:
+      tmpdate = PVALS['bdate']
+      PVALS['bdate'] = PVALS['edate']
+      PVALS['edate'] = tmpdate
+   
+   PgLOG.cmdlog("checkicoads {}".format(' '.join(argv)))
+   check_imma_data()
+   PgLOG.cmdlog()
+   sys.exit(0)
+
+#
+# check imma data
+#
+def check_imma_data():
+
+   mcnt = init_months()
+   if mcnt == 1: PVALS['mproc'] = 1
+   if PVALS['mproc'] > 1:
+      PgSIG.start_none_daemon('writeicoads', '', PgLOG.PGLOG['CURUID'], PVALS['mproc'], 300, 1)
+
+   for midx in range(mcnt):
+      fname = PVALS['fname'][midx]
+      if op.isfile(fname +".cnt"): continue    # monthly file counted already
+      if PVALS['mproc'] > 1:
+         stat = PgSIG.start_child("checkicoads_{}".format(midx), PgLOG.LOGWRN, 1)  # try to start a child process
+         if stat <= 0:
+            sys.exit(1)   # something wrong
+         elif PgSIG.PGSIG['PPID'] > 1:
+            check_imma_file(fname, midx)
+            sys.exit(0)  # stop child process
+         else:
+            PgDBI.pgdisconnect(0)  # disconnect database for reconnection
+            continue   # continue for next midx
+      else:
+         check_imma_file(fname, midx)
+
+   if PVALS['mproc'] > 1: PgSIG.check_child(None, 0, PgLOG.LOGWRN, 1)
+
+   dump_final_counts()
+
+#
+# compare icoads records from given file name and IVADDB
+#
+def check_imma_file(fname, midx):
+
+   PgLOG.pglog("Count IMMA records in File '{}'".format(fname), PgLOG.WARNLG)
+   flag = PVALS['flag'][midx]
+
+   acnts = [0]*PgIMMA.TABLECOUNT
+   acounts = [0]*PgIMMA.TABLECOUNT
+
+   if flag&1:
+      IMMA = open(fname, 'r')
+      line = IMMA.readline()   
+      while line:
+         if PVALS['readall'] and re.match(r'^98', line):
+             PgIMMA.get_imma_multiple_counts(line, acnts)
+         else:
+            PgIMMA.get_imma_counts(line, acnts)
+         line.IMMA.readline()
+      IMMA.close()
+      for i in range(PgIMMA.TABLECOUNT): acounts[i] = acnts[i]
+
+   if flag&2:
+      PgLOG.pglog("Count IMMA records in in IVADDB", PgLOG.WARNLG)
+      cdate = bdate = PVALS['bmdate'][midx]
+      edate = PVALS['emdate'][midx]
+      while cdate <= edate:
+         acnts = PgIMMA.count_imma_records(cdate, 0, PVALS['readall'])
+         cdate = PgUtil.adddate(cdate, 0, 0, 1)
+         if acnts:
+            for i in range(PgIMMA.TABLECOUNT): acounts[i] -= acnts[i]
+
+   dump_monthly_counts(fname, acounts)
+
+#
+# dump monthly counts
+#
+def dump_monthly_counts(fname, acounts):
+
+   oname = fname + ".cnt"
+   IMMA = open(oname, 'w')
+   IMMA.write("{}, {}\n".format(fname, ', '.join(acounts)))
+   IMMA.close()
+
+#
+# concat all monthly counts into one
+#
+def dump_final_counts():
+
+   fname = "ICOADS_DIFF_COUNTS.csv"
+
+   IMMA = open(fname, 'w')
+   IMMA.write("FileName, {}\n".format(', '.join(PgIMMA.IMMA_NAMES)))
+   IMMA.close()
+   PgLOG.pgsystem("cat *.cnt >> " + fname)
+
+#
+# initialize the month list
+#
+def init_months():
+
+   seps = ["<" , ">"];  # temporal pattern delimiters
+   match = "[^" + seps[1] + "]+"
+
+   ms = re.search(r'{}({}){}'.format(seps[0], match, seps[1]), PVALS['fpattern'])
+   if ms:
+      tpattern = ms.group(1)
+      treplace = "{}{}{}".format(seps[0], tpattern, seps[1])
+   else:
+      PgLOG.pglog(PVALS['fpattern'] + ": Not temporal pattern found to get month list", PgLOG.LGEREX)
+
+   bdate = PVALS['bdate']
+   done = midx = 0
+   while True:
+      edate = PgUtil.enddate(bdate, 0, 'M')
+      if PgUtil.diffdate(PVALS['edate'], edate) <= 0:
+         edate = PVALS['edate']
+         done = 1
+      mdate = PgUtil.format_date(bdate, tpattern)
+      fname = PVALS['fpattern'].replace(treplace, mdate)
+      flag = 0
+      if op.isfile(fname): flag += 1
+      if PgDBI.pgget("cntldb.inventory", "", "date BETWEEN '{}' AND '{}'".format(bdate, edate), PgLOG.LGEREX):
+         flag += 2
+      if flag:
+         PVALS['bmdate'].append(bdate)
+         PVALS['emdate'].append(edate)
+         PVALS['fname'].append(fname)
+         PVALS['flag'].append(flag)
+         midx += 1
+      if done: break
+      bdate = PgUtil.adddate(edate, 0, 0, 1)
+
+   return midx
+
+#
+# call main() to start program
+#
+if __name__ == "__main__": main()

rda_python_icoads-1.0.9/src/rda_python_icoads/cleanicoads.py

@@ -0,0 +1,175 @@
+#!/usr/bin/env python3
+#
+##################################################################################
+#
+#     Title : cleanicoads
+#    Author : Zaihua Ji, zji@ucar.edu
+#      Date : 12/30/2020
+#             2025-03-03 transferred to package rda_python_icoads from
+#             https://github.com/NCAR/rda-icoads.git
+#   Purpose : clean up one or all IMMA1 attms in IVADDB for given period
+#
+#    Github : https://github.com/NCAR/rda-python-icoads.git
+#
+##################################################################################
+
+import sys
+import os
+import re
+from os import path as op
+from rda_python_common import PgLOG
+from rda_python_common import PgDBI
+from rda_python_common import PgUtil
+from . import PgIMMA
+
+PVALS = {
+   'bdate' : None,
+   'edate' : None,
+   'aname' : None,
+   'tinfo' : {},
+   'tcnt' : 0,
+   'dcnd' : None,
+   'uatti' : '',
+   'names' : None
+}
+
+#
+# main function to run dsarch
+#
+def main():
+
+   option = ''
+   files = []
+   leaduid = 0
+   chkexist = 0
+   readall = 0
+   argv = sys.argv[1:]
+
+   for arg in argv:
+      if arg == "-b":
+         PgLOG.PGLOG['BCKGRND'] = 1
+      elif arg == "-a":
+         option = 'a'
+      elif re.match(r'^-', arg):
+         PgLOG.pglog(arg + ": Invalid Option", PgLOG.LGWNEX)
+      elif option:
+         PVALS['aname'] = arg
+         option = ''
+      elif not PVALS['bdate']:
+         PVALS['bdate'] = arg
+      elif not PVALS['edate']:
+         PVALS['edate'] = arg
+      else:
+         PgLOG.pglog(arg + ": Invalid parameter", PgLOG.LGWNEX)
+
+   if not PVALS['bdate']:
+      print("Usage: cleanicoads [-a ATTNAME] BDATE EDATE")
+      print("   Option -a - clean a single attm for given attm name")
+      PgLOG.pgexit()
+
+   PgLOG.PGLOG['LOGFILE'] = "icoads.log"
+   PgDBI.ivaddb_dbname()
+   PgLOG.cmdlog("cleanicoads {}".format(' '.join(argv)))
+   set_table_info()
+   clean_imma_data()   
+   PgLOG.cmdlog()
+   PgLOG.pgexit()
+
+#
+# set the table index list
+#
+def set_table_info():
+
+   table = "cntldb.inventory"
+   if PVALS['edate']:
+      PVALS['dcnd'] = "date BETWEEN '{}' AND '{}'".format(PVALS['bdate'], PVALS['edate'])
+   else:
+      PVALS['dcnd'] = "date >= '{}'".format(PVALS['bdate'])
+
+   PVALS['tinfo'] = PgDBI.pgmget(table, "tidx, min(miniidx) bidx, max(maxiidx) eidx", PVALS['dcnd'] + " GROUP BY tidx", PgLOG.LGEREX)
+   PVALS['tcnt'] = len(PVALS['tinfo']['tidx']) if PVALS['tinfo'] else 0
+
+   if not PVALS['tcnt']:
+      PgLOG.pglog("{}: No data found in IVADDB for {}".format(table, PVALS['dcnd']), PgLOG.LGEREX)   
+
+#
+# clean up imma data
+#
+def clean_imma_data():
+
+   table = "cntldb.inventory"
+
+   for i in range(PVALS['tcnt']):
+      tidx = PVALS['tinfo']['tidx'][i]
+      cnd = "iidx BETWEEN {} AND {}".format(PVALS['tinfo']['bidx'][i], PVALS['tinfo']['eidx'][i])
+      if PVALS['aname']:
+         clean_one_attm_for_tidx(PVALS['aname'], tidx, cnd)
+      else:
+         clean_imma_data_for_tidx(tidx, cnd)
+
+   cnt = PgDBI.pgdel(table, PVALS['dcnd'], PgLOG.LGEREX)
+   s = 's' if cnt > 1 else ''
+   PgLOG.pglog("{}: {} record{} deleted for {}".format(table, cnt, s, PVALS['dcnd']), PgLOG.LOGWRN)
+
+#
+# clean up imma data for table index
+#
+def clean_imma_data_for_tidx(tidx, cnd):
+
+   PgLOG.pglog("Clean IMMA data for table index {}...".format(tidx), PgLOG.LOGWRN)
+
+   for i in range(PgIMMA.TABLECOUNT):
+      aname = PgIMMA.IMMA_NAMES[i]
+      clean_one_attm_for_tidx(aname, tidx, cnd)
+
+#
+# clean up one attm data for table index
+#
+def clean_one_attm_for_tidx(aname, tidx, cnd):
+
+   table = "{}_{}".format(aname, tidx)
+   if not PgDBI.pgcheck(table): return 0  # not record to delete
+
+   if aname == 'iuida': clean_itidx_for_tidx(table, cnd) 
+
+   cnt = PgDBI.pgdel(table, cnd, PgLOG.LGEREX)
+   s = 's' if cnt > 1 else ''
+   PgLOG.pglog("{}: {} record{} deleted for {}".format(table, cnt, s, cnd), PgLOG.LOGWRN)
+
+   cnt = PgDBI.pgget(table, "", "", PgLOG.LGEREX)
+   clean_iattm_for_tidx(aname, tidx, cnt)
+
+#
+# clean up table itidx for table index
+#
+def clean_itidx_for_tidx(table, cnd):
+
+   tname = "cntldb.itidx"
+   uids = PgDBI.pgmget(table, "distinct (substring(uid, 1, 2)) uida", cnd, PgLOG.LGEREX)
+   ucnt = len(uids['uida']) if uids else 0
+   for i in range(ucnt):
+      table = "{}_{}".format(tname, uids['uida'][i].lower())
+      if not PgDBI.pgcheck(table): continue
+      cnt = PgDBI.pgdel(table, cnd, PgLOG.LOGWRN)
+      s = 's' if cnt > 1 else ''
+      PgLOG.pglog("{}: {} record{} deleted".format(table, cnt, s), PgLOG.LOGWRN)
+
+#
+# clean up table iattm for table index
+#
+def clean_iattm_for_tidx(aname, tidx, cnt):
+
+   table = "cntldb.iattm"
+   cnd = "attm = '{}' AND tidx = {}".format(aname, tidx)
+   pgrec = {'count' : cnt}
+   PgDBI.pgupdt(table, pgrec, cnd, PgLOG.LGWNEX)
+   PgLOG.pglog("{}: Set count to {} for {}".format(table, cnt, cnd), PgLOG.LOGWRN)
+   
+   table += "_daily"
+   cnd += " AND " + PVALS['dcnd']
+   PgDBI.pgdel(table, PVALS['dcnd'], PgLOG.LGWNEX)
+
+#
+# call main() to start program
+#
+if __name__ == "__main__": main()

rda_python_icoads-1.0.9/src/rda_python_icoads/counticoads.py

@@ -0,0 +1,153 @@
+#!/usr/bin/env python3
+#
+##################################################################################
+#
+#     Title : counticoads
+#    Author : Zaihua Ji, zji@ucar.edu
+#      Date : 12/30/2020
+#             2025-03-03 transferred to package rda_python_icoads from
+#             https://github.com/NCAR/rda-icoads.git
+#   Purpose : read ICOADS data from IVADDB and count out daily, monthly or year records
+#             by attms
+#
+#    Github : https://github.com/NCAR/rda-python-icoads.git
+#
+##################################################################################
+
+import sys
+import re
+from rda_python_common import PgLOG
+from rda_python_common import PgDBI
+from rda_python_common import PgUtil
+from . import PgIMMA
+
+PVALS = {
+   'bdate' : None,
+   'edate' : None,
+   'bpdate' : [],
+   'epdate' : [],
+   'period' : [],
+   'group' : None,
+   'names' : None
+}
+
+#
+# main function to run dsarch
+#
+def main():
+
+   option = ''
+   argv = sys.argv[1:]
+   for arg in argv:
+      if arg == "-b":
+         PgLOG.PGLOG['BCKGRND'] = 1
+      elif arg == "-g":
+         option = 'g'
+      elif re.match(r'^-', arg):
+         PgLOG.pglog(arg + ": Invalid Option", PgLOG.LGWNEX)
+      elif option:
+         PVALS['group'] = arg
+         option = ''
+      elif not PVALS['bdate']:
+         PVALS['bdate'] = arg
+      elif not PVALS['edate']:
+         PVALS['edate'] = arg
+      else:
+         PgLOG.pglog(arg + ": Invalid parameter", PgLOG.LGWNEX)
+   
+   PgDBI.ivaddb_dbname()
+   if not (PVALS['bdate'] and PVALS['edate'] and re.match(r'^(daily|monthly|yearly)$', PVALS['group'])):
+      pgrec = PgDBI.pgget("cntldb.inventory", "min(date) bdate, max(date) edate", '', PgLOG.LGEREX)
+      print("Usage: counticoads -g GroupBy (daily|monthly|yearly) BeginDate EndDate")
+      print("   Group by Daily, Monthly or Yearly is mandatory")
+      print("   Set BeginDate and EndDate between '{} and '{}'".format(pgrec['bdate'], pgrec['edate']))
+      sys.exit(0)
+   
+   if PgUtil.diffdate(PVALS['bdate'], PVALS['edate']) > 0:
+      tmpdate = PVALS['bdate']
+      PVALS['bdate'] = PVALS['edate']
+      PVALS['edate'] = tmpdate
+
+   PgLOG.PGLOG['LOGFILE'] = "icoads.log"
+   PgLOG.cmdlog("counticoads {}".format(' '.format(argv)))
+   PVALS['names'] = '/'.join(PgIMMA.IMMA_NAMES)
+   fname = "ICOADS_COUNTS_{}_{}-{}.txt" .format(PVALS['group'].upper(), PVALS['bdate'], PVALS['edate'])
+   IMMA = open(fname, 'w')
+   IMMA.write("{}, {}\n".format(PVALS['group'], ', '.join(PgIMMA.IMMA_NAMES)))
+   count_imma_data(IMMA)
+   IMMA.close()
+
+   PgLOG.cmdlog()
+   sys.exit(0)
+
+#
+# count imaa data
+#
+def count_imma_data(IMMA):
+
+   pcnt = init_periods()
+   tcounts = [0]*PgIMMA.TABLECOUNT
+
+   for pidx in range(pcnt):
+      acnts = count_period_imma(pidx)
+      IMMA.write("{}' {}\n".format(PVALS['period'][pidx], ', '.join(acnts)))
+      for i in range(PgIMMA.TABLECOUNT): tcounts[i] += acnts[i]
+
+   if pcnt > 1:
+      IMMA.write("Total, {}\n".format(', '.join(tcounts)))
+      PgLOG.pglog("{}({}) for {} {} periods".format('/'.join(tcounts), PVALS['names'], pcnt, PVALS['group']), PgLOG.LOGWRN)
+
+
+
+#
+# read icoads record from given file name and save them into RDADB
+#
+def count_period_imma(pidx):
+
+   PgLOG.pglog("count IMMA1 records for {} period {} from IVADDB".format(PVALS['group'], PVALS['period'][pidx]), PgLOG.WARNLG)
+   acounts = [0]*PgIMMA.TABLECOUNT
+   date = PVALS['bpdate'][pidx]
+   while date <= PVALS['epdate'][pidx]:
+      acnts = PgIMMA.count_imma_records(date)
+      if acnts:
+         for i in range(PgIMMA.TABLECOUNT): acounts[i] += acnts[i]
+      date = PgUtil.adddate(date, 0, 0, 1)
+
+   PgLOG.pglog("{}({}) for {} period {}".format('/'.join(acounts), PVALS['names'], PVALS['group'], PVALS['period'][pidx]), PgLOG.LOGWRN)
+   return acounts
+
+#
+# initialize (daily|monthly|yearly) periods
+#
+def init_periods():
+
+   bdate = PVALS['bdate']
+   if PVALS['group'] == "yearly":
+      dfmt = "YYYY"
+      eflg = "Y"
+   elif PVALS['group'] == 'monthly':
+      dfmt = "YYYY-MM"
+      eflg = "M"
+   else:  # must be daily
+      dfmt = "YYYY-MM-DD"
+      eflg = ""
+
+   pcnt = 0
+   while True:
+      pcnt += 1
+      PVALS['bpdate'].append(bdate)
+      PVALS['period'].append(PgUtil.format_date(bdate, dfmt))
+      edate = PgUtil.enddate(bdate, 0, eflg) if eflg else bdate
+      if PgUtil.diffdate(PVALS['edate'], edate) > 0:
+         PVALS['epdate'].append(edate)
+         bdate = PgUtil.adddate(edate, 0, 0, 1)
+      else:
+         PVALS['epdate'].append(PVALS['edate'])
+         break
+
+   return pcnt
+
+#
+# call main() to start program
+#
+if __name__ == "__main__": main()