rda-python-icoads 1.0.7__tar.gz → 1.0.10__tar.gz

rda_python_icoads-1.0.10/MANIFEST.in

@@ -0,0 +1,6 @@
+include src/rda_python_icoads/README_R3.0_Subset.html
+include src/rda_python_icoads/rdimma1_csv.f
+include src/rda_python_icoads/msg
+include src/rda_python_icoads/R3.0-stat_doc.pdf
+include src/rda_python_icoads/msg3.0_subset_readme.txt
+include src/rda_python_icoads/msgsubset.f

{rda_python_icoads-1.0.7 → rda_python_icoads-1.0.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rda_python_icoads
-Version: 1.0.7
+Version: 1.0.10
 Summary: RDA python package to manage RDA ICOADS datasets
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-icoads

rda_python_icoads-1.0.10/pyproject.toml

@@ -0,0 +1,61 @@
+[build-system]
+requires = [
+  "setuptools>=61.0",
+]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "rda_python_icoads"
+version = "1.0.10"
+authors = [
+  { name="Zaihua Ji", email="zji@ucar.edu" },
+]
+description = "RDA python package to manage RDA ICOADS datasets"
+readme = "README.md"
+requires-python = ">=3.7"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+    "Development Status :: 5 - Production/Stable",
+]
+dependencies = [
+  "rda_python_common",
+]
+
+[tool.setuptools]
+include-package-data = true
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.setuptools.package-data]
+"rda_python_icoads" = [
+  "README_R3.0_Subset.html",
+  "rdimma1_csv.f",
+  "msg",
+  "R3.0-stat_doc.pdf",
+  "msg3.0_subset_readme.txt",
+  "msgsubset.f",
+]
+
+[project.urls]
+"Homepage" = "https://github.com/NCAR/rda-python-icoads"
+
+[project.scripts]
+"imma1_subset" = "rda_python_icoads.imma1_subset:main"
+"msg3_subset" = "rda_python_icoads.msg3_subset:main"
+"checkicoads" = "rda_python_icoads.checkicoads:main"
+"cleanicoads" = "rda_python_icoads.cleanicoads:main"
+"counticoads" = "rda_python_icoads.counticoads:main"
+"countattm" = "rda_python_icoads.countattm:main"
+"countattmvar" = "rda_python_icoads.countattmvar:main"
+"countsst" = "rda_python_icoads.countsst:main"
+"fillicoads" = "rda_python_icoads.fillicoads:main"
+"fillinventory" = "rda_python_icoads.fillinventory:main"
+"fillitable" = "rda_python_icoads.fillitable:main"
+"fillmonth" = "rda_python_icoads.fillmonth:main"
+"fixidx" = "rda_python_icoads.fixidx:main"
+"msg_download" = "rda_python_icoads.msg_download:main"
+"maxsst" = "rda_python_icoads.maxsst:main"
+"writeicoads" = "rda_python_icoads.writeicoads:main"

rda_python_icoads-1.0.10/src/rda_python_icoads/checkicoads.py

@@ -0,0 +1,222 @@
+#!/usr/bin/env python3
+#
+##################################################################################
+#
+#     Title : checkicoads
+#    Author : Zaihua Ji, zji@ucar.edu
+#      Date : 12/30/2020
+#             2025-03-03 transferred to package rda_python_icoads from
+#             https://github.com/NCAR/rda-icoads.git
+#   Purpose : check and compare ICOADS data files and IVADDB records
+#
+#    Github : https://github.com/NCAR/rda-python-icoads.git
+#
+##################################################################################
+
+import sys
+import os
+import re
+from os import path as op
+from rda_python_common import PgLOG
+from rda_python_common import PgDBI
+from rda_python_common import PgUtil
+from rda_python_common import PgOPT
+from rda_python_common import PgSIG
+from . import PgIMMA
+
+PVALS = {
+   'bdate' : None,
+   'edate' : None,
+   'bmdate' : [],
+   'emdate' : [],
+   'fname' : [],
+   'flag' : [],   # 1 - file exists, 2 - db records exist, 3 - both
+   'mproc' : 10,
+   'fpattern' : "IMMA1_R3.0.0_<YYYY-MM>",
+   'readall' : 0
+}
+
+#
+# main function to run dsarch
+#
+def main():
+
+   option = ''
+   argv = sys.argv[1:]
+   
+   for arg in argv:
+      if arg == "-b":
+         PgLOG.PGLOG['BCKGRND'] = 1
+      elif arg == "-a":
+         PVALS['readall'] = 1
+      elif arg == "-f":
+         option = 'f'
+      elif arg == "-m":
+         option = 'm'
+      elif re.match(r'^-', arg):
+         PgLOG.pglog(arg + ": Invalid Option", PgLOG.LGWNEX)
+      elif option:
+         if option == 'f':
+            PVALS['fpattern'] = arg
+         elif option == 'm':
+            PVALS['mproc'] = arg
+         option = ''
+      elif not PVALS['bdate']:
+         PVALS['bdate'] = arg
+      elif not PVALS['edate']:
+         PVALS['edate'] = arg
+      else:
+         PgLOG.pglog(arg + ": Invalid parameter", PgLOG.LGWNEX)
+   
+   PgLOG.PGLOG['LOGFILE'] = "icoads.log"
+   PgDBI.ivaddb_dbname()
+   
+   if not (PVALS['bdate'] and PVALS['edate']):
+      pgrec = PgDBI.pgget("cntldb.inventory", "min(date) bdate, max(date) edate", '', PgLOG.LGEREX)
+      print("Usage: checkicoads [-a] [-m mproc] [-f FilePattern] BeginDate EndDate")
+      print("   Default FilePattern is " + PVALS['fpattern'])
+      print("   Option -a - read all attms, including multi-line ones, such as IVAD and REANQC")
+      print("   Option -m - start up to given number of processes, one for each month (Default to 10)")
+      print("   Set BeginDate and EndDate between '{}' and '{}'".format(pgrec['bdate'], pgrec['edate']))
+      sys.exit(0)
+
+   if PgUtil.diffdate(PVALS['bdate'], PVALS['edate']) > 0:
+      tmpdate = PVALS['bdate']
+      PVALS['bdate'] = PVALS['edate']
+      PVALS['edate'] = tmpdate
+   
+   PgLOG.cmdlog("checkicoads {}".format(' '.join(argv)))
+   check_imma_data()
+   PgLOG.cmdlog()
+   sys.exit(0)
+
+#
+# check imma data
+#
+def check_imma_data():
+
+   mcnt = init_months()
+   if mcnt == 1: PVALS['mproc'] = 1
+   if PVALS['mproc'] > 1:
+      PgSIG.start_none_daemon('writeicoads', '', PgLOG.PGLOG['CURUID'], PVALS['mproc'], 300, 1)
+
+   for midx in range(mcnt):
+      fname = PVALS['fname'][midx]
+      if op.isfile(fname +".cnt"): continue    # monthly file counted already
+      if PVALS['mproc'] > 1:
+         stat = PgSIG.start_child("checkicoads_{}".format(midx), PgLOG.LOGWRN, 1)  # try to start a child process
+         if stat <= 0:
+            sys.exit(1)   # something wrong
+         elif PgSIG.PGSIG['PPID'] > 1:
+            check_imma_file(fname, midx)
+            sys.exit(0)  # stop child process
+         else:
+            PgDBI.pgdisconnect(0)  # disconnect database for reconnection
+            continue   # continue for next midx
+      else:
+         check_imma_file(fname, midx)
+
+   if PVALS['mproc'] > 1: PgSIG.check_child(None, 0, PgLOG.LOGWRN, 1)
+
+   dump_final_counts()
+
+#
+# compare icoads records from given file name and IVADDB
+#
+def check_imma_file(fname, midx):
+
+   PgLOG.pglog("Count IMMA records in File '{}'".format(fname), PgLOG.WARNLG)
+   flag = PVALS['flag'][midx]
+
+   acnts = [0]*PgIMMA.TABLECOUNT
+   acounts = [0]*PgIMMA.TABLECOUNT
+
+   if flag&1:
+      IMMA = open(fname, 'r')
+      line = IMMA.readline()   
+      while line:
+         if PVALS['readall'] and re.match(r'^98', line):
+             PgIMMA.get_imma_multiple_counts(line, acnts)
+         else:
+            PgIMMA.get_imma_counts(line, acnts)
+         line.IMMA.readline()
+      IMMA.close()
+      for i in range(PgIMMA.TABLECOUNT): acounts[i] = acnts[i]
+
+   if flag&2:
+      PgLOG.pglog("Count IMMA records in in IVADDB", PgLOG.WARNLG)
+      cdate = bdate = PVALS['bmdate'][midx]
+      edate = PVALS['emdate'][midx]
+      while cdate <= edate:
+         acnts = PgIMMA.count_imma_records(cdate, 0, PVALS['readall'])
+         cdate = PgUtil.adddate(cdate, 0, 0, 1)
+         if acnts:
+            for i in range(PgIMMA.TABLECOUNT): acounts[i] -= acnts[i]
+
+   dump_monthly_counts(fname, acounts)
+
+#
+# dump monthly counts
+#
+def dump_monthly_counts(fname, acounts):
+
+   oname = fname + ".cnt"
+   IMMA = open(oname, 'w')
+   IMMA.write("{}, {}\n".format(fname, ', '.join(acounts)))
+   IMMA.close()
+
+#
+# concat all monthly counts into one
+#
+def dump_final_counts():
+
+   fname = "ICOADS_DIFF_COUNTS.csv"
+
+   IMMA = open(fname, 'w')
+   IMMA.write("FileName, {}\n".format(', '.join(PgIMMA.IMMA_NAMES)))
+   IMMA.close()
+   PgLOG.pgsystem("cat *.cnt >> " + fname)
+
+#
+# initialize the month list
+#
+def init_months():
+
+   seps = ["<" , ">"];  # temporal pattern delimiters
+   match = "[^" + seps[1] + "]+"
+
+   ms = re.search(r'{}({}){}'.format(seps[0], match, seps[1]), PVALS['fpattern'])
+   if ms:
+      tpattern = ms.group(1)
+      treplace = "{}{}{}".format(seps[0], tpattern, seps[1])
+   else:
+      PgLOG.pglog(PVALS['fpattern'] + ": Not temporal pattern found to get month list", PgLOG.LGEREX)
+
+   bdate = PVALS['bdate']
+   done = midx = 0
+   while True:
+      edate = PgUtil.enddate(bdate, 0, 'M')
+      if PgUtil.diffdate(PVALS['edate'], edate) <= 0:
+         edate = PVALS['edate']
+         done = 1
+      mdate = PgUtil.format_date(bdate, tpattern)
+      fname = PVALS['fpattern'].replace(treplace, mdate)
+      flag = 0
+      if op.isfile(fname): flag += 1
+      if PgDBI.pgget("cntldb.inventory", "", "date BETWEEN '{}' AND '{}'".format(bdate, edate), PgLOG.LGEREX):
+         flag += 2
+      if flag:
+         PVALS['bmdate'].append(bdate)
+         PVALS['emdate'].append(edate)
+         PVALS['fname'].append(fname)
+         PVALS['flag'].append(flag)
+         midx += 1
+      if done: break
+      bdate = PgUtil.adddate(edate, 0, 0, 1)
+
+   return midx
+
+#
+# call main() to start program
+#
+if __name__ == "__main__": main()

rda_python_icoads-1.0.10/src/rda_python_icoads/cleanicoads.py

@@ -0,0 +1,175 @@
+#!/usr/bin/env python3
+#
+##################################################################################
+#
+#     Title : cleanicoads
+#    Author : Zaihua Ji, zji@ucar.edu
+#      Date : 12/30/2020
+#             2025-03-03 transferred to package rda_python_icoads from
+#             https://github.com/NCAR/rda-icoads.git
+#   Purpose : clean up one or all IMMA1 attms in IVADDB for given period
+#
+#    Github : https://github.com/NCAR/rda-python-icoads.git
+#
+##################################################################################
+
+import sys
+import os
+import re
+from os import path as op
+from rda_python_common import PgLOG
+from rda_python_common import PgDBI
+from rda_python_common import PgUtil
+from . import PgIMMA
+
+PVALS = {
+   'bdate' : None,
+   'edate' : None,
+   'aname' : None,
+   'tinfo' : {},
+   'tcnt' : 0,
+   'dcnd' : None,
+   'uatti' : '',
+   'names' : None
+}
+
+#
+# main function to run dsarch
+#
+def main():
+
+   option = ''
+   files = []
+   leaduid = 0
+   chkexist = 0
+   readall = 0
+   argv = sys.argv[1:]
+
+   for arg in argv:
+      if arg == "-b":
+         PgLOG.PGLOG['BCKGRND'] = 1
+      elif arg == "-a":
+         option = 'a'
+      elif re.match(r'^-', arg):
+         PgLOG.pglog(arg + ": Invalid Option", PgLOG.LGWNEX)
+      elif option:
+         PVALS['aname'] = arg
+         option = ''
+      elif not PVALS['bdate']:
+         PVALS['bdate'] = arg
+      elif not PVALS['edate']:
+         PVALS['edate'] = arg
+      else:
+         PgLOG.pglog(arg + ": Invalid parameter", PgLOG.LGWNEX)
+
+   if not PVALS['bdate']:
+      print("Usage: cleanicoads [-a ATTNAME] BDATE EDATE")
+      print("   Option -a - clean a single attm for given attm name")
+      PgLOG.pgexit()
+
+   PgLOG.PGLOG['LOGFILE'] = "icoads.log"
+   PgDBI.ivaddb_dbname()
+   PgLOG.cmdlog("cleanicoads {}".format(' '.join(argv)))
+   set_table_info()
+   clean_imma_data()   
+   PgLOG.cmdlog()
+   PgLOG.pgexit()
+
+#
+# set the table index list
+#
+def set_table_info():
+
+   table = "cntldb.inventory"
+   if PVALS['edate']:
+      PVALS['dcnd'] = "date BETWEEN '{}' AND '{}'".format(PVALS['bdate'], PVALS['edate'])
+   else:
+      PVALS['dcnd'] = "date >= '{}'".format(PVALS['bdate'])
+
+   PVALS['tinfo'] = PgDBI.pgmget(table, "tidx, min(miniidx) bidx, max(maxiidx) eidx", PVALS['dcnd'] + " GROUP BY tidx", PgLOG.LGEREX)
+   PVALS['tcnt'] = len(PVALS['tinfo']['tidx']) if PVALS['tinfo'] else 0
+
+   if not PVALS['tcnt']:
+      PgLOG.pglog("{}: No data found in IVADDB for {}".format(table, PVALS['dcnd']), PgLOG.LGEREX)   
+
+#
+# clean up imma data
+#
+def clean_imma_data():
+
+   table = "cntldb.inventory"
+
+   for i in range(PVALS['tcnt']):
+      tidx = PVALS['tinfo']['tidx'][i]
+      cnd = "iidx BETWEEN {} AND {}".format(PVALS['tinfo']['bidx'][i], PVALS['tinfo']['eidx'][i])
+      if PVALS['aname']:
+         clean_one_attm_for_tidx(PVALS['aname'], tidx, cnd)
+      else:
+         clean_imma_data_for_tidx(tidx, cnd)
+
+   cnt = PgDBI.pgdel(table, PVALS['dcnd'], PgLOG.LGEREX)
+   s = 's' if cnt > 1 else ''
+   PgLOG.pglog("{}: {} record{} deleted for {}".format(table, cnt, s, PVALS['dcnd']), PgLOG.LOGWRN)
+
+#
+# clean up imma data for table index
+#
+def clean_imma_data_for_tidx(tidx, cnd):
+
+   PgLOG.pglog("Clean IMMA data for table index {}...".format(tidx), PgLOG.LOGWRN)
+
+   for i in range(PgIMMA.TABLECOUNT):
+      aname = PgIMMA.IMMA_NAMES[i]
+      clean_one_attm_for_tidx(aname, tidx, cnd)
+
+#
+# clean up one attm data for table index
+#
+def clean_one_attm_for_tidx(aname, tidx, cnd):
+
+   table = "{}_{}".format(aname, tidx)
+   if not PgDBI.pgcheck(table): return 0  # not record to delete
+
+   if aname == 'iuida': clean_itidx_for_tidx(table, cnd) 
+
+   cnt = PgDBI.pgdel(table, cnd, PgLOG.LGEREX)
+   s = 's' if cnt > 1 else ''
+   PgLOG.pglog("{}: {} record{} deleted for {}".format(table, cnt, s, cnd), PgLOG.LOGWRN)
+
+   cnt = PgDBI.pgget(table, "", "", PgLOG.LGEREX)
+   clean_iattm_for_tidx(aname, tidx, cnt)
+
+#
+# clean up table itidx for table index
+#
+def clean_itidx_for_tidx(table, cnd):
+
+   tname = "cntldb.itidx"
+   uids = PgDBI.pgmget(table, "distinct (substring(uid, 1, 2)) uida", cnd, PgLOG.LGEREX)
+   ucnt = len(uids['uida']) if uids else 0
+   for i in range(ucnt):
+      table = "{}_{}".format(tname, uids['uida'][i].lower())
+      if not PgDBI.pgcheck(table): continue
+      cnt = PgDBI.pgdel(table, cnd, PgLOG.LOGWRN)
+      s = 's' if cnt > 1 else ''
+      PgLOG.pglog("{}: {} record{} deleted".format(table, cnt, s), PgLOG.LOGWRN)
+
+#
+# clean up table iattm for table index
+#
+def clean_iattm_for_tidx(aname, tidx, cnt):
+
+   table = "cntldb.iattm"
+   cnd = "attm = '{}' AND tidx = {}".format(aname, tidx)
+   pgrec = {'count' : cnt}
+   PgDBI.pgupdt(table, pgrec, cnd, PgLOG.LGWNEX)
+   PgLOG.pglog("{}: Set count to {} for {}".format(table, cnt, cnd), PgLOG.LOGWRN)
+   
+   table += "_daily"
+   cnd += " AND " + PVALS['dcnd']
+   PgDBI.pgdel(table, PVALS['dcnd'], PgLOG.LGWNEX)
+
+#
+# call main() to start program
+#
+if __name__ == "__main__": main()

rda_python_icoads-1.0.10/src/rda_python_icoads/countattm.py

@@ -0,0 +1,125 @@
+#!/usr/bin/env python3
+#
+##################################################################################
+#
+#     Title : countattm
+#    Author : Zaihua Ji, zji@ucar.edu
+#      Date : 01/09/2021
+#             2025-03-04 transferred to package rda_python_icoads from
+#             https://github.com/NCAR/rda-icoads.git
+#   Purpose : process ICOADS data files in IMMA format and count the matching, 
+#             unmatching and empty records
+#
+#    Github : https://github.com/NCAR/rda-python-icoads.git
+#
+##################################################################################
+
+import sys
+import re
+from rda_python_common import PgLOG
+from rda_python_common import PgDBI
+from . import PgIMMA
+
+PVALS = {
+   'group' : None,
+   'files' : [],
+   'aname' : None,
+   'bym' : None,
+   'eym' : None
+}
+
+ACOUNTS = {}
+
+#
+# main function
+#
+def main():
+
+   option = ''
+   argv = sys.argv[1:]
+
+   for arg in argv:
+      if arg == "-b":
+         PgLOG.PGLOG['BCKGRND'] = 1
+      elif arg == '-g':
+         option = 'g'
+      elif re.match(r'^-', arg):
+         PgLOG.pglog(arg + ": Invalid Option", PgLOG.LGWNEX)
+      elif option:
+         PVALS['group'] = arg
+         option = ''
+      else:
+         PVALS['files'].append(arg)
+   
+   if not (PVALS['files'] and re.match(r'^(monthly|yearly)$', PVALS['group'])):
+      print("Usage: countattm -g GroupBy (monthly|yearly) FileNameList")
+      print("   Group by Monthly or Yearly is mandatory")
+      print("   At least one file name needs to be present to count icoads attm data")
+      sys.exit(0)
+
+   PgLOG.PGLOG['LOGFILE'] = "icoads.log"
+   PgDBI.ivaddb_dbname()
+   PgLOG.cmdlog("countattm {}".format(' '.join(argv)))
+   for file in PVALS['files']: count_attm_file(file)
+   dump_attm_counts()
+   PgLOG.cmdlog()
+   sys.exit(0)
+
+#
+# read icoads record from given file name and count the records
+#
+def count_attm_file(fname):
+
+   PgLOG.pglog("Count attm records in File '{}'".format(fname), PgLOG.WARNLG)
+   
+   # Get file month
+   ms = re.search(r'(\d\d\d\d)-(\d\d)', fname)
+   if ms:
+      yr = ms.group(1)
+      mn = ms.group(2)
+      ym = "{}-{}".format(yr, mn)
+      if not PVALS['bym']: PVALS['bym'] = ym
+      PVALS['eym'] = ym
+      key = yr if PVALS['group'] == "yearly"  else ym
+      if key not in ACOUNTS:
+         ACOUNTS[key] = {'match' : 0, 'unmatch' : 0, 'empty' : 0, 'total' : 0}
+   else:
+      PgLOG.pglog(fname + ": miss year/month values in file name", PgLOG.LGEREX)
+
+   ATTM = open(fname, 'r')
+   acnt = 0
+   line = ATTM.readline()
+   # check and record standalone attm name
+   if not PVALS['aname']: PVALS['aname'] = PgIMMA.identify_attm_name(line)
+   while line:
+      ACOUNTS[key]['total'] += 1
+      # commet out these two line for normal records
+      line = line.rstrip()
+      if len(line) < 20:
+         ACOUNTS[key]['empty'] += 1
+      else:
+         idate = PgIMMA.get_imma_date(line)
+         if idate or idate is None:
+            ACOUNTS[key]['match'] += 1
+         else:
+            ACOUNTS[key]['unmatch'] += 1
+      line = ATTM.readline()
+   ATTM.close()
+
+#
+# dump attm counts by group
+#
+def dump_attm_counts():
+   
+   fname = "{}_COUNTS_{}_{}-{}.txt".format(PVALS['aname'], PVALS['group'].upper(), PVALS['bym'], PVALS['eym'])
+   ATTM = open(fname, 'w')
+   ATTM.write(PVALS['group'] + ", match, unmatch, empty, total\n")
+   
+   for key in sorted(ACOUNTS):
+      ATTM.write("{}, {}, {}, {}, {}\n".format(key, ACOUNTS[key]['match'],
+                 ACOUNTS[key]['unmatch'], ACOUNTS[key]['empty'], ACOUNTS[key]['total']))
+
+#
+# call main() to start program
+#
+if __name__ == "__main__": main()