rda-python-icoads 1.0.9__tar.gz → 1.0.10__tar.gz

{rda_python_icoads-1.0.9 → rda_python_icoads-1.0.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rda_python_icoads
-Version: 1.0.9
+Version: 1.0.10
 Summary: RDA python package to manage RDA ICOADS datasets
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-icoads

{rda_python_icoads-1.0.9 → rda_python_icoads-1.0.10}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "rda_python_icoads"
-version = "1.0.9"
+version = "1.0.10"
 authors = [
   { name="Zaihua Ji", email="zji@ucar.edu" },
 ]
@@ -48,9 +48,14 @@ where = ["src"]
 "checkicoads" = "rda_python_icoads.checkicoads:main"
 "cleanicoads" = "rda_python_icoads.cleanicoads:main"
 "counticoads" = "rda_python_icoads.counticoads:main"
+"countattm" = "rda_python_icoads.countattm:main"
+"countattmvar" = "rda_python_icoads.countattmvar:main"
+"countsst" = "rda_python_icoads.countsst:main"
 "fillicoads" = "rda_python_icoads.fillicoads:main"
 "fillinventory" = "rda_python_icoads.fillinventory:main"
 "fillitable" = "rda_python_icoads.fillitable:main"
 "fillmonth" = "rda_python_icoads.fillmonth:main"
+"fixidx" = "rda_python_icoads.fixidx:main"
 "msg_download" = "rda_python_icoads.msg_download:main"
+"maxsst" = "rda_python_icoads.maxsst:main"
 "writeicoads" = "rda_python_icoads.writeicoads:main"

rda_python_icoads-1.0.10/src/rda_python_icoads/countattm.py

@@ -0,0 +1,125 @@
+#!/usr/bin/env python3
+#
+##################################################################################
+#
+#     Title : countattm
+#    Author : Zaihua Ji, zji@ucar.edu
+#      Date : 01/09/2021
+#             2025-03-04 transferred to package rda_python_icoads from
+#             https://github.com/NCAR/rda-icoads.git
+#   Purpose : process ICOADS data files in IMMA format and count the matching, 
+#             unmatching and empty records
+#
+#    Github : https://github.com/NCAR/rda-python-icoads.git
+#
+##################################################################################
+
+import sys
+import re
+from rda_python_common import PgLOG
+from rda_python_common import PgDBI
+from . import PgIMMA
+
+PVALS = {
+   'group' : None,
+   'files' : [],
+   'aname' : None,
+   'bym' : None,
+   'eym' : None
+}
+
+ACOUNTS = {}
+
+#
+# main function
+#
+def main():
+
+   option = ''
+   argv = sys.argv[1:]
+
+   for arg in argv:
+      if arg == "-b":
+         PgLOG.PGLOG['BCKGRND'] = 1
+      elif arg == '-g':
+         option = 'g'
+      elif re.match(r'^-', arg):
+         PgLOG.pglog(arg + ": Invalid Option", PgLOG.LGWNEX)
+      elif option:
+         PVALS['group'] = arg
+         option = ''
+      else:
+         PVALS['files'].append(arg)
+   
+   if not (PVALS['files'] and re.match(r'^(monthly|yearly)$', PVALS['group'])):
+      print("Usage: countattm -g GroupBy (monthly|yearly) FileNameList")
+      print("   Group by Monthly or Yearly is mandatory")
+      print("   At least one file name needs to be present to count icoads attm data")
+      sys.exit(0)
+
+   PgLOG.PGLOG['LOGFILE'] = "icoads.log"
+   PgDBI.ivaddb_dbname()
+   PgLOG.cmdlog("countattm {}".format(' '.join(argv)))
+   for file in PVALS['files']: count_attm_file(file)
+   dump_attm_counts()
+   PgLOG.cmdlog()
+   sys.exit(0)
+
+#
+# read icoads record from given file name and count the records
+#
+def count_attm_file(fname):
+
+   PgLOG.pglog("Count attm records in File '{}'".format(fname), PgLOG.WARNLG)
+   
+   # Get file month
+   ms = re.search(r'(\d\d\d\d)-(\d\d)', fname)
+   if ms:
+      yr = ms.group(1)
+      mn = ms.group(2)
+      ym = "{}-{}".format(yr, mn)
+      if not PVALS['bym']: PVALS['bym'] = ym
+      PVALS['eym'] = ym
+      key = yr if PVALS['group'] == "yearly"  else ym
+      if key not in ACOUNTS:
+         ACOUNTS[key] = {'match' : 0, 'unmatch' : 0, 'empty' : 0, 'total' : 0}
+   else:
+      PgLOG.pglog(fname + ": miss year/month values in file name", PgLOG.LGEREX)
+
+   ATTM = open(fname, 'r')
+   acnt = 0
+   line = ATTM.readline()
+   # check and record standalone attm name
+   if not PVALS['aname']: PVALS['aname'] = PgIMMA.identify_attm_name(line)
+   while line:
+      ACOUNTS[key]['total'] += 1
+      # commet out these two line for normal records
+      line = line.rstrip()
+      if len(line) < 20:
+         ACOUNTS[key]['empty'] += 1
+      else:
+         idate = PgIMMA.get_imma_date(line)
+         if idate or idate is None:
+            ACOUNTS[key]['match'] += 1
+         else:
+            ACOUNTS[key]['unmatch'] += 1
+      line = ATTM.readline()
+   ATTM.close()
+
+#
+# dump attm counts by group
+#
+def dump_attm_counts():
+   
+   fname = "{}_COUNTS_{}_{}-{}.txt".format(PVALS['aname'], PVALS['group'].upper(), PVALS['bym'], PVALS['eym'])
+   ATTM = open(fname, 'w')
+   ATTM.write(PVALS['group'] + ", match, unmatch, empty, total\n")
+   
+   for key in sorted(ACOUNTS):
+      ATTM.write("{}, {}, {}, {}, {}\n".format(key, ACOUNTS[key]['match'],
+                 ACOUNTS[key]['unmatch'], ACOUNTS[key]['empty'], ACOUNTS[key]['total']))
+
+#
+# call main() to start program
+#
+if __name__ == "__main__": main()

rda_python_icoads-1.0.10/src/rda_python_icoads/countattmvar.py

@@ -0,0 +1,237 @@
+#!/usr/bin/env python3
+#
+##################################################################################
+#
+#     Title : countattmvar
+#    Author : Zaihua Ji, zji@ucar.edu
+#      Date : 01/09/2021
+#             2025-03-04 transferred to package rda_python_icoads from
+#             https://github.com/NCAR/rda-icoads.git
+#   Purpose : read ICOADS data from IVADDB and count out daily, monthly or year records
+#             by attms/variable
+#
+#    Github : https://github.com/NCAR/rda-python-icoads.git
+#
+##################################################################################
+
+import sys
+import os
+import re
+from os import path as op
+from rda_python_common import PgLOG
+from rda_python_common import PgDBI
+from rda_python_common import PgUtil
+from . import PgIMMA
+
+PVALS = {
+   'bdate' : None,
+   'edate' : None,
+   'bpdate' : [],
+   'epdate' : [],
+   'period' : [],
+   'group' : None,
+   'aname' : None,
+   'jname' : None,
+   'fname' : None,
+   'oname' : None,
+   'wcnd' : None,
+   'atables' : {},   # cache atable names processed. value 1 table exists; 0 not exsits
+   'jtables' : {}    # cache jtable names processed. value 1 table exists; 0 not exsits
+}
+
+#
+# main function to run dsarch
+#
+def main():
+
+   option = None
+   argv = sys.argv[1:]
+   
+   for arg in argv:
+      if arg == "-b":
+         PgLOG.PGLOG['BCKGRND'] = 1
+         continue
+      ms = re.match(r'^-[afgjow])$', arg)
+      if ms:
+         option = ms.group(1)
+         continue
+      if re.match(r'^-', arg): PgLOG.pglog(arg + ": Invalid Option", PgLOG.LGWNEX)
+      elif option:
+         if option == 'a':
+            PVALS['aname'] = arg
+         elif option == 'g':
+            PVALS['group'] = arg
+         elif option == 'j':
+            PVALS['jname'] = arg
+         elif option == 'f':
+            PVALS['fname'] = arg
+         elif option == 'o':
+            PVALS['oname'] = arg
+         elif option == 'w':
+            PVALS['wcnd'] = arg
+         option = ''
+      elif not PVALS['bdate']:
+         PVALS['bdate'] = arg
+      elif not PVALS['edate']:
+         PVALS['edate'] = arg
+      else:
+         PgLOG.pglog(arg + ": Invalid parameter", PgLOG.LGWNEX)
+   
+   PgDBI.ivaddb_dbname()
+   
+   if(not (PVALS['bdate'] and PVALS['edate']) or not re.match(r'^(daily|monthly|yearly)$', PVALS['group']) or 
+      not (PVALS['aname'] and PVALS['fname'] and PVALS['wcnd'])):
+      pgrec = PgDBI.pgget("cntldb.inventory", "min(date) bdate, max(date) edate", '', PgLOG.LGEREX)
+      print("Usage: countattmvar -g GroupBy (daily|monthly|yearly) -a AttmName -f FieldName -w WhereCondition [-j JoinName] BeginDate EndDate")
+      print("   Group by Daily, Monthly or Yearly is mandatory")
+      print("   Set BeginDate and EndDate between '' and '{}'".format(pgrec['bdate'], pgrec['edate']))
+      sys.exit(0)
+
+   if PgUtil.diffdate(PVALS['bdate'], PVALS['edate']) > 0:
+      tmpdate = PVALS['bdate']
+      PVALS['bdate'] = PVALS['edate']
+      PVALS['edate'] = tmpdate
+   
+   if PVALS['jname'] and (PVALS['jname'] == PVALS['aname'] or PVALS['jname'] == "icoreloc"): PVALS['jname'] = None
+   
+   PgLOG.PGLOG['LOGFILE'] = "icoads.log"
+   PgLOG.cmdlog("countattmvar {}".format(' '.join(argv)))
+   
+   if not PVALS['oname']:
+      PVALS['oname'] = "{}.{}_COUNTS_{}_{}-{}.txt".format(PVALS['aname'], PVALS['fname'],
+                        PVALS['group'].upper(), PVALS['bdate'], PVALS['edate'])
+
+   IMMA = open(PVALS['oname'], 'w')
+   IMMA.write("{}, {}\n".format(PVALS['group'], PVALS['fname']))
+   count_attm_variable(IMMA)
+   IMMA.close()
+   PgLOG.PgLOG.cmdlog()
+   sys.exit(0)
+
+#
+# count the variable of a attm daily/month/y/yearly
+#
+def count_attm_variable(IMMA):
+
+   pcnt = init_periods()
+   tcnt = 0
+   for pidx in range(pcnt):
+      if PVALS['group'] == 'daily':
+         acnt = count_daily_attm_variable(PVALS['bpdate'][pidx])
+      else:
+         acnt = count_period_attm_variable(pidx)
+      if not acnt: continue
+      IMMA.write("{}, {}\n".format(PVALS['period'][pidx], acnt))
+      tcnt += acnt
+
+   if pcnt > 1:
+      IMMA.write("Total, {}\n".format(tcnt))
+      PgLOG.pglog("{}.{}: {} total for {} {} periods".format(PVALS['aname'], PVALS['fname'], tcnt, pcnt, PVALS['group']), PgLOG.LOGWRN)
+
+#
+# read icoads record from given file name and save them into RDADB
+#
+def count_period_attm_variable(pidx):
+
+   bdate = PVALS['bpdate'][pidx]
+   edate = PVALS['epdate'][pidx]
+   btidx = PgIMMA.date2tidx(bdate)
+   etidx = PgIMMA.date2tidx(edate)
+   tblcnt = etidx-btidx+1
+   PgLOG.pglog("Counting {}.{} for {} from IVADDB".format(PVALS['aname'], PVALS['fname'], PVALS['period'][pidx]), PgLOG.WARNLG)
+   
+   # get acount from the first table
+   acount = 0
+   if tblcnt == 1:
+      cnds = ["date BETWEEN '{}' AND '{}' AND ".format(bdate, edate)]
+   else:
+      cnds = ['']*tblcnt
+      cnds[0] = "date >='{}' AND ".format(bdate)
+      cnds[tblcnt-1] = "date <= '{}' AND ".format(edate)
+   for i in range(tblcnt): acount += count_table_attm_variable(cnds[i], i + btidx)
+
+   if acount > 0:
+      PgLOG.pglog("{}.{}: {} for {} of {}".format(PVALS['aname'], PVALS['fname'], acount, PVALS['wcnd'], PVALS['period'][pidx]), PgLOG.LOGWRN)
+   return acount
+
+#
+# count IMMA records for given date
+#
+def count_daily_attm_variable(cdate):
+
+   tidx = PgIMMA.date2tidx(cdate)
+   if not tidx: return 0
+   PgLOG.pglog("Counting {}.{} for {} from IVADDB".format(PVALS['aname'], PVALS['fname'], cdate), PgLOG.WARNLG)
+   acount = count_table_attm_variable("date = '{}' AND ".format(cdate), tidx)
+   if acount > 0:
+      PgLOG.pglog("{}.{}: {} for {} of ".format(PVALS['aname'], PVALS['fname'], acount, PVALS['wcnd'], cdate), PgLOG.LOGWRN)
+   return acount
+
+#
+# count IMMA records from one table
+#
+def count_table_attm_variable(dcnd, tidx):
+
+   if PVALS['jname']:
+      jtable = "{}_{}".format(PVALS['jname'], tidx)
+      if tidx not in PVALS['jtables']: PVALS['jtables'][tidx] = PgDBI.pgcheck(jtable)
+      if not PVALS['jtables'][tidx]: return 0
+
+   atable = "{}_{}".format(PVALS['aname'], tidx)
+   if tidx not in PVALS['atables']: PVALS['atables'][tidx] = PgDBI.pgcheck(atable)
+   if not PVALS['atables'][tidx]: return 0
+
+   if PVALS['aname'] == 'icoreloc':
+      if PVALS['jname']:
+         table = "{} j, {} n".format(jtable, atable)
+         jcnd = "j.iidx = n.iidx AND "
+      else:
+         table = atable
+         jcnd = ""
+   else:
+      mtable = "icoreloc_{}".format(tidx)
+      if PVALS['jname']:
+         table = "{}m, {} j. {} n".format(mtable, jtable, atable)
+         jcnd = "m.iidx = j.iidx AND m.iidx = n.iidx AND "
+      else:
+         table = "{} m, {} n".format(mtable, atable)
+         cnd = "{} AND m.iidx = n.iidx AND "
+
+   return PgDBI.pgget(table, "", dcnd + jcnd + PVALS['wcnd'], PgLOG.LGEREX)
+
+#
+# initialize the group periods
+#
+def init_periods():
+
+   bdate = PVALS['bdate']
+   
+   if PVALS['group'] == "yearly":
+      dfmt = "YYYY"
+      eflg = "Y"
+   elif PVALS['group'] == 'monthly':
+      dfmt = "YYYY-MM"
+      eflg = "M"
+   else:  # must be daily
+      dfmt = "YYYY-MM-DD"
+      eflg = ""
+
+   pcnt = 0
+   while True:
+      pcnt += 1
+      PVALS['bpdate'].append(bdate)
+      PVALS['period'].append(PgUtil.format_date(bdate, dfmt))
+      edate = PgUtil.enddate(bdate, 0, eflg) if eflg else bdate
+      if PgUtil.diffdate(edate, PVALS['edate']) < 0:
+         PVALS['epdate'].append(edate)
+         bdate = PgUtil.adddate(edate, 0, 0, 1)
+      else:
+         PVALS['epdate'].append(PVALS['edate'])
+         break
+
+   return pcnt
+
+#
+# call main() to start program
+#
+if __name__ == "__main__": main()

rda_python_icoads-1.0.10/src/rda_python_icoads/countsst.py

@@ -0,0 +1,221 @@
+#!/usr/bin/env python3
+#
+##################################################################################
+#
+#     Title : countsst
+#    Author : Zaihua Ji, zji@ucar.edu
+#      Date : 01/09/2021
+#             2025-03-04 transferred to package rda_python_icoads from
+#             https://github.com/NCAR/rda-icoads.git
+#   Purpose : read ICOADS data from IVADDB and count SST records by dail, monthly
+#             or yearly
+#
+#    Github : https://github.com/NCAR/rda-python-icoads.git
+#
+##################################################################################
+
+import sys
+import os
+import re
+from os import path as op
+from rda_python_common import PgLOG
+from rda_python_common import PgDBI
+from rda_python_common import PgUtil
+from . import PgIMMA
+
+PVALS = {
+   'bdate' : None,
+   'edate' : None,
+   'bpdate' : [],
+   'epdate' : [],
+   'period' : [],
+   'group' : None,
+   'oname' : None
+}
+
+#
+# main function to run dsarch
+#
+def main():
+
+   option = None
+   argv = sys.argv[1:]
+
+   for arg in argv:
+      if arg == "-b":
+         PgLOG.PGLOG['BCKGRND'] = 1
+         continue
+      ms = re.match(r'-([go])$', arg)
+      if ms:
+         option = ms.group(1)
+         continue
+      if re.match(r'^-', arg): PgLOG.pglog(arg + ": Invalid Option", PgLOG.LGWNEX)
+      elif option:
+         if option == 'g':
+            PVALS['group'] = arg
+         elif option == 'o':
+            PVALS['oname'] = arg
+         option = ''
+      elif not PVALS['bdate']:
+         PVALS['bdate'] = arg
+      elif not PVALS['edate']:
+         PVALS['edate'] = arg
+      else:
+         PgLOG.pglog(arg + ": Invalid parameter", PgLOG.LGWNEX)
+   
+   PgDBI.ivaddb_dbname()
+
+   if not (PVALS['bdate'] and PVALS['edate'] and re.match(r'^(daily|monthly|yearly)$', PVALS['group'])):
+      pgrec = PgDBI.pgget("cntldb.inventory", "min(date) bdate, max(date) edate", '', PgLOG.LGEREX)
+      print("Usage: countsst -g GroupBy (daily|monthly|yearly) BeginDate EndDate")
+      print("   Group by Daily, Monthly or Yearly is mandatory")
+      print("   Set BeginDate and EndDate between '{}' and '{}'".format(pgrec['bdate'], pgrec['edate']))
+      sys.exit(0)
+   
+   if PgUtil.diffdate(PVALS['bdate'], PVALS['edate']) > 0:
+      tmpdate = PVALS['bdate']
+      PVALS['bdate'] = PVALS['edate']
+      PVALS['edate'] = tmpdate
+
+   PgLOG.PGLOG['LOGFILE'] = "icoads.log"
+   PgLOG.cmdlog("countsst {}".format(' '.join(argv)))
+
+   if not PVALS['oname']:
+      PVALS['oname'] = "SST_COUNTS_{}_{}-{}.txt".format(PVALS['group'].upper(), PVALS['bdate'], PVALS['edate'])
+
+   IMMA = open(PVALS['oname'], 'w')
+   IMMA.write(PVALS['group'] + ", NOCN, ENH, SST, TOTAL\n")
+   count_sst(IMMA)
+   IMMA.close()
+   PgLOG.cmdlog()
+   sys.exit(0)
+
+#
+# count the SST values daily/monthly/yearly
+#
+def count_sst(fd):
+
+   pcnt = init_periods()
+   tcnts = [0]*4
+   getdaily = 1 if PVALS['group'] == 'daily' else 0
+
+   for pidx in range(pcnt):
+      if getdaily:
+         acnts = count_daily_sst(PVALS['bpdate'][pidx])
+      else:
+         acnts = count_period_sst(pidx)
+
+      line = PVALS['period'][pidx]
+      for i in range(4):
+         line += ", {}".format(acnts[i])
+         tcnts[i] += acnts[i]
+      fd.write(line + "\n")
+
+   if pcnt > 1:
+      line = "Total"
+      for i in range(4):
+         line += ", {}".format(tcnts[i])
+      fd.write(line + "\n")
+      PgLOG.pglog("{} total for {} {} periods".format(tcnts[3], pcnt, PVALS['group']), PgLOG.LOGWRN)
+
+#
+# read icoads record from given file name and save them into RDADB
+#
+def count_period_sst(pidx):
+
+   bdate = PVALS['bpdate'][pidx]
+   edate = PVALS['epdate'][pidx]
+   btidx = PgIMMA.date2tidx(bdate)
+   etidx = PgIMMA.date2tidx(edate)
+   tblcnt = etidx-btidx+1
+   acounts = [0]*4
+   acnds = ['']*tblcnt
+   mcnds = ['']*tblcnt
+
+   PgLOG.pglog("Counting SST for {} from IVADDB".format(PVALS['period'][pidx]), PgLOG.WARNLG)
+
+   if tblcnt == 1:
+      acnds[0] = "date BETWEEN '{}' AND '{}'".format(bdate, edate)
+      mcnds[0] = "time BETWEEN '{} 00:00:00' AND '{} 23:59:59'".format(bdate, edate)
+   else:
+      acnds[0] = "date >= '{}'".format(bdate)
+      mcnds[0] = "time >= '{} 00:00:00'".format(bdate)
+      acnds[tblcnt-1] = "date <= '{}'".format(edate)
+      mcnds[tblcnt-1] = "time <= '{} 23:59:59'".format(edate)
+
+   for i in range(tblcnt): count_table_sst(btidx+i, acounts, acnds[i], mcnds[i])
+   if acounts[3]: PgLOG.pglog("{} for {}".format(acounts[3], PVALS['period'][pidx]), PgLOG.LOGWRN)
+
+   return acounts
+
+#
+# count IMMA records for given date
+#
+def count_daily_sst(cdate):
+
+   tidx = PgIMMA.date2tidx(cdate)
+   acounts = [0]*4
+ 
+   PgLOG.pglog("Counting SST for {} from IVADDB".format(cdate), PgLOG.WARNLG)
+   acnd = "date = '{}'".format(cdate)
+   mcnd = "time BETWEEN '{} 00:00:00' AND '{} 23:59:59'".format(cdate, cdate)
+   count_table_sst(tidx, acounts, acnd, mcnd)
+   if acounts[3]: PgLOG.pglog("{} for {}".format(acounts[3], cdate), PgLOG.LOGWRN)
+
+   return acounts
+
+#
+# count IMMA records from one table
+#
+def count_table_sst(tidx, acounts, acnd, mcnd):
+
+   table = "icoreloc_{}".format(tidx)
+   cnt = PgDBI.pgget(table, "", acnd, PgLOG.LGEREX)
+   if not cnt: return acounts
+   acounts[3] += cnt
+
+   table = 'domsdb.idoms_{}'.format(tidx)
+   if mcnd: mcnd += ' AND '
+   mcnd += "sea_water_temperature IS NOT NULL"
+   acounts[2] += PgDBI.pgget(table, "", mcnd, PgLOG.LGEREX)
+   acnd = mcnd + " AND sea_water_temperature_quality = 0"
+   acounts[1] += PgDBI.pgget(table, "", acnd, PgLOG.LGEREX)
+   acnd = mcnd + " AND sea_water_temperature_depth > 0"
+   acounts[0] += PgDBI.pgget(table, "", acnd, PgLOG.LGEREX)
+
+   return acounts
+
+#
+# initialize period list
+#
+def init_periods():
+
+   bdate = PVALS['bdate']   
+   if PVALS['group'] == "yearly":
+      dfmt = "YYYY"
+      eflg = "Y"
+   elif PVALS['group'] == 'monthly':
+      dfmt = "YYYY-MM"
+      eflg = "M"
+   else:  # must be daily
+      dfmt = "YYYY-MM-DD"
+      eflg = ""
+   pcnt = 0
+   while True:
+      pcnt += 1
+      PVALS['bpdate'].append(bdate)
+      PVALS['period'].append(PgUtil.format_date(bdate, dfmt))
+      edate = PgUtil.enddate(bdate, 0, eflg) if eflg else bdate
+      if PgUtil.diffdate(PVALS['edate'], edate) > 0:
+         PVALS['epdate'].append(edate)
+         bdate = PgUtil.adddate(edate, 0, 0, 1)
+      else:
+         PVALS['epdate'].append(PVALS['edate'])
+         break
+
+   return pcnt
+
+#
+# call main() to start program
+#
+if __name__ == "__main__": main()

rda_python_icoads-1.0.10/src/rda_python_icoads/fixiidx.py

@@ -0,0 +1,70 @@
+#!/usr/bin/env python3
+#
+##################################################################################
+#
+#     Title : fixiidx
+#    Author : Zaihua Ji, zji@ucar.edu
+#      Date : 06/23/2023
+#             2025-03-04 transferred to package rda_python_icoads from
+#             https://github.com/NCAR/rda-icoads.git
+#   Purpose : read ICOADS data from IVADDB and fix the iidx values in ireanqc and iivad
+#             attms
+#
+#    Github : https://github.com/NCAR/rda-python-icoads.git
+#
+##################################################################################
+
+import sys
+from rda_python_common import PgLOG
+from rda_python_common import PgDBI
+
+IIDX = {}
+
+def main():
+
+   argv = sys.argv[1:]
+   if argv and argv[0] == "-b": PgLOG.PGLOG['BCKGRND'] = 1
+   PgDBI.ivaddb_dbname()
+   fixiidx('iivad', 8, 24)
+   fixiidx('ireanqc', 1, 25)
+
+
+def fixiidx(aname, t1, t2):
+
+   global IIDX
+   count = 100000
+   while t1 <= t2:
+      IIDX = {}
+      tcnt = fcnt = 0
+      offset = 0
+      tidx = t1
+      t1 += 1
+      tname = "{}_{}".format(aname, tidx)
+      while True:
+         pgrecs = PgDBI.pgmget(tname, 'lidx, iidx, uid', 'OFFSET {} LIMIT {}'.format(offset, count))
+         if not pgrecs: break
+         cnt = len(pgrecs['lidx'])
+         for i in range(cnt):
+            iidx = get_iidx(pgrecs['uid'][i], tidx)
+            if iidx != pgrecs['iidx'][i]:
+               fcnt += PgDBI.pgexec("UPDATE {} SET iidx = {} WHERE lidx = {}".format(tname, iidx, pgrecs['lidx'][i]), PgLOG.LGEREX)         
+         offset += count
+         tcnt += cnt
+         PgLOG.pglog("{}/{} records fixed for {}.iidx".format(fcnt, tcnt, tname), PgLOG.LOGWRN)
+
+
+def get_iidx(uid, tidx):
+
+   if uid not in IIDX:
+      tname = 'iuida_{}'.format(tidx)
+      cnd = "uid = '{}'".format(uid)
+      pgrec = PgDBI.pgget(tname, 'iidx', cnd)
+      if not pgrec: PgLOG.pglog("{}: Error get iidx for {}".format(tname, cnd), PgLOG.LGEREX)
+      IIDX[uid] = pgrec['iidx']
+
+   return IIDX[uid]
+
+#
+# call main() to start program
+#
+if __name__ == "__main__": main()

rda_python_icoads-1.0.10/src/rda_python_icoads/maxsst.py

@@ -0,0 +1,262 @@
+#!/usr/bin/env python3
+#
+##################################################################################
+#
+#     Title : maxsst
+#    Author : Zaihua Ji, zji@ucar.edu
+#      Date : 01/09/2021
+#             2025-03-04 transferred to package rda_python_icoads from
+#             https://github.com/NCAR/rda-icoads.git
+#   Purpose : read ICOADS data from IVADDB and find maximum SST records dailly, monthly
+#             or yearly, and their associated time and locations
+#
+#    Github : https://github.com/NCAR/rda-python-icoads.git
+#
+##################################################################################
+
+import sys
+import os
+import re
+from os import path as op
+from rda_python_common import PgLOG
+from rda_python_common import PgDBI
+from rda_python_common import PgUtil
+from . import PgIMMA
+
+PVALS = {
+   'bdate' : None,
+   'edate' : None,
+   'bpdate' : [],
+   'epdate' : [],
+   'period' : [],
+   'group' : None,
+   'oname' : None
+}
+
+LFLDS = 'yr, mo, dy, hr, lat, lon, id'
+RFLDS = 'r.iidx, r.uid, sst, it, si'
+IFLDS = 'dck, sid, pt'
+TFLDS = ['yr', 'mo', 'dy', 'hr', 'lat', 'lon', 'id','sst', 'it', 'si', 'dck', 'sid', 'pt', 'uid']
+TITLE = ','.join(TFLDS)
+
+MAXSST = 400
+
+#
+# main function
+#
+def main():
+
+   option = None
+   argv = sys.argv[1:]
+
+   for arg in argv:
+      if arg == "-b":
+         PgLOG.PGLOG['BCKGRND'] = 1
+         continue
+      ms = re.match(r'-([go])$', arg)
+      if ms:
+         option = ms.group(1)
+         continue
+      if re.match(r'^-', arg): PgLOG.pglog(arg + ": Invalid Option", PgLOG.LGWNEX)
+      elif option:
+         if option == 'g':
+            PVALS['group'] = arg
+         elif option == 'o':
+            PVALS['oname'] = arg
+         option = ''
+      elif not PVALS['bdate']:
+         PVALS['bdate'] = arg
+      elif not PVALS['edate']:
+         PVALS['edate'] = arg
+      else:
+         PgLOG.pglog(arg + ": Invalid parameter", PgLOG.LGWNEX)
+   
+   PgDBI.ivaddb_dbname()
+
+   if not (PVALS['bdate'] and PVALS['edate'] and re.match(r'^(daily|monthly|yearly|all)$', PVALS['group'])):
+      pgrec = PgDBI.pgget("cntldb.inventory", "min(date) bdate, max(date) edate", '', PgLOG.LGEREX)
+      print("Usage: maxsst -g GroupBy (daily|monthly|yearly|all) BeginDate EndDate")
+      print("   Group by Daily, Monthly, Yearly or All is mandatory")
+      print("   Set BeginDate and EndDate between '{}' and '{}'".format(pgrec['bdate'], pgrec['edate']))
+      sys.exit(0)
+   
+   if PgUtil.diffdate(PVALS['bdate'], PVALS['edate']) > 0:
+      tmpdate = PVALS['bdate']
+      PVALS['bdate'] = PVALS['edate']
+      PVALS['edate'] = tmpdate
+
+   PgLOG.PGLOG['LOGFILE'] = "icoads.log"
+   PgLOG.cmdlog("maxsst {}".format(' '.join(argv)))
+
+   if not PVALS['oname']:
+      PVALS['oname'] = "SST_MAXIMUMS_{}_{}-{}.csv".format(PVALS['group'].upper(), PVALS['bdate'], PVALS['edate'])
+
+   IMMA = open(PVALS['oname'], 'w')
+   IMMA.write("period, {}\n".format(', '.join(TFLDS)))
+   maximum_sst(IMMA)
+   IMMA.close()
+   PgLOG.cmdlog()
+   sys.exit(0)
+
+#
+# maximum the SST values daily/monthly/yearly
+#
+def maximum_sst(fd):
+
+   pmax = init_periods()
+   getdaily = 1 if PVALS['group'] == 'daily' else 0
+
+   for pidx in range(pmax):
+      if getdaily:
+         maxrec = maximum_daily_sst(PVALS['bpdate'][pidx])
+      else:
+         maxrec = maximum_period_sst(pidx)
+      if not maxrec: continue
+      mcnt = len(maxrec['iidx'])
+      for i in range(mcnt):
+         line = PVALS['period'][pidx]
+         for fld in TFLDS:
+            line += ", {}".format(maxrec[fld][i])
+         fd.write(line + "\n")
+
+#
+# read icoads record from given file name and save them into RDADB
+#
+def maximum_period_sst(pidx):
+
+   bdate = PVALS['bpdate'][pidx]
+   edate = PVALS['epdate'][pidx]
+   btidx = PgIMMA.date2tidx(bdate, False)
+   etidx = PgIMMA.date2tidx(edate, True)
+   tblmax = etidx-btidx+1
+   
+   maxrec = {}
+   cnds = ['']*tblmax
+   itable = 'cntldb.inventory'
+   bcnd = "tidx = {} AND date >= '{}'"
+   ecnd = "tidx = {} AND date <= '{}'"
+   brec = PgDBI.pgget(itable, 'min(miniidx) iidx', bcnd.format(btidx, bdate))
+   erec = PgDBI.pgget(itable, 'max(maxiidx) iidx', ecnd.format(etidx, edate))
+
+   PgLOG.pglog("Find MAXIMUM SST for {} from IVADDB".format(PVALS['period'][pidx]), PgLOG.WARNLG)
+
+   if tblmax == 1:
+      if brec and erec['iidx'] >= brec['iidx']:
+        icnd = "iidx BETWEEN {} AND {}".format(brec['iidx'], erec['iidx'])
+        maximum_table_sst(btidx, maxrec, icnd)
+   else:
+      for i in range(tblmax):
+         if i == 0:
+            if brec:
+               icnd = "iidx >= {}".format(brec['iidx'])
+               maximum_table_sst(btidx+i, maxrec, icnd)
+         elif i == tblmax-1:
+            if erec:
+               icnd = "iidx >= {}".format(erec['iidx'])
+               maximum_table_sst(etidx, maxrec, icnd)
+         else:
+            maximum_table_sst(btidx + i, maxrec, '')
+
+   if maxrec:
+       mcnt = len(maxrec['iidx'])
+       s = 's' if mcnt > 1 else ''
+       PgLOG.pglog("MAX SST {}: {} record{} for {}".format(maxrec['sst'][0], mcnt, s, PVALS['period'][pidx]), PgLOG.LOGWRN)
+
+   return maxrec
+
+#
+# maximum IMMA records for given date
+#
+def maximum_daily_sst(cdate):
+
+   tidx = PgIMMA.date2tidx(cdate)
+   maxrec = {}
+   itable = 'cntldb.inventory'
+   PgLOG.pglog("Find MAXIMUM SST for {} from IVADDB".format(cdate), PgLOG.WARNLG)
+   bcnd = "tidx = {} AND date >= '{}'"
+   ecnd = "tidx = {} AND date <= '{}'"
+   brec = PgDBI.pgget(itable, 'min(miniidx) iidx', ecnd.format(tidx, cdate))
+   erec = PgDBI.pgget(itable, 'max(maxiidx) iidx', ecnd.format(tidx, cdate))
+   if brec and erec['iidx'] >= brec['iidx']:
+      icnd = "iidx BETWEEN {} AND {}".format(brec['iidx'], erec['iidx'])
+      maximum_table_sst(tidx, maxrec, icnd)
+      if maxrec:
+          mcnt = len(maxrec['iidx'])
+          s = 's' if mcnt > 1 else ''
+          PgLOG.pglog("MAX SST {}: {} record{} for {}".format(maxrec['sst'][0], mcnt, s, cdate), PgLOG.LOGWRN)
+
+   return maxrec
+
+#
+# maximum IMMA records from one table
+#
+def maximum_table_sst(tidx, maxrec, cnd):
+
+   rtable = "icorereg_{}".format(tidx)
+   ltable = "icoreloc_{}".format(tidx)
+   itable = "iicoads_{}".format(tidx)
+   jtables = "{} r, {} i".format(rtable, itable)
+   if cnd: cnd = 'r.{} AND '.format(cnd)
+   jcnd = cnd + "r.iidx = i.iidx AND pt = 13 AND "
+#   mcnd = jcnd + "sst < {} AND si >= 0 AND it >= 0".format(MAXSST)
+   mcnd = jcnd + "sst < {}".format(MAXSST)
+   if maxrec: mcnd += " AND sst > {}".format(maxrec['sst'][0])
+   srec = PgDBI.pgget(jtables, 'max(sst) sst', mcnd, PgLOG.LGEREX)
+   if srec['sst'] is None: return
+
+#   mcnd = jcnd + "sst = {} AND si >= 0 AND it >= 0".format(srec['sst'])
+   mcnd = jcnd + "sst = {}".format(srec['sst'])
+   srec = PgDBI.pgmget(jtables, RFLDS, mcnd, PgLOG.LGEREX)
+   for fld in srec: maxrec[fld] = srec[fld]
+
+   mcnt = len(srec['iidx'])
+   if mcnt == 1:
+      mcnd = 'iidx = {}'.format(srec['iidx'][0])
+   else:
+      mcnd = 'iidx IN ({})'.format(','.join(map(str, srec['iidx'])))
+
+   srec = PgDBI.pgmget(ltable, LFLDS, mcnd, PgLOG.LGEREX)
+   for fld in srec: maxrec[fld] = srec[fld]
+
+   srec = PgDBI.pgmget(itable, IFLDS, mcnd, PgLOG.LGEREX)
+   for fld in srec: maxrec[fld] = srec[fld]
+
+#
+# initialize period list
+#
+def init_periods():
+
+   bdate = PVALS['bdate']   
+   if PVALS['group'] == "all":
+      PVALS['bpdate'].append(bdate)
+      PVALS['period'].append('ALL')
+      PVALS['epdate'].append(PVALS['edate'])
+      return 1
+   elif PVALS['group'] == "yearly":
+      dfmt = "YYYY"
+      eflg = "Y"
+   elif PVALS['group'] == 'monthly':
+      dfmt = "YYYY-MM"
+      eflg = "M"
+   else:  # must be daily
+      dfmt = "YYYY-MM-DD"
+      eflg = ""
+   pmax = 0
+   while True:
+      pmax += 1
+      PVALS['bpdate'].append(bdate)
+      PVALS['period'].append(PgUtil.format_date(bdate, dfmt))
+      edate = PgUtil.enddate(bdate, 0, eflg) if eflg else bdate
+      if PgUtil.diffdate(PVALS['edate'], edate) > 0:
+         PVALS['epdate'].append(edate)
+         bdate = PgUtil.adddate(edate, 0, 0, 1)
+      else:
+         PVALS['epdate'].append(PVALS['edate'])
+         break
+
+   return pmax
+
+#
+# call main() to start program
+#
+if __name__ == "__main__": main()

{rda_python_icoads-1.0.9 → rda_python_icoads-1.0.10}/src/rda_python_icoads/msg3_subset.py

@@ -108,7 +108,7 @@ def main():
          if PVALS['rdir']: PgLOG.pglog("{}: Request Directory ({}) given already".format(arg, PVALS['rdir']), PgLOG.LGEREX)
          PVALS['rdir'] = arg
 
-   PgLOG.cmdlog("msg3.0_subset {}".format(' '.join(argv)))
+   PgLOG.cmdlog("msg3_subset {}".format(' '.join(argv)))
    PGRQST = PgSubset.valid_subset_request(PVALS['ridx'], PVALS['rdir'], IDSID, PgLOG.LGWNEX)
    if not PVALS['rdir']: PVALS['rdir'] = PgLOG.join_paths(PgLOG.PGLOG['RQSTHOME'], PGRQST['rqstid'])
    process_subset_request()

{rda_python_icoads-1.0.9 → rda_python_icoads-1.0.10}/src/rda_python_icoads.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rda_python_icoads
-Version: 1.0.9
+Version: 1.0.10
 Summary: RDA python package to manage RDA ICOADS datasets
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-icoads

{rda_python_icoads-1.0.9 → rda_python_icoads-1.0.10}/src/rda_python_icoads.egg-info/SOURCES.txt

@@ -8,12 +8,17 @@ src/rda_python_icoads/README_R3.0_Subset.html
 src/rda_python_icoads/__init__.py
 src/rda_python_icoads/checkicoads.py
 src/rda_python_icoads/cleanicoads.py
+src/rda_python_icoads/countattm.py
+src/rda_python_icoads/countattmvar.py
 src/rda_python_icoads/counticoads.py
+src/rda_python_icoads/countsst.py
 src/rda_python_icoads/fillicoads.py
 src/rda_python_icoads/fillinventory.py
 src/rda_python_icoads/fillitable.py
 src/rda_python_icoads/fillmonth.py
+src/rda_python_icoads/fixiidx.py
 src/rda_python_icoads/imma1_subset.py
+src/rda_python_icoads/maxsst.py
 src/rda_python_icoads/msg
 src/rda_python_icoads/msg3.0_subset_readme.txt
 src/rda_python_icoads/msg3_subset.py

{rda_python_icoads-1.0.9 → rda_python_icoads-1.0.10}/src/rda_python_icoads.egg-info/entry_points.txt

@@ -1,12 +1,17 @@
 [console_scripts]
 checkicoads = rda_python_icoads.checkicoads:main
 cleanicoads = rda_python_icoads.cleanicoads:main
+countattm = rda_python_icoads.countattm:main
+countattmvar = rda_python_icoads.countattmvar:main
 counticoads = rda_python_icoads.counticoads:main
+countsst = rda_python_icoads.countsst:main
 fillicoads = rda_python_icoads.fillicoads:main
 fillinventory = rda_python_icoads.fillinventory:main
 fillitable = rda_python_icoads.fillitable:main
 fillmonth = rda_python_icoads.fillmonth:main
+fixidx = rda_python_icoads.fixidx:main
 imma1_subset = rda_python_icoads.imma1_subset:main
+maxsst = rda_python_icoads.maxsst:main
 msg3_subset = rda_python_icoads.msg3_subset:main
 msg_download = rda_python_icoads.msg_download:main
 writeicoads = rda_python_icoads.writeicoads:main