rda-python-icoads 1.0.9__py3-none-any.whl → 1.0.11__py3-none-any.whl

rda_python_icoads/fixiidx.py

@@ -0,0 +1,70 @@
+#!/usr/bin/env python3
+#
+##################################################################################
+#
+#     Title : fixiidx
+#    Author : Zaihua Ji, zji@ucar.edu
+#      Date : 06/23/2023
+#             2025-03-04 transferred to package rda_python_icoads from
+#             https://github.com/NCAR/rda-icoads.git
+#   Purpose : read ICOADS data from IVADDB and fix the iidx values in ireanqc and iivad
+#             attms
+#
+#    Github : https://github.com/NCAR/rda-python-icoads.git
+#
+##################################################################################
+
+import sys
+from rda_python_common import PgLOG
+from rda_python_common import PgDBI
+
+IIDX = {}
+
+def main():
+
+   argv = sys.argv[1:]
+   if argv and argv[0] == "-b": PgLOG.PGLOG['BCKGRND'] = 1
+   PgDBI.ivaddb_dbname()
+   fixiidx('iivad', 8, 24)
+   fixiidx('ireanqc', 1, 25)
+
+
+def fixiidx(aname, t1, t2):
+
+   global IIDX
+   count = 100000
+   while t1 <= t2:
+      IIDX = {}
+      tcnt = fcnt = 0
+      offset = 0
+      tidx = t1
+      t1 += 1
+      tname = "{}_{}".format(aname, tidx)
+      while True:
+         pgrecs = PgDBI.pgmget(tname, 'lidx, iidx, uid', 'OFFSET {} LIMIT {}'.format(offset, count))
+         if not pgrecs: break
+         cnt = len(pgrecs['lidx'])
+         for i in range(cnt):
+            iidx = get_iidx(pgrecs['uid'][i], tidx)
+            if iidx != pgrecs['iidx'][i]:
+               fcnt += PgDBI.pgexec("UPDATE {} SET iidx = {} WHERE lidx = {}".format(tname, iidx, pgrecs['lidx'][i]), PgLOG.LGEREX)         
+         offset += count
+         tcnt += cnt
+         PgLOG.pglog("{}/{} records fixed for {}.iidx".format(fcnt, tcnt, tname), PgLOG.LOGWRN)
+
+
+def get_iidx(uid, tidx):
+
+   if uid not in IIDX:
+      tname = 'iuida_{}'.format(tidx)
+      cnd = "uid = '{}'".format(uid)
+      pgrec = PgDBI.pgget(tname, 'iidx', cnd)
+      if not pgrec: PgLOG.pglog("{}: Error get iidx for {}".format(tname, cnd), PgLOG.LGEREX)
+      IIDX[uid] = pgrec['iidx']
+
+   return IIDX[uid]
+
+#
+# call main() to start program
+#
+if __name__ == "__main__": main()

rda_python_icoads/maxsst.py

@@ -0,0 +1,262 @@
+#!/usr/bin/env python3
+#
+##################################################################################
+#
+#     Title : maxsst
+#    Author : Zaihua Ji, zji@ucar.edu
+#      Date : 01/09/2021
+#             2025-03-04 transferred to package rda_python_icoads from
+#             https://github.com/NCAR/rda-icoads.git
+#   Purpose : read ICOADS data from IVADDB and find maximum SST records dailly, monthly
+#             or yearly, and their associated time and locations
+#
+#    Github : https://github.com/NCAR/rda-python-icoads.git
+#
+##################################################################################
+
+import sys
+import os
+import re
+from os import path as op
+from rda_python_common import PgLOG
+from rda_python_common import PgDBI
+from rda_python_common import PgUtil
+from . import PgIMMA
+
+PVALS = {
+   'bdate' : None,
+   'edate' : None,
+   'bpdate' : [],
+   'epdate' : [],
+   'period' : [],
+   'group' : None,
+   'oname' : None
+}
+
+LFLDS = 'yr, mo, dy, hr, lat, lon, id'
+RFLDS = 'r.iidx, r.uid, sst, it, si'
+IFLDS = 'dck, sid, pt'
+TFLDS = ['yr', 'mo', 'dy', 'hr', 'lat', 'lon', 'id','sst', 'it', 'si', 'dck', 'sid', 'pt', 'uid']
+TITLE = ','.join(TFLDS)
+
+MAXSST = 400
+
+#
+# main function
+#
+def main():
+
+   option = None
+   argv = sys.argv[1:]
+
+   for arg in argv:
+      if arg == "-b":
+         PgLOG.PGLOG['BCKGRND'] = 1
+         continue
+      ms = re.match(r'-([go])$', arg)
+      if ms:
+         option = ms.group(1)
+         continue
+      if re.match(r'^-', arg): PgLOG.pglog(arg + ": Invalid Option", PgLOG.LGWNEX)
+      elif option:
+         if option == 'g':
+            PVALS['group'] = arg
+         elif option == 'o':
+            PVALS['oname'] = arg
+         option = ''
+      elif not PVALS['bdate']:
+         PVALS['bdate'] = arg
+      elif not PVALS['edate']:
+         PVALS['edate'] = arg
+      else:
+         PgLOG.pglog(arg + ": Invalid parameter", PgLOG.LGWNEX)
+   
+   PgDBI.ivaddb_dbname()
+
+   if not (PVALS['bdate'] and PVALS['edate'] and re.match(r'^(daily|monthly|yearly|all)$', PVALS['group'])):
+      pgrec = PgDBI.pgget("cntldb.inventory", "min(date) bdate, max(date) edate", '', PgLOG.LGEREX)
+      print("Usage: maxsst -g GroupBy (daily|monthly|yearly|all) BeginDate EndDate")
+      print("   Group by Daily, Monthly, Yearly or All is mandatory")
+      print("   Set BeginDate and EndDate between '{}' and '{}'".format(pgrec['bdate'], pgrec['edate']))
+      sys.exit(0)
+   
+   if PgUtil.diffdate(PVALS['bdate'], PVALS['edate']) > 0:
+      tmpdate = PVALS['bdate']
+      PVALS['bdate'] = PVALS['edate']
+      PVALS['edate'] = tmpdate
+
+   PgLOG.PGLOG['LOGFILE'] = "icoads.log"
+   PgLOG.cmdlog("maxsst {}".format(' '.join(argv)))
+
+   if not PVALS['oname']:
+      PVALS['oname'] = "SST_MAXIMUMS_{}_{}-{}.csv".format(PVALS['group'].upper(), PVALS['bdate'], PVALS['edate'])
+
+   IMMA = open(PVALS['oname'], 'w')
+   IMMA.write("period, {}\n".format(', '.join(TFLDS)))
+   maximum_sst(IMMA)
+   IMMA.close()
+   PgLOG.cmdlog()
+   sys.exit(0)
+
+#
+# maximum the SST values daily/monthly/yearly
+#
+def maximum_sst(fd):
+
+   pmax = init_periods()
+   getdaily = 1 if PVALS['group'] == 'daily' else 0
+
+   for pidx in range(pmax):
+      if getdaily:
+         maxrec = maximum_daily_sst(PVALS['bpdate'][pidx])
+      else:
+         maxrec = maximum_period_sst(pidx)
+      if not maxrec: continue
+      mcnt = len(maxrec['iidx'])
+      for i in range(mcnt):
+         line = PVALS['period'][pidx]
+         for fld in TFLDS:
+            line += ", {}".format(maxrec[fld][i])
+         fd.write(line + "\n")
+
+#
+# read icoads record from given file name and save them into RDADB
+#
+def maximum_period_sst(pidx):
+
+   bdate = PVALS['bpdate'][pidx]
+   edate = PVALS['epdate'][pidx]
+   btidx = PgIMMA.date2tidx(bdate, False)
+   etidx = PgIMMA.date2tidx(edate, True)
+   tblmax = etidx-btidx+1
+   
+   maxrec = {}
+   cnds = ['']*tblmax
+   itable = 'cntldb.inventory'
+   bcnd = "tidx = {} AND date >= '{}'"
+   ecnd = "tidx = {} AND date <= '{}'"
+   brec = PgDBI.pgget(itable, 'min(miniidx) iidx', bcnd.format(btidx, bdate))
+   erec = PgDBI.pgget(itable, 'max(maxiidx) iidx', ecnd.format(etidx, edate))
+
+   PgLOG.pglog("Find MAXIMUM SST for {} from IVADDB".format(PVALS['period'][pidx]), PgLOG.WARNLG)
+
+   if tblmax == 1:
+      if brec and erec['iidx'] >= brec['iidx']:
+        icnd = "iidx BETWEEN {} AND {}".format(brec['iidx'], erec['iidx'])
+        maximum_table_sst(btidx, maxrec, icnd)
+   else:
+      for i in range(tblmax):
+         if i == 0:
+            if brec:
+               icnd = "iidx >= {}".format(brec['iidx'])
+               maximum_table_sst(btidx+i, maxrec, icnd)
+         elif i == tblmax-1:
+            if erec:
+               icnd = "iidx >= {}".format(erec['iidx'])
+               maximum_table_sst(etidx, maxrec, icnd)
+         else:
+            maximum_table_sst(btidx + i, maxrec, '')
+
+   if maxrec:
+       mcnt = len(maxrec['iidx'])
+       s = 's' if mcnt > 1 else ''
+       PgLOG.pglog("MAX SST {}: {} record{} for {}".format(maxrec['sst'][0], mcnt, s, PVALS['period'][pidx]), PgLOG.LOGWRN)
+
+   return maxrec
+
+#
+# maximum IMMA records for given date
+#
+def maximum_daily_sst(cdate):
+
+   tidx = PgIMMA.date2tidx(cdate)
+   maxrec = {}
+   itable = 'cntldb.inventory'
+   PgLOG.pglog("Find MAXIMUM SST for {} from IVADDB".format(cdate), PgLOG.WARNLG)
+   bcnd = "tidx = {} AND date >= '{}'"
+   ecnd = "tidx = {} AND date <= '{}'"
+   brec = PgDBI.pgget(itable, 'min(miniidx) iidx', ecnd.format(tidx, cdate))
+   erec = PgDBI.pgget(itable, 'max(maxiidx) iidx', ecnd.format(tidx, cdate))
+   if brec and erec['iidx'] >= brec['iidx']:
+      icnd = "iidx BETWEEN {} AND {}".format(brec['iidx'], erec['iidx'])
+      maximum_table_sst(tidx, maxrec, icnd)
+      if maxrec:
+          mcnt = len(maxrec['iidx'])
+          s = 's' if mcnt > 1 else ''
+          PgLOG.pglog("MAX SST {}: {} record{} for {}".format(maxrec['sst'][0], mcnt, s, cdate), PgLOG.LOGWRN)
+
+   return maxrec
+
+#
+# maximum IMMA records from one table
+#
+def maximum_table_sst(tidx, maxrec, cnd):
+
+   rtable = "icorereg_{}".format(tidx)
+   ltable = "icoreloc_{}".format(tidx)
+   itable = "iicoads_{}".format(tidx)
+   jtables = "{} r, {} i".format(rtable, itable)
+   if cnd: cnd = 'r.{} AND '.format(cnd)
+   jcnd = cnd + "r.iidx = i.iidx AND pt = 13 AND "
+#   mcnd = jcnd + "sst < {} AND si >= 0 AND it >= 0".format(MAXSST)
+   mcnd = jcnd + "sst < {}".format(MAXSST)
+   if maxrec: mcnd += " AND sst > {}".format(maxrec['sst'][0])
+   srec = PgDBI.pgget(jtables, 'max(sst) sst', mcnd, PgLOG.LGEREX)
+   if srec['sst'] is None: return
+
+#   mcnd = jcnd + "sst = {} AND si >= 0 AND it >= 0".format(srec['sst'])
+   mcnd = jcnd + "sst = {}".format(srec['sst'])
+   srec = PgDBI.pgmget(jtables, RFLDS, mcnd, PgLOG.LGEREX)
+   for fld in srec: maxrec[fld] = srec[fld]
+
+   mcnt = len(srec['iidx'])
+   if mcnt == 1:
+      mcnd = 'iidx = {}'.format(srec['iidx'][0])
+   else:
+      mcnd = 'iidx IN ({})'.format(','.join(map(str, srec['iidx'])))
+
+   srec = PgDBI.pgmget(ltable, LFLDS, mcnd, PgLOG.LGEREX)
+   for fld in srec: maxrec[fld] = srec[fld]
+
+   srec = PgDBI.pgmget(itable, IFLDS, mcnd, PgLOG.LGEREX)
+   for fld in srec: maxrec[fld] = srec[fld]
+
+#
+# initialize period list
+#
+def init_periods():
+
+   bdate = PVALS['bdate']   
+   if PVALS['group'] == "all":
+      PVALS['bpdate'].append(bdate)
+      PVALS['period'].append('ALL')
+      PVALS['epdate'].append(PVALS['edate'])
+      return 1
+   elif PVALS['group'] == "yearly":
+      dfmt = "YYYY"
+      eflg = "Y"
+   elif PVALS['group'] == 'monthly':
+      dfmt = "YYYY-MM"
+      eflg = "M"
+   else:  # must be daily
+      dfmt = "YYYY-MM-DD"
+      eflg = ""
+   pmax = 0
+   while True:
+      pmax += 1
+      PVALS['bpdate'].append(bdate)
+      PVALS['period'].append(PgUtil.format_date(bdate, dfmt))
+      edate = PgUtil.enddate(bdate, 0, eflg) if eflg else bdate
+      if PgUtil.diffdate(PVALS['edate'], edate) > 0:
+         PVALS['epdate'].append(edate)
+         bdate = PgUtil.adddate(edate, 0, 0, 1)
+      else:
+         PVALS['epdate'].append(PVALS['edate'])
+         break
+
+   return pmax
+
+#
+# call main() to start program
+#
+if __name__ == "__main__": main()

rda_python_icoads/msg3_subset.py

@@ -108,7 +108,7 @@ def main():
          if PVALS['rdir']: PgLOG.pglog("{}: Request Directory ({}) given already".format(arg, PVALS['rdir']), PgLOG.LGEREX)
          PVALS['rdir'] = arg
 
-   PgLOG.cmdlog("msg3.0_subset {}".format(' '.join(argv)))
+   PgLOG.cmdlog("msg3_subset {}".format(' '.join(argv)))
    PGRQST = PgSubset.valid_subset_request(PVALS['ridx'], PVALS['rdir'], IDSID, PgLOG.LGWNEX)
    if not PVALS['rdir']: PVALS['rdir'] = PgLOG.join_paths(PgLOG.PGLOG['RQSTHOME'], PGRQST['rqstid'])
    process_subset_request()

{rda_python_icoads-1.0.9.dist-info → rda_python_icoads-1.0.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: rda_python_icoads
-Version: 1.0.9
+Version: 1.0.11
 Summary: RDA python package to manage RDA ICOADS datasets
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-icoads
@@ -12,6 +12,7 @@ Requires-Python: >=3.7
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: rda_python_common
+Dynamic: license-file
 
 RDA python package to manage RDA ICOADS datasets.
 

{rda_python_icoads-1.0.9.dist-info → rda_python_icoads-1.0.11.dist-info}/RECORD

@@ -1,25 +1,30 @@
-rda_python_icoads/PgIMMA.py,sha256=VJVFXUzOpGhlm-oHV35zkX_I-MMo-pJQt31ZOIlIDeo,58125
+rda_python_icoads/PgIMMA.py,sha256=V3w5A3ZozWBMvBoEuL-XqY6kUetQQodkEAqGb3Uvd2g,58958
 rda_python_icoads/R3.0-stat_doc.pdf,sha256=7f9wKlaTV7ZWXCqyZ4MZ3QcL5pw-Hz623A3xe8QCoFM,59515
 rda_python_icoads/README_R3.0_Subset.html,sha256=tf5Om69KsZZ9ksIjIwtGQUtU8O4U-fqOrIxrl_4tRPo,6878
 rda_python_icoads/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
 rda_python_icoads/checkicoads.py,sha256=nbA09uqaVBf9BHPWWeLFke3sRAGgV7CyOG4VIKI33wY,6599
 rda_python_icoads/cleanicoads.py,sha256=A7JuwYbaVHjVpZFuQZBd9YMQTdMnu7YiqSmbnq058BI,4936
+rda_python_icoads/countattm.py,sha256=syrEKhO-z1Xb-SjtbqHhzSvywuC_72hyex9K_0uUwzg,3578
+rda_python_icoads/countattmvar.py,sha256=BYUSK2qTi0wKHg97gAOye95BJj8c3w728aoumL0C5ns,7498
 rda_python_icoads/counticoads.py,sha256=GB7mRXXQ80Ty_1UF4hyYxrO0IPOeUCq5JUhoxkuvXZs,4555
-rda_python_icoads/fillicoads.py,sha256=xIj9vm3LdhjwQar28B77vfpe7gJDGHCUZ_08zzRnTuo,4239
+rda_python_icoads/countsst.py,sha256=Sx3RicOdEQOXjXrBEUxnTTt6xgoeM2qPLwIMKBBIdps,6362
+rda_python_icoads/fillicoads.py,sha256=9AmMvchdKIundNBzZ4sRC1LJKplD_-qEzmgv6zYCOvQ,4530
 rda_python_icoads/fillinventory.py,sha256=eRNw6laHsF_Ry5SVOJCulDNa4T-9FTT2UObt__Pr3IM,4144
 rda_python_icoads/fillitable.py,sha256=_zCUuYZifMZUoBO8JSY0DcoQOM33smQ1WtWJWnlH3ks,9941
 rda_python_icoads/fillmonth.py,sha256=Abo_5KoX4Zs-vlYcEvovo14xEiO_BNgUxV5fFxT-L3M,2546
+rda_python_icoads/fixiidx.py,sha256=hPOuwB2I46KGE4qWF1f1T6vi7914JICOmE1youneNm4,2036
 rda_python_icoads/imma1_subset.py,sha256=8r130jzm4b9IPjMJBwwRvWrdgxDnXC8KbQjFwiE7LPA,23704
+rda_python_icoads/maxsst.py,sha256=pybyuke-2I2ZJ9EOfp0iyyR3IJig5a5p3PnDG78ArTs,8082
 rda_python_icoads/msg,sha256=J5lkNELaJke718h8VJgqbR0v_nkivStfW0xKz30sVJw,27153
 rda_python_icoads/msg3.0_subset_readme.txt,sha256=Nb3rFq4lkxBDhSzS5XLCgCkZpozUGefy2JkUh1gk0Wo,4144
-rda_python_icoads/msg3_subset.py,sha256=piTGWpsUjbCD2J6q3SylQRlzux5RqW4-ifoSMbvFK_s,13026
+rda_python_icoads/msg3_subset.py,sha256=8pSs1nzwNNg1n8h1S4WHn8phsBN72PnGhkTxxglL8Ms,13024
 rda_python_icoads/msg_download.py,sha256=SqJVdVs6kN-ZLh1d4rSMS4IkRdRrYf5c4Y45zV7pJVQ,6704
 rda_python_icoads/msgsubset.f,sha256=KuM2WAk0ZFEH8LBhHKiGWt8zaW_0ARJXYV2kiD5w5xs,21253
 rda_python_icoads/rdimma1_csv.f,sha256=_hOHvA0UiTykCIsj7ZYkmooNmUq29-p7ZnyjeSyvt8U,22443
 rda_python_icoads/writeicoads.py,sha256=nNgcPTFw957qsBkMVXgrPAkj3qmG8g5b6NeePp4NkNk,5270
-rda_python_icoads-1.0.9.dist-info/LICENSE,sha256=1dck4EAQwv8QweDWCXDx-4Or0S8YwiCstaso_H57Pno,1097
-rda_python_icoads-1.0.9.dist-info/METADATA,sha256=gEgX9ihZd2B002Z7_knUJWgqybCtUa9Fqp6mHcdJjyk,772
-rda_python_icoads-1.0.9.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
-rda_python_icoads-1.0.9.dist-info/entry_points.txt,sha256=AQnzGqKNOCg8X7dRieCscw3tH8_Y05RCw9_ij4hlIOY,557
-rda_python_icoads-1.0.9.dist-info/top_level.txt,sha256=PjJe4PwUIiNxPlNg2d8JNPY1SLSu-_GHAdXe-hUVU1U,18
-rda_python_icoads-1.0.9.dist-info/RECORD,,
+rda_python_icoads-1.0.11.dist-info/licenses/LICENSE,sha256=1dck4EAQwv8QweDWCXDx-4Or0S8YwiCstaso_H57Pno,1097
+rda_python_icoads-1.0.11.dist-info/METADATA,sha256=NifFZKjPfCWq9ZuV3LLLv-ueRsoWP20J2_wYw8_x1UI,795
+rda_python_icoads-1.0.11.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+rda_python_icoads-1.0.11.dist-info/entry_points.txt,sha256=O4wzIJ3tibs6BADt2-9O51suWVggGg3CKVq6PJ2TC1g,774
+rda_python_icoads-1.0.11.dist-info/top_level.txt,sha256=PjJe4PwUIiNxPlNg2d8JNPY1SLSu-_GHAdXe-hUVU1U,18
+rda_python_icoads-1.0.11.dist-info/RECORD,,

{rda_python_icoads-1.0.9.dist-info → rda_python_icoads-1.0.11.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.8.2)
+Generator: setuptools (80.9.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

{rda_python_icoads-1.0.9.dist-info → rda_python_icoads-1.0.11.dist-info}/entry_points.txt

@@ -1,12 +1,17 @@
 [console_scripts]
 checkicoads = rda_python_icoads.checkicoads:main
 cleanicoads = rda_python_icoads.cleanicoads:main
+countattm = rda_python_icoads.countattm:main
+countattmvar = rda_python_icoads.countattmvar:main
 counticoads = rda_python_icoads.counticoads:main
+countsst = rda_python_icoads.countsst:main
 fillicoads = rda_python_icoads.fillicoads:main
 fillinventory = rda_python_icoads.fillinventory:main
 fillitable = rda_python_icoads.fillitable:main
 fillmonth = rda_python_icoads.fillmonth:main
+fixidx = rda_python_icoads.fixidx:main
 imma1_subset = rda_python_icoads.imma1_subset:main
+maxsst = rda_python_icoads.maxsst:main
 msg3_subset = rda_python_icoads.msg3_subset:main
 msg_download = rda_python_icoads.msg_download:main
 writeicoads = rda_python_icoads.writeicoads:main