rda-python-icoads 1.0.19__tar.gz → 3.0.1__tar.gz

rda_python_icoads-3.0.1/PKG-INFO

@@ -0,0 +1,99 @@
+Metadata-Version: 2.4
+Name: rda_python_icoads
+Version: 3.0.1
+Summary: RDA python package to manage RDA ICOADS datasets
+Author-email: Zaihua Ji <zji@ucar.edu>
+Project-URL: Homepage, https://github.com/NCAR/rda-python-icoads
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Development Status :: 5 - Production/Stable
+Requires-Python: >=3.7
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: numpy
+Requires-Dist: rda_python_common>=3.0.14
+Requires-Dist: rda_python_dsrqst>=2.0.4
+Dynamic: license-file
+
+# rda_python_icoads
+
+RDA Python package to manage RDA ICOADS datasets.
+
+## Prerequisite
+
+To dynamically generate a PDF readme fitting for each data subset, the Linux
+commandline utility `wkhtmltopdf` must be installed.  See
+<https://wkhtmltopdf.org/index.html> for download and install instructions.
+
+## Programs
+
+The package installs the following command-line utilities, all running as the
+current user:
+
+- `imma1_subset` — generate IMMA1-format ICOADS subset deliveries
+- `msg3_subset` — generate MSG-format ICOADS R3 subsets
+- `msg_download` — download MSG ICOADS source files
+- `checkicoads` — check ICOADS archive integrity
+- `cleanicoads` — clean up stale ICOADS files
+- `counticoads`, `countattm`, `countattmvar`, `countsst` — record/attribute counters
+- `fillicoads`, `fillinventory`, `fillitable`, `fillmonth` — populate ICOADS DB tables
+- `fixiidx` — fix ICOADS indices
+- `maxsst` — compute maximum SST values
+- `writeicoads` — write ICOADS records
+
+Run any program with no arguments to print its usage details.
+
+## Environment setup
+
+Create a Python environment first; package installs in the next section run
+inside whichever environment you activate here.
+
+### Option A — Python venv (DECS machines)
+
+```bash
+python3 -m venv $ENVHOME          # e.g. /glade/u/home/gdexdata/gdexmsenv
+source $ENVHOME/bin/activate
+```
+
+### Option B — Conda (DAV/Casper)
+
+```bash
+conda create --prefix $ENVHOME python=3.12   # e.g. /glade/work/gdexdata/conda-envs/pg-gdex
+conda activate $ENVHOME
+```
+
+## Installing rda-python-icoads
+
+Pick whichever install mode fits your workflow.  All variants pull in the
+transitive dependency (`rda_python_common`) automatically.
+
+For local development, clone this repo alongside your project and install it
+in editable mode so that changes are picked up without re-installing:
+
+```bash
+git clone https://github.com/NCAR/rda-python-icoads.git
+cd rda-python-icoads
+pip install -e .
+```
+
+To test a specific branch (e.g. an in-progress feature or fix branch), pass
+`-b/--branch` to `git clone`:
+
+```bash
+git clone -b <branch-name> https://github.com/NCAR/rda-python-icoads.git
+cd rda-python-icoads
+pip install -e .
+```
+
+For a regular (non-editable) install from a checkout:
+
+```bash
+pip install /path/to/rda-python-icoads
+```
+
+For a production install on a system that uses the published distribution:
+
+```bash
+pip install rda_python_icoads
+```

rda_python_icoads-3.0.1/README.md

@@ -0,0 +1,81 @@
+# rda_python_icoads
+
+RDA Python package to manage RDA ICOADS datasets.
+
+## Prerequisite
+
+To dynamically generate a PDF readme fitting for each data subset, the Linux
+commandline utility `wkhtmltopdf` must be installed.  See
+<https://wkhtmltopdf.org/index.html> for download and install instructions.
+
+## Programs
+
+The package installs the following command-line utilities, all running as the
+current user:
+
+- `imma1_subset` — generate IMMA1-format ICOADS subset deliveries
+- `msg3_subset` — generate MSG-format ICOADS R3 subsets
+- `msg_download` — download MSG ICOADS source files
+- `checkicoads` — check ICOADS archive integrity
+- `cleanicoads` — clean up stale ICOADS files
+- `counticoads`, `countattm`, `countattmvar`, `countsst` — record/attribute counters
+- `fillicoads`, `fillinventory`, `fillitable`, `fillmonth` — populate ICOADS DB tables
+- `fixiidx` — fix ICOADS indices
+- `maxsst` — compute maximum SST values
+- `writeicoads` — write ICOADS records
+
+Run any program with no arguments to print its usage details.
+
+## Environment setup
+
+Create a Python environment first; package installs in the next section run
+inside whichever environment you activate here.
+
+### Option A — Python venv (DECS machines)
+
+```bash
+python3 -m venv $ENVHOME          # e.g. /glade/u/home/gdexdata/gdexmsenv
+source $ENVHOME/bin/activate
+```
+
+### Option B — Conda (DAV/Casper)
+
+```bash
+conda create --prefix $ENVHOME python=3.12   # e.g. /glade/work/gdexdata/conda-envs/pg-gdex
+conda activate $ENVHOME
+```
+
+## Installing rda-python-icoads
+
+Pick whichever install mode fits your workflow.  All variants pull in the
+transitive dependency (`rda_python_common`) automatically.
+
+For local development, clone this repo alongside your project and install it
+in editable mode so that changes are picked up without re-installing:
+
+```bash
+git clone https://github.com/NCAR/rda-python-icoads.git
+cd rda-python-icoads
+pip install -e .
+```
+
+To test a specific branch (e.g. an in-progress feature or fix branch), pass
+`-b/--branch` to `git clone`:
+
+```bash
+git clone -b <branch-name> https://github.com/NCAR/rda-python-icoads.git
+cd rda-python-icoads
+pip install -e .
+```
+
+For a regular (non-editable) install from a checkout:
+
+```bash
+pip install /path/to/rda-python-icoads
+```
+
+For a production install on a system that uses the published distribution:
+
+```bash
+pip install rda_python_icoads
+```

{rda_python_icoads-1.0.19 → rda_python_icoads-3.0.1}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "rda_python_icoads"
-version = "1.0.19"
+version = "3.0.1"
 authors = [
   { name="Zaihua Ji", email="zji@ucar.edu" },
 ]
@@ -20,7 +20,9 @@ classifiers = [
     "Development Status :: 5 - Production/Stable",
 ]
 dependencies = [
-  "rda_python_common",
+  "numpy",
+  "rda_python_common>=3.0.14",
+  "rda_python_dsrqst>=2.0.4",
 ]
 
 [tool.setuptools]
@@ -55,7 +57,7 @@ where = ["src"]
 "fillinventory" = "rda_python_icoads.fillinventory:main"
 "fillitable" = "rda_python_icoads.fillitable:main"
 "fillmonth" = "rda_python_icoads.fillmonth:main"
-"fixidx" = "rda_python_icoads.fixidx:main"
+"fixiidx" = "rda_python_icoads.fixiidx:main"
 "msg_download" = "rda_python_icoads.msg_download:main"
 "maxsst" = "rda_python_icoads.maxsst:main"
 "writeicoads" = "rda_python_icoads.writeicoads:main"

rda_python_icoads-3.0.1/src/rda_python_icoads/checkicoads.py

@@ -0,0 +1,224 @@
+#!/usr/bin/env python3
+#
+##################################################################################
+#
+#     Title : checkicoads
+#    Author : Zaihua Ji, zji@ucar.edu
+#      Date : 12/30/2020
+#             2025-03-03 transferred to package rda_python_icoads from
+#             https://github.com/NCAR/rda-icoads.git
+#             2026-09-04 convert to class CheckIcoads
+#   Purpose : check and compare ICOADS data files and IVADDB records
+#
+#    Github : https://github.com/NCAR/rda-python-icoads.git
+#
+##################################################################################
+
+import sys
+import os
+import re
+from os import path as op
+from .pg_imma import PgIMMA
+
+class CheckIcoads(PgIMMA):
+
+   def __init__(self):
+      super().__init__()
+      self.PVALS = {
+         'bdate' : None,
+         'edate' : None,
+         'bmdate' : [],
+         'emdate' : [],
+         'fname' : [],
+         'flag' : [],   # 1 - file exists, 2 - db records exist, 3 - both
+         'mproc' : 10,
+         'fpattern' : "IMMA1_R3.0.0_<YYYY-MM>",
+         'readall' : 0
+      }
+
+   #
+   # main function to run dsarch
+   #
+   def main(self):
+
+      option = ''
+      argv = sys.argv[1:]
+
+      for arg in argv:
+         if arg == "-b":
+            self.PGLOG['BCKGRND'] = 1
+         elif arg == "-a":
+            self.PVALS['readall'] = 1
+         elif arg == "-f":
+            option = 'f'
+         elif arg == "-m":
+            option = 'm'
+         elif re.match(r'^-', arg):
+            self.pglog(arg + ": Invalid Option", self.LGWNEX)
+         elif option:
+            if option == 'f':
+               self.PVALS['fpattern'] = arg
+            elif option == 'm':
+               self.PVALS['mproc'] = arg
+            option = ''
+         elif not self.PVALS['bdate']:
+            self.PVALS['bdate'] = arg
+         elif not self.PVALS['edate']:
+            self.PVALS['edate'] = arg
+         else:
+            self.pglog(arg + ": Invalid parameter", self.LGWNEX)
+
+      self.PGLOG['LOGFILE'] = "icoads.log"
+      self.ivaddb_dbname()
+
+      if not (self.PVALS['bdate'] and self.PVALS['edate']):
+         pgrec = self.pgget("cntldb.inventory", "min(date) bdate, max(date) edate", '', self.LGEREX)
+         print("Usage: checkicoads [-a] [-m mproc] [-f FilePattern] BeginDate EndDate")
+         print("   Default FilePattern is " + self.PVALS['fpattern'])
+         print("   Option -a - read all attms, including multi-line ones, such as IVAD and REANQC")
+         print("   Option -m - start up to given number of processes, one for each month (Default to 10)")
+         print("   Set BeginDate and EndDate between '{}' and '{}'".format(pgrec['bdate'], pgrec['edate']))
+         sys.exit(0)
+
+      if self.diffdate(self.PVALS['bdate'], self.PVALS['edate']) > 0:
+         tmpdate = self.PVALS['bdate']
+         self.PVALS['bdate'] = self.PVALS['edate']
+         self.PVALS['edate'] = tmpdate
+
+      self.cmdlog("checkicoads {}".format(' '.join(argv)))
+      self.check_imma_data()
+      self.cmdlog()
+      sys.exit(0)
+
+   #
+   # check imma data
+   #
+   def check_imma_data(self):
+
+      mcnt = self.init_months()
+      if mcnt == 1: self.PVALS['mproc'] = 1
+      if self.PVALS['mproc'] > 1:
+         self.start_none_daemon('writeicoads', '', self.PGLOG['CURUID'], self.PVALS['mproc'], 300, 1)
+
+      for midx in range(mcnt):
+         fname = self.PVALS['fname'][midx]
+         if op.isfile(fname +".cnt"): continue    # monthly file counted already
+         if self.PVALS['mproc'] > 1:
+            stat = self.start_child("checkicoads_{}".format(midx), self.LOGWRN, 1)  # try to start a child process
+            if stat <= 0:
+               sys.exit(1)   # something wrong
+            elif self.PGSIG['PPID'] > 1:
+               self.check_imma_file(fname, midx)
+               sys.exit(0)  # stop child process
+            else:
+               self.pgdisconnect(0)  # disconnect database for reconnection
+               continue   # continue for next midx
+         else:
+            self.check_imma_file(fname, midx)
+
+      if self.PVALS['mproc'] > 1: self.check_child(None, 0, self.LOGWRN, 1)
+
+      self.dump_final_counts()
+
+   #
+   # compare icoads records from given file name and IVADDB
+   #
+   def check_imma_file(self, fname, midx):
+
+      self.pglog("Count IMMA records in File '{}'".format(fname), self.WARNLG)
+      flag = self.PVALS['flag'][midx]
+
+      acnts = [0]*self.TABLECOUNT
+      acounts = [0]*self.TABLECOUNT
+
+      if flag&1:
+         IMMA = open(fname, 'r')
+         line = IMMA.readline()   
+         while line:
+            if self.PVALS['readall'] and re.match(r'^98', line):
+                self.get_imma_multiple_counts(line, acnts)
+            else:
+               self.get_imma_counts(line, acnts)
+            line.IMMA.readline()
+         IMMA.close()
+         for i in range(self.TABLECOUNT): acounts[i] = acnts[i]
+
+      if flag&2:
+         self.pglog("Count IMMA records in in IVADDB", self.WARNLG)
+         cdate = bdate = self.PVALS['bmdate'][midx]
+         edate = self.PVALS['emdate'][midx]
+         while cdate <= edate:
+            acnts = self.count_imma_records(cdate, 0, self.PVALS['readall'])
+            cdate = self.adddate(cdate, 0, 0, 1)
+            if acnts:
+               for i in range(self.TABLECOUNT): acounts[i] -= acnts[i]
+
+      self.dump_monthly_counts(fname, acounts)
+
+   #
+   # dump monthly counts
+   #
+   def dump_monthly_counts(self, fname, acounts):
+
+      oname = fname + ".cnt"
+      IMMA = open(oname, 'w')
+      IMMA.write("{}, {}\n".format(fname, ', '.join(acounts)))
+      IMMA.close()
+
+   #
+   # concat all monthly counts into one
+   #
+   def dump_final_counts(self):
+
+      fname = "ICOADS_DIFF_COUNTS.csv"
+
+      IMMA = open(fname, 'w')
+      IMMA.write("FileName, {}\n".format(', '.join(self.IMMA_NAMES)))
+      IMMA.close()
+      self.pgsystem("cat *.cnt >> " + fname)
+
+   #
+   # initialize the month list
+   #
+   def init_months(self):
+
+      seps = ["<" , ">"];  # temporal pattern delimiters
+      match = "[^" + seps[1] + "]+"
+
+      ms = re.search(r'{}({}){}'.format(seps[0], match, seps[1]), self.PVALS['fpattern'])
+      if ms:
+         tpattern = ms.group(1)
+         treplace = "{}{}{}".format(seps[0], tpattern, seps[1])
+      else:
+         self.pglog(self.PVALS['fpattern'] + ": Not temporal pattern found to get month list", self.LGEREX)
+
+      bdate = self.PVALS['bdate']
+      done = midx = 0
+      while True:
+         edate = self.enddate(bdate, 0, 'M')
+         if self.diffdate(self.PVALS['edate'], edate) <= 0:
+            edate = self.PVALS['edate']
+            done = 1
+         mdate = self.format_date(bdate, tpattern)
+         fname = self.PVALS['fpattern'].replace(treplace, mdate)
+         flag = 0
+         if op.isfile(fname): flag += 1
+         if self.pgget("cntldb.inventory", "", "date BETWEEN '{}' AND '{}'".format(bdate, edate), self.LGEREX):
+            flag += 2
+         if flag:
+            self.PVALS['bmdate'].append(bdate)
+            self.PVALS['emdate'].append(edate)
+            self.PVALS['fname'].append(fname)
+            self.PVALS['flag'].append(flag)
+            midx += 1
+         if done: break
+         bdate = self.adddate(edate, 0, 0, 1)
+
+      return midx
+
+# main function to execute this script
+def main():
+   CheckIcoads().main()
+
+# call main() to start program
+if __name__ == "__main__": main()

rda_python_icoads-3.0.1/src/rda_python_icoads/cleanicoads.py

@@ -0,0 +1,180 @@
+#!/usr/bin/env python3
+#
+##################################################################################
+#
+#     Title : cleanicoads
+#    Author : Zaihua Ji, zji@ucar.edu
+#      Date : 12/30/2020
+#             2025-03-03 transferred to package rda_python_icoads from
+#             https://github.com/NCAR/rda-icoads.git
+#             2026-09-04 convert to class CleanIcoads
+#   Purpose : clean up one or all IMMA1 attms in IVADDB for given period
+#
+#    Github : https://github.com/NCAR/rda-python-icoads.git
+#
+##################################################################################
+
+import sys
+import os
+import re
+from os import path as op
+from .pg_imma import PgIMMA
+
+class CleanIcoads(PgIMMA):
+
+   def __init__(self):
+      super().__init__()
+      self.PVALS = {
+         'bdate' : None,
+         'edate' : None,
+         'aname' : None,
+         'tinfo' : {},
+         'tcnt' : 0,
+         'dcnd' : None,
+         'uatti' : '',
+         'names' : None
+      }
+
+   #
+   # main function to run dsarch
+   #
+   def main(self):
+
+      option = ''
+      files = []
+      leaduid = 0
+      chkexist = 0
+      readall = 0
+      argv = sys.argv[1:]
+
+      for arg in argv:
+         if arg == "-b":
+            self.PGLOG['BCKGRND'] = 1
+         elif arg == "-a":
+            option = 'a'
+         elif re.match(r'^-', arg):
+            self.pglog(arg + ": Invalid Option", self.LGWNEX)
+         elif option:
+            self.PVALS['aname'] = arg
+            option = ''
+         elif not self.PVALS['bdate']:
+            self.PVALS['bdate'] = arg
+         elif not self.PVALS['edate']:
+            self.PVALS['edate'] = arg
+         else:
+            self.pglog(arg + ": Invalid parameter", self.LGWNEX)
+
+      if not self.PVALS['bdate']:
+         print("Usage: cleanicoads [-a ATTNAME] BDATE EDATE")
+         print("   Option -a - clean a single attm for given attm name")
+         self.pgexit()
+
+      self.PGLOG['LOGFILE'] = "icoads.log"
+      self.set_scname(dbname = 'ivaddb', scname = self.IVADSC, lnname = 'ivaddb', dbhost = self.PGLOG['PMISCHOST'])
+      self.cmdlog("cleanicoads {}".format(' '.join(argv)))
+      self.set_table_info()
+      self.clean_imma_data()   
+      self.cmdlog()
+      self.pgexit()
+
+   #
+   # set the table index list
+   #
+   def set_table_info(self):
+
+      table = f"{self.CNTLSC}.inventory"
+      if self.PVALS['edate']:
+         self.PVALS['dcnd'] = "date BETWEEN '{}' AND '{}'".format(self.PVALS['bdate'], self.PVALS['edate'])
+      else:
+         self.PVALS['dcnd'] = "date >= '{}'".format(self.PVALS['bdate'])
+
+      self.PVALS['tinfo'] = self.pgmget(table, "tidx, min(miniidx) bidx, max(maxiidx) eidx", self.PVALS['dcnd'] + " GROUP BY tidx", self.LGEREX)
+      self.PVALS['tcnt'] = len(self.PVALS['tinfo']['tidx']) if self.PVALS['tinfo'] else 0
+
+      if not self.PVALS['tcnt']:
+         self.pglog("{}: No data found in IVADDB for {}".format(table, self.PVALS['dcnd']), self.LGEREX)   
+
+   #
+   # clean up imma data
+   #
+   def clean_imma_data(self):
+
+      table = f"{self.CNTLSC}.inventory"
+
+      for i in range(self.PVALS['tcnt']):
+         tidx = self.PVALS['tinfo']['tidx'][i]
+         cnd = "iidx BETWEEN {} AND {}".format(self.PVALS['tinfo']['bidx'][i], self.PVALS['tinfo']['eidx'][i])
+         if self.PVALS['aname']:
+            self.clean_one_attm_for_tidx(self.PVALS['aname'], tidx, cnd)
+         else:
+            self.clean_imma_data_for_tidx(tidx, cnd)
+
+      if not self.PVALS['aname']:
+         cnt = self.pgdel(table, self.PVALS['dcnd'], self.LGEREX)
+         s = 's' if cnt > 1 else ''
+         self.pglog("{}: {} record{} deleted for {}".format(table, cnt, s, self.PVALS['dcnd']), self.LOGWRN)
+
+   #
+   # clean up imma data for table index
+   #
+   def clean_imma_data_for_tidx(self, tidx, cnd):
+
+      self.pglog("Clean IMMA data for table index {}...".format(tidx), self.LOGWRN)
+
+      for i in range(self.TABLECOUNT):
+         aname = self.IMMA_NAMES[i]
+         self.clean_one_attm_for_tidx(aname, tidx, cnd)
+
+   #
+   # clean up one attm data for table index
+   #
+   def clean_one_attm_for_tidx(self, aname, tidx, cnd):
+
+      table = f"{self.IVADSC}.{aname}_{tidx}"
+      if not self.pgcheck(table): return 0  # not record to delete
+
+      if aname == 'iuida': self.clean_itidx_for_tidx(table, cnd) 
+
+      cnt = self.pgdel(table, cnd, self.LGEREX)
+      s = 's' if cnt > 1 else ''
+      self.pglog("{}: {} record{} deleted for {}".format(table, cnt, s, cnd), self.LOGWRN)
+
+      cnt = self.pgget(table, "", "", self.LGEREX)
+      self.clean_iattm_for_tidx(aname, tidx, cnt)
+
+   #
+   # clean up table itidx for table index
+   #
+   def clean_itidx_for_tidx(self, table, cnd):
+
+      tname = f"{self.CNTLSC}.itidx"
+      uids = self.pgmget(table, "distinct (substring(uid, 1, 2)) uida", cnd, self.LGEREX)
+      ucnt = len(uids['uida']) if uids else 0
+      for i in range(ucnt):
+         table = "{}_{}".format(tname, uids['uida'][i].lower())
+         if not self.pgcheck(table): continue
+         cnt = self.pgdel(table, cnd, self.LOGWRN)
+         s = 's' if cnt > 1 else ''
+         self.pglog("{}: {} record{} deleted".format(table, cnt, s), self.LOGWRN)
+
+   #
+   # clean up table iattm for table index
+   #
+   def clean_iattm_for_tidx(self, aname, tidx, cnt):
+
+      table = f"{self.CNTLSC}.iattm"
+      cnd = "attm = '{}' AND tidx = {}".format(aname, tidx)
+      pgrec = {'count' : cnt}
+      self.pgupdt(table, pgrec, cnd, self.LGWNEX)
+      self.pglog("{}: Set count to {} for {}".format(table, cnt, cnd), self.LOGWRN)
+
+      table += "_daily"
+      cnd += " AND " + self.PVALS['dcnd']
+      self.pgdel(table, self.PVALS['dcnd'], self.LGWNEX)
+
+# main function to execute this script
+def main():
+   CleanIcoads().main()
+
+# call main() to start program
+if __name__ == "__main__": main()