rda-python-icoads 3.0.3__tar.gz → 3.0.4__tar.gz

{rda_python_icoads-3.0.3/src/rda_python_icoads.egg-info → rda_python_icoads-3.0.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_icoads
-Version: 3.0.3
+Version: 3.0.4
 Summary: RDA python package to manage RDA ICOADS datasets
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-icoads

{rda_python_icoads-3.0.3 → rda_python_icoads-3.0.4}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "rda_python_icoads"
-version = "3.0.3"
+version = "3.0.4"
 authors = [
   { name="Zaihua Ji", email="zji@ucar.edu" },
 ]

{rda_python_icoads-3.0.3 → rda_python_icoads-3.0.4}/src/rda_python_icoads/fillicoads.py

@@ -29,8 +29,7 @@ class FillIcoads(PgIMMA):
          'names' : None,
          'files' : [],
          'dates' : [],
-         'dtlen' : 0,
-         'zipped' : 0
+         'dtlen' : 0
       }
 
    #
@@ -112,12 +111,6 @@ class FillIcoads(PgIMMA):
       fcnt = 0
       tcounts = [0]*self.TABLECOUNT
       for file in self.PVALS['files']:
-         pfile = self.get_partial_month_file(file) if addinventory else None
-         if pfile:
-            fcnt += 1
-            acnts = self.process_imma_file(pfile, addinventory)
-            for i in range(self.TABLECOUNT): tcounts[i] += acnts[i]
-            if self.PVALS['zipped']: self.pgsystem("gzip " + pfile, self.LOGWRN, 5)
          fcnt += 1
          acnts = self.process_imma_file(file, addinventory)
          for i in range(self.TABLECOUNT): tcounts[i] += acnts[i]
@@ -152,49 +145,21 @@ class FillIcoads(PgIMMA):
       return self.format_date(pgrec['mdate']) if pgrec and pgrec['mdate'] else None
 
    #
-   # locate a data file, gunzip it if only the gzipped one is on file
+   # the iidx values are allocated sequentially, so no date can be filled while an earlier one is
+   # still missing; a month must be filled up before the next month is filled, and a month cannot
+   # be skipped; months are sequenced by fillmonth, this only guards a direct call
    #
-   def locate_imma_file(self, fname):
-
-      self.PVALS['zipped'] = 0
-      if op.isfile(fname): return fname
-
-      zfile = fname + ".gz"
-      if op.isfile(zfile):
-         self.pgsystem("gunzip " + zfile, self.LOGWRN, 5)
-         if op.isfile(fname):
-            self.PVALS['zipped'] = 1
-            return fname
-
-      return None
-
-   #
-   # a month is filled up only if its last day is in the inventory; if an earlier month is
-   # filled partially, return the name of its data file, to fill up the missing dates before
-   # the given file is filled
-   #
-   def get_partial_month_file(self, fname):
+   def check_fill_order(self, fdate):
 
       mdate = self.get_max_filled_date()
-      if not mdate: return None   # nothing filled yet
-
-      edate = self.enddate(mdate, 0, 'M')
-      if mdate == edate: return None   # the latest filled month is filled up
-
-      fdate = self.get_imma_file_date(fname)
-      if not fdate or self.diffdate(fdate, edate) <= 0: return None   # the given file holds the partial month
-
-      pgrec = self.pgget(f"{self.CNTLSC}.inventory", "fname", "date = '{}'".format(mdate), self.LGEREX)
-      pfile = self.locate_imma_file(pgrec['fname']) if pgrec and pgrec['fname'] else None
-      if not pfile:
-         self.pglog("{}: month is filled through {} only, and its data file is not found in {} to "
-                    "fill it up before filling {}".format(mdate[0:7], mdate, op.abspath('.'), fdate[0:7]), self.LGEREX)
+      if not mdate: return                              # nothing filled yet
+      if self.diffdate(mdate, fdate) >= 0: return       # the file holds filled dates already
 
-      # the leading '>' propagates the note up through the calling processes, into the email report
-      self.pglog("> {}: filled through {} only, fill it up from {} before filling {}".format(
-                 mdate[0:7], mdate, pfile, fdate[0:7]), self.LOGWRN)
+      rdate = self.adddate(mdate, 0, 0, 1)              # the first missing date
+      if self.diffdate(fdate[0:7] + "-01", rdate) <= 0: return   # the file covers it
 
-      return pfile
+      self.pglog("{}: filled through {} only, fill from {} before filling {}".format(
+                 mdate[0:7], mdate, rdate, fdate[0:7]), self.LGEREX)
 
    #
    # skip the dates that are filled already if the month of the given file is resumed
@@ -245,7 +210,9 @@ class FillIcoads(PgIMMA):
       edate = self.PVALS['dates'][1] if self.PVALS['dtlen'] > 1 else None
 
       fdate = self.get_imma_file_date(fname) if iname else None
-      if fdate: bdate = self.get_resume_date(fname, fdate, bdate)
+      if fdate:
+         if self.PVALS['dtlen'] == 0: self.check_fill_order(fdate)
+         bdate = self.get_resume_date(fname, fdate, bdate)
 
       IMMA = open(fname, 'r', encoding = 'latin_1')
 

{rda_python_icoads-3.0.3 → rda_python_icoads-3.0.4}/src/rda_python_icoads/fillmonth.py

@@ -8,6 +8,7 @@
 #             2025-03-03 transferred to package rda_python_icoads from
 #             https://github.com/NCAR/rda-icoads.git
 #             2026-09-04 convert to class FillMonth
+#             2026-09-09 fill each missing month, in order, up to the given month
 #   Purpose : process ICOADS monthly data file in IMMA1 format and fill into IVADDB
 #
 #    Github : https://github.com/NCAR/rda-python-icoads.git
@@ -18,8 +19,9 @@ import sys
 import re
 from os import path as op
 from rda_python_common.pg_util import PgUtil
+from rda_python_common.pg_dbi import PgDBI
 
-class FillMonth(PgUtil):
+class FillMonth(PgUtil, PgDBI):
 
    def __init__(self):
       super().__init__()
@@ -27,7 +29,8 @@ class FillMonth(PgUtil):
          'filenames' : ["IMMA1_R3.0.2_", "IMMA1_R3.0.3_"],   # IMMA1_R3.0.3_ since 2025-08
          'fillicoads' : "fillicoads -i ",
          'fillitable' : "fillitable -t -v dck pt sid -r ",
-         'cdmsmonth' : "cdmsmonth "
+         'cdmsmonth' : "cdmsmonth ",
+         'inventory' : "cntldb.inventory"
       }
 
    #
@@ -36,7 +39,7 @@ class FillMonth(PgUtil):
    def main(self):
 
       argv = sys.argv[1:]
-      smonth = srange = None
+      smonth = None
 
       for arg in argv:
          if arg == "-b":
@@ -47,7 +50,6 @@ class FillMonth(PgUtil):
             ms = re.match(r'^(\d+)-(\d+)', arg)
             if ms:
                smonth = "{:04}-{:02}".format(int(ms.group(1)), int(ms.group(2)))
-               srange = "{}-01 {}".format(smonth, self.enddate(smonth, 0, 'M'))
             else:
                self.pglog(arg +": Invalid month format", self.LGWNEX)
          else:
@@ -56,14 +58,62 @@ class FillMonth(PgUtil):
       if not smonth:
          print("Usage: fillmonth ProcessMonth")
          print("   Provide a month (YYYY-MM), to fill monthly IMMA1 into IVADDB")
+         print("   Any earlier month not filled up yet is filled first, in order")
          sys.exit(0)
 
       self.PGLOG['LOGFILE'] = "icoads.log"
+      self.ivaddb_dbname()
       self.cmdlog("fillmonth {}".format(' '.join(argv)))
-      self.fill_monthly_data(smonth, srange)
+      months = self.get_fill_months(smonth)
+      if len(months) > 1:
+         self.pglog("> {}: fill {} months, {} through {}, in order".format(
+                    smonth, len(months), months[0][0], months[-1][0]), self.LOGWRN)
+      mcnt = len(months)
+      for i in range(mcnt):
+         (cmonth, bdate, edate) = months[i]
+         self.fill_monthly_data(cmonth, "{} {}".format(bdate, edate))
+         if self.month_filled(cmonth, edate) or (i + 1) == mcnt: continue
+         # a later month cannot be filled while this one is short of dates
+         self.pglog("> {}: filled partially, stop before {}".format(cmonth, months[i+1][0]), self.LOGWRN)
+         break
       self.cmdlog()
       sys.exit(0)
 
+   #
+   # the iidx values are allocated sequentially, so every month between the last filled date
+   # and the given month must be filled, in order; return a [month, bdate, edate] per month,
+   # with bdate on the first month set to resume a partially filled month
+   #
+   def get_fill_months(self, smonth):
+
+      edate = self.enddate(smonth, 0, 'M')
+      pgrec = self.pgget(self.CMDS['inventory'], "max(date) mdate", "", self.LGEREX)
+      mdate = self.format_date(pgrec['mdate']) if pgrec and pgrec['mdate'] else None
+      if not mdate: return [[smonth, smonth + "-01", edate]]   # nothing filled yet
+
+      bdate = self.adddate(mdate, 0, 0, 1)   # resume the partial month, or start the next one
+      if self.diffdate(bdate, edate) > 0: return [[smonth, smonth + "-01", edate]]   # refill an earlier month
+
+      months = []
+      while self.diffdate(bdate, edate) <= 0:
+         cmonth = bdate[0:7]
+         cedate = self.enddate(cmonth, 0, 'M')
+         if self.diffdate(cedate, edate) > 0: cedate = edate
+         months.append([cmonth, bdate, cedate])
+         bdate = self.adddate(cedate, 0, 0, 1)
+
+      return months
+
+   #
+   # check if the given month is filled through its last date
+   #
+   def month_filled(self, cmonth, edate):
+
+      pgrec = self.pgget(self.CMDS['inventory'], "max(date) mdate",
+                         "date BETWEEN '{}-01' AND '{}'".format(cmonth, edate), self.LGEREX)
+
+      return (pgrec and pgrec['mdate'] and self.format_date(pgrec['mdate']) == edate)
+
    #
    # locate the monthly IMMA1 file, gunzip it if only the gzipped one is on file
    #

{rda_python_icoads-3.0.3 → rda_python_icoads-3.0.4/src/rda_python_icoads.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_icoads
-Version: 3.0.3
+Version: 3.0.4
 Summary: RDA python package to manage RDA ICOADS datasets
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-icoads