rda-python-icoads 3.0.2__tar.gz → 3.0.4__tar.gz

{rda_python_icoads-3.0.2/src/rda_python_icoads.egg-info → rda_python_icoads-3.0.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_icoads
-Version: 3.0.2
+Version: 3.0.4
 Summary: RDA python package to manage RDA ICOADS datasets
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-icoads

{rda_python_icoads-3.0.2 → rda_python_icoads-3.0.4}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "rda_python_icoads"
-version = "3.0.2"
+version = "3.0.4"
 authors = [
   { name="Zaihua Ji", email="zji@ucar.edu" },
 ]

{rda_python_icoads-3.0.2 → rda_python_icoads-3.0.4}/src/rda_python_icoads/fillicoads.py

@@ -117,6 +117,85 @@ class FillIcoads(PgIMMA):
 
       if fcnt > 1: self.pglog("{} ({}) filled for {} files".format('/'.join(map(str, tcounts)), self.PVALS['names'], fcnt), self.LOGWRN)
 
+   #
+   # get the first data date in a given imma file
+   #
+   def get_imma_file_date(self, fname):
+
+      fdate = None
+      IMMA = open(fname, 'r', encoding = 'latin_1')
+      line = IMMA.readline()
+      self.identify_attm_name(line)   # the date offset depends on the attm name
+      while line:
+         if not (self.PVALS['uatti'] and line[0:2] == self.PVALS['uatti']):
+            fdate = self.get_imma_date(line)
+            if fdate: break
+         line = IMMA.readline()
+      IMMA.close()
+
+      return fdate
+
+   #
+   # get the latest date filled into the inventory
+   #
+   def get_max_filled_date(self):
+
+      pgrec = self.pgget(f"{self.CNTLSC}.inventory", "max(date) mdate", "", self.LGEREX)
+
+      return self.format_date(pgrec['mdate']) if pgrec and pgrec['mdate'] else None
+
+   #
+   # the iidx values are allocated sequentially, so no date can be filled while an earlier one is
+   # still missing; a month must be filled up before the next month is filled, and a month cannot
+   # be skipped; months are sequenced by fillmonth, this only guards a direct call
+   #
+   def check_fill_order(self, fdate):
+
+      mdate = self.get_max_filled_date()
+      if not mdate: return                              # nothing filled yet
+      if self.diffdate(mdate, fdate) >= 0: return       # the file holds filled dates already
+
+      rdate = self.adddate(mdate, 0, 0, 1)              # the first missing date
+      if self.diffdate(fdate[0:7] + "-01", rdate) <= 0: return   # the file covers it
+
+      self.pglog("{}: filled through {} only, fill from {} before filling {}".format(
+                 mdate[0:7], mdate, rdate, fdate[0:7]), self.LGEREX)
+
+   #
+   # skip the dates that are filled already if the month of the given file is resumed
+   #
+   def get_resume_date(self, fname, fdate, bdate):
+
+      mdate = self.get_max_filled_date()
+      if not mdate: return bdate                                              # nothing filled yet
+      if self.diffdate(mdate, fdate) < 0: return bdate                        # none of this file is filled yet
+      if self.diffdate(mdate, self.enddate(fdate, 0, 'M')) > 0: return bdate  # refilling an earlier month
+
+      rdate = self.adddate(mdate, 0, 0, 1)
+      if bdate and self.diffdate(bdate, rdate) >= 0: return bdate
+
+      self.pglog("> {}: filled through {} already, continue filling from {}".format(
+                 op.basename(fname), mdate, rdate), self.LOGWRN)
+
+      return rdate
+
+   #
+   # log an error, without stopping, if the month of the given file is not filled up, so that
+   # a partial fill is reported by the calling process and the month is resumed on a rerun
+   #
+   def check_month_filled(self, fname, fdate, bdate):
+
+      edate = self.enddate(fdate, 0, 'M')
+      pgrec = self.pgget(f"{self.CNTLSC}.inventory", "max(date) mdate",
+                         "date BETWEEN '{}-01' AND '{}'".format(fdate[0:7], edate), self.LGEREX)
+      mdate = self.format_date(pgrec['mdate']) if pgrec and pgrec['mdate'] else None
+      if mdate == edate:
+         if bdate: self.pglog("> {}: filled up through {}".format(fdate[0:7], edate), self.LOGWRN)
+         return
+
+      self.pglog("{}: PARTIAL fill of {}, filled through {} of {}; rerun to fill the remaining dates".format(
+                 op.basename(fname), fdate[0:7], (mdate if mdate else 'nothing'), edate), self.LOGERR)
+
    #
    # read icoads record from given file name and save them into IVADDB
    #
@@ -125,19 +204,29 @@ class FillIcoads(PgIMMA):
       iname = fname if addinventory else None
       self.pglog("Record IMMA records in File '{}' into IVADDB".format(fname), self.WARNLG)
 
-      IMMA = open(fname, 'r', encoding = 'latin_1')
       acounts = [0]*self.TABLECOUNT
       records = {}
+      bdate = self.PVALS['dates'][0] if self.PVALS['dtlen'] > 0 else None
+      edate = self.PVALS['dates'][1] if self.PVALS['dtlen'] > 1 else None
+
+      fdate = self.get_imma_file_date(fname) if iname else None
+      if fdate:
+         if self.PVALS['dtlen'] == 0: self.check_fill_order(fdate)
+         bdate = self.get_resume_date(fname, fdate, bdate)
+
+      IMMA = open(fname, 'r', encoding = 'latin_1')
 
       # get the first valid date and do initialization
+      cdate = None
       line = IMMA.readline()
       self.identify_attm_name(line)  # check and record standalone attm name
       while line:
-         idate = cdate = self.get_imma_date(line)
-         if cdate and (self.PVALS['dtlen'] == 0 or self.diffdate(cdate, self.PVALS['dates'][0]) >= 0):
-            self.init_indices_for_date(cdate, iname)
-            records = self.get_imma_records(cdate, line, records)
-            break
+         if not (self.PVALS['uatti'] and line[0:2] == self.PVALS['uatti']):   # a standalone attm line holds no date
+            cdate = self.get_imma_date(line)
+            if cdate and (not bdate or self.diffdate(cdate, bdate) >= 0):
+               self.init_indices_for_date(cdate, iname)
+               records = self.get_imma_records(cdate, line, records)
+               break
          line = IMMA.readline()
 
       line = IMMA.readline()
@@ -152,7 +241,7 @@ class FillIcoads(PgIMMA):
                   for i in range(self.TABLECOUNT): acounts[i] += acnts[i]
                   records = {}
                   cdate = idate
-                  if self.PVALS['dtlen'] == 2 and self.diffdate(cdate, self.PVALS['dates'][1]) > 0: break
+                  if edate and self.diffdate(cdate, edate) > 0: break
                   self.init_indices_for_date(cdate, iname)
                records = self.get_imma_records(idate, line, records)
          line = IMMA.readline()
@@ -165,6 +254,8 @@ class FillIcoads(PgIMMA):
 
       self.pglog("{} ({}) filled from {}".format(' '.join(map(str, acounts)), self.PVALS['names'], op.basename(fname)), self.LOGWRN)
 
+      if fdate and self.PVALS['dtlen'] == 0: self.check_month_filled(fname, fdate, bdate)
+
       return acounts
 
 # main function to execute this script

{rda_python_icoads-3.0.2 → rda_python_icoads-3.0.4}/src/rda_python_icoads/fillmonth.py

@@ -8,6 +8,7 @@
 #             2025-03-03 transferred to package rda_python_icoads from
 #             https://github.com/NCAR/rda-icoads.git
 #             2026-09-04 convert to class FillMonth
+#             2026-09-09 fill each missing month, in order, up to the given month
 #   Purpose : process ICOADS monthly data file in IMMA1 format and fill into IVADDB
 #
 #    Github : https://github.com/NCAR/rda-python-icoads.git
@@ -18,8 +19,9 @@ import sys
 import re
 from os import path as op
 from rda_python_common.pg_util import PgUtil
+from rda_python_common.pg_dbi import PgDBI
 
-class FillMonth(PgUtil):
+class FillMonth(PgUtil, PgDBI):
 
    def __init__(self):
       super().__init__()
@@ -27,7 +29,8 @@ class FillMonth(PgUtil):
          'filenames' : ["IMMA1_R3.0.2_", "IMMA1_R3.0.3_"],   # IMMA1_R3.0.3_ since 2025-08
          'fillicoads' : "fillicoads -i ",
          'fillitable' : "fillitable -t -v dck pt sid -r ",
-         'cdmsmonth' : "cdmsmonth "
+         'cdmsmonth' : "cdmsmonth ",
+         'inventory' : "cntldb.inventory"
       }
 
    #
@@ -36,7 +39,7 @@ class FillMonth(PgUtil):
    def main(self):
 
       argv = sys.argv[1:]
-      smonth = srange = None
+      smonth = None
 
       for arg in argv:
          if arg == "-b":
@@ -47,7 +50,6 @@ class FillMonth(PgUtil):
             ms = re.match(r'^(\d+)-(\d+)', arg)
             if ms:
                smonth = "{:04}-{:02}".format(int(ms.group(1)), int(ms.group(2)))
-               srange = "{}-01 {}".format(smonth, self.enddate(smonth, 0, 'M'))
             else:
                self.pglog(arg +": Invalid month format", self.LGWNEX)
          else:
@@ -56,14 +58,62 @@ class FillMonth(PgUtil):
       if not smonth:
          print("Usage: fillmonth ProcessMonth")
          print("   Provide a month (YYYY-MM), to fill monthly IMMA1 into IVADDB")
+         print("   Any earlier month not filled up yet is filled first, in order")
          sys.exit(0)
 
       self.PGLOG['LOGFILE'] = "icoads.log"
+      self.ivaddb_dbname()
       self.cmdlog("fillmonth {}".format(' '.join(argv)))
-      self.fill_monthly_data(smonth, srange)
+      months = self.get_fill_months(smonth)
+      if len(months) > 1:
+         self.pglog("> {}: fill {} months, {} through {}, in order".format(
+                    smonth, len(months), months[0][0], months[-1][0]), self.LOGWRN)
+      mcnt = len(months)
+      for i in range(mcnt):
+         (cmonth, bdate, edate) = months[i]
+         self.fill_monthly_data(cmonth, "{} {}".format(bdate, edate))
+         if self.month_filled(cmonth, edate) or (i + 1) == mcnt: continue
+         # a later month cannot be filled while this one is short of dates
+         self.pglog("> {}: filled partially, stop before {}".format(cmonth, months[i+1][0]), self.LOGWRN)
+         break
       self.cmdlog()
       sys.exit(0)
 
+   #
+   # the iidx values are allocated sequentially, so every month between the last filled date
+   # and the given month must be filled, in order; return a [month, bdate, edate] per month,
+   # with bdate on the first month set to resume a partially filled month
+   #
+   def get_fill_months(self, smonth):
+
+      edate = self.enddate(smonth, 0, 'M')
+      pgrec = self.pgget(self.CMDS['inventory'], "max(date) mdate", "", self.LGEREX)
+      mdate = self.format_date(pgrec['mdate']) if pgrec and pgrec['mdate'] else None
+      if not mdate: return [[smonth, smonth + "-01", edate]]   # nothing filled yet
+
+      bdate = self.adddate(mdate, 0, 0, 1)   # resume the partial month, or start the next one
+      if self.diffdate(bdate, edate) > 0: return [[smonth, smonth + "-01", edate]]   # refill an earlier month
+
+      months = []
+      while self.diffdate(bdate, edate) <= 0:
+         cmonth = bdate[0:7]
+         cedate = self.enddate(cmonth, 0, 'M')
+         if self.diffdate(cedate, edate) > 0: cedate = edate
+         months.append([cmonth, bdate, cedate])
+         bdate = self.adddate(cedate, 0, 0, 1)
+
+      return months
+
+   #
+   # check if the given month is filled through its last date
+   #
+   def month_filled(self, cmonth, edate):
+
+      pgrec = self.pgget(self.CMDS['inventory'], "max(date) mdate",
+                         "date BETWEEN '{}-01' AND '{}'".format(cmonth, edate), self.LGEREX)
+
+      return (pgrec and pgrec['mdate'] and self.format_date(pgrec['mdate']) == edate)
+
    #
    # locate the monthly IMMA1 file, gunzip it if only the gzipped one is on file
    #

{rda_python_icoads-3.0.2 → rda_python_icoads-3.0.4}/src/rda_python_icoads/pg_imma.py

@@ -375,7 +375,7 @@ class PgIMMA(PgOPT):
          self.UIDOFFSET = 4
          self.UIDLENGTH = 15
          atti = line[15:17]
-         CURRN3 = int(line[12])
+         self.CURRN3 = int(line[12])
       else:
          atti = None
 
@@ -622,8 +622,28 @@ class PgIMMA(PgOPT):
    #
    # add multiple imma records into different tables in RDADB
    #
+   # all writes for one date are wrapped in a single transaction, so an interrupted
+   # fill can never leave an inventory record behind without its data; the date is
+   # then simply refilled when the file is processed again
+   #
    def add_imma_records(self, cdate, records):
 
+      self.starttran()
+      try:
+         acnts = self.add_date_records(cdate, records)
+      except BaseException:
+         self.aborttran()
+         self.pglog("{}: records rolled back for the date".format(cdate), self.LOGWRN)
+         raise
+      self.endtran()
+
+      return acnts
+
+   #
+   # add all records of one date into the attm and the control tables
+   #
+   def add_date_records(self, cdate, records):
+
       if self.INVENTORY and self.IMMA_NAMES[0] in records:   # add counting record into inventory table
          rcnt = len(records[self.IMMA_NAMES[0]]['iidx'])
          if rcnt > 0: self.INVENTORY = self.add_inventory_record(self.INVENTORY['fname'], cdate, rcnt, self.INVENTORY)

{rda_python_icoads-3.0.2 → rda_python_icoads-3.0.4/src/rda_python_icoads.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_icoads
-Version: 3.0.2
+Version: 3.0.4
 Summary: RDA python package to manage RDA ICOADS datasets
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-icoads