rda-python-icoads 3.0.1__py3-none-any.whl → 3.0.3__py3-none-any.whl

rda_python_icoads/fillicoads.py

@@ -29,7 +29,8 @@ class FillIcoads(PgIMMA):
          'names' : None,
          'files' : [],
          'dates' : [],
-         'dtlen' : 0
+         'dtlen' : 0,
+         'zipped' : 0
       }
 
    #
@@ -111,12 +112,125 @@ class FillIcoads(PgIMMA):
       fcnt = 0
       tcounts = [0]*self.TABLECOUNT
       for file in self.PVALS['files']:
+         pfile = self.get_partial_month_file(file) if addinventory else None
+         if pfile:
+            fcnt += 1
+            acnts = self.process_imma_file(pfile, addinventory)
+            for i in range(self.TABLECOUNT): tcounts[i] += acnts[i]
+            if self.PVALS['zipped']: self.pgsystem("gzip " + pfile, self.LOGWRN, 5)
          fcnt += 1
          acnts = self.process_imma_file(file, addinventory)
          for i in range(self.TABLECOUNT): tcounts[i] += acnts[i]
 
       if fcnt > 1: self.pglog("{} ({}) filled for {} files".format('/'.join(map(str, tcounts)), self.PVALS['names'], fcnt), self.LOGWRN)
 
+   #
+   # get the first data date in a given imma file
+   #
+   def get_imma_file_date(self, fname):
+
+      fdate = None
+      IMMA = open(fname, 'r', encoding = 'latin_1')
+      line = IMMA.readline()
+      self.identify_attm_name(line)   # the date offset depends on the attm name
+      while line:
+         if not (self.PVALS['uatti'] and line[0:2] == self.PVALS['uatti']):
+            fdate = self.get_imma_date(line)
+            if fdate: break
+         line = IMMA.readline()
+      IMMA.close()
+
+      return fdate
+
+   #
+   # get the latest date filled into the inventory
+   #
+   def get_max_filled_date(self):
+
+      pgrec = self.pgget(f"{self.CNTLSC}.inventory", "max(date) mdate", "", self.LGEREX)
+
+      return self.format_date(pgrec['mdate']) if pgrec and pgrec['mdate'] else None
+
+   #
+   # locate a data file, gunzip it if only the gzipped one is on file
+   #
+   def locate_imma_file(self, fname):
+
+      self.PVALS['zipped'] = 0
+      if op.isfile(fname): return fname
+
+      zfile = fname + ".gz"
+      if op.isfile(zfile):
+         self.pgsystem("gunzip " + zfile, self.LOGWRN, 5)
+         if op.isfile(fname):
+            self.PVALS['zipped'] = 1
+            return fname
+
+      return None
+
+   #
+   # a month is filled up only if its last day is in the inventory; if an earlier month is
+   # filled partially, return the name of its data file, to fill up the missing dates before
+   # the given file is filled
+   #
+   def get_partial_month_file(self, fname):
+
+      mdate = self.get_max_filled_date()
+      if not mdate: return None   # nothing filled yet
+
+      edate = self.enddate(mdate, 0, 'M')
+      if mdate == edate: return None   # the latest filled month is filled up
+
+      fdate = self.get_imma_file_date(fname)
+      if not fdate or self.diffdate(fdate, edate) <= 0: return None   # the given file holds the partial month
+
+      pgrec = self.pgget(f"{self.CNTLSC}.inventory", "fname", "date = '{}'".format(mdate), self.LGEREX)
+      pfile = self.locate_imma_file(pgrec['fname']) if pgrec and pgrec['fname'] else None
+      if not pfile:
+         self.pglog("{}: month is filled through {} only, and its data file is not found in {} to "
+                    "fill it up before filling {}".format(mdate[0:7], mdate, op.abspath('.'), fdate[0:7]), self.LGEREX)
+
+      # the leading '>' propagates the note up through the calling processes, into the email report
+      self.pglog("> {}: filled through {} only, fill it up from {} before filling {}".format(
+                 mdate[0:7], mdate, pfile, fdate[0:7]), self.LOGWRN)
+
+      return pfile
+
+   #
+   # skip the dates that are filled already if the month of the given file is resumed
+   #
+   def get_resume_date(self, fname, fdate, bdate):
+
+      mdate = self.get_max_filled_date()
+      if not mdate: return bdate                                              # nothing filled yet
+      if self.diffdate(mdate, fdate) < 0: return bdate                        # none of this file is filled yet
+      if self.diffdate(mdate, self.enddate(fdate, 0, 'M')) > 0: return bdate  # refilling an earlier month
+
+      rdate = self.adddate(mdate, 0, 0, 1)
+      if bdate and self.diffdate(bdate, rdate) >= 0: return bdate
+
+      self.pglog("> {}: filled through {} already, continue filling from {}".format(
+                 op.basename(fname), mdate, rdate), self.LOGWRN)
+
+      return rdate
+
+   #
+   # log an error, without stopping, if the month of the given file is not filled up, so that
+   # a partial fill is reported by the calling process and the month is resumed on a rerun
+   #
+   def check_month_filled(self, fname, fdate, bdate):
+
+      edate = self.enddate(fdate, 0, 'M')
+      pgrec = self.pgget(f"{self.CNTLSC}.inventory", "max(date) mdate",
+                         "date BETWEEN '{}-01' AND '{}'".format(fdate[0:7], edate), self.LGEREX)
+      mdate = self.format_date(pgrec['mdate']) if pgrec and pgrec['mdate'] else None
+      if mdate == edate:
+         if bdate: self.pglog("> {}: filled up through {}".format(fdate[0:7], edate), self.LOGWRN)
+         return
+
+      self.pglog("{}: PARTIAL fill of {}, filled through {} of {}; rerun to fill the remaining dates".format(
+                 op.basename(fname), fdate[0:7], (mdate if mdate else 'nothing'), edate), self.LOGERR)
+
    #
    # read icoads record from given file name and save them into IVADDB
    #
@@ -125,19 +239,27 @@ class FillIcoads(PgIMMA):
       iname = fname if addinventory else None
       self.pglog("Record IMMA records in File '{}' into IVADDB".format(fname), self.WARNLG)
 
-      IMMA = open(fname, 'r', encoding = 'latin_1')
       acounts = [0]*self.TABLECOUNT
       records = {}
+      bdate = self.PVALS['dates'][0] if self.PVALS['dtlen'] > 0 else None
+      edate = self.PVALS['dates'][1] if self.PVALS['dtlen'] > 1 else None
+
+      fdate = self.get_imma_file_date(fname) if iname else None
+      if fdate: bdate = self.get_resume_date(fname, fdate, bdate)
+
+      IMMA = open(fname, 'r', encoding = 'latin_1')
 
       # get the first valid date and do initialization
+      cdate = None
       line = IMMA.readline()
       self.identify_attm_name(line)  # check and record standalone attm name
       while line:
-         idate = cdate = self.get_imma_date(line)
-         if cdate and (self.PVALS['dtlen'] == 0 or self.diffdate(cdate, self.PVALS['dates'][0]) >= 0):
-            self.init_indices_for_date(cdate, iname)
-            records = self.get_imma_records(cdate, line, records)
-            break
+         if not (self.PVALS['uatti'] and line[0:2] == self.PVALS['uatti']):   # a standalone attm line holds no date
+            cdate = self.get_imma_date(line)
+            if cdate and (not bdate or self.diffdate(cdate, bdate) >= 0):
+               self.init_indices_for_date(cdate, iname)
+               records = self.get_imma_records(cdate, line, records)
+               break
          line = IMMA.readline()
 
       line = IMMA.readline()
@@ -152,7 +274,7 @@ class FillIcoads(PgIMMA):
                   for i in range(self.TABLECOUNT): acounts[i] += acnts[i]
                   records = {}
                   cdate = idate
-                  if self.PVALS['dtlen'] == 2 and self.diffdate(cdate, self.PVALS['dates'][1]) > 0: break
+                  if edate and self.diffdate(cdate, edate) > 0: break
                   self.init_indices_for_date(cdate, iname)
                records = self.get_imma_records(idate, line, records)
          line = IMMA.readline()
@@ -165,6 +287,8 @@ class FillIcoads(PgIMMA):
 
       self.pglog("{} ({}) filled from {}".format(' '.join(map(str, acounts)), self.PVALS['names'], op.basename(fname)), self.LOGWRN)
 
+      if fdate and self.PVALS['dtlen'] == 0: self.check_month_filled(fname, fdate, bdate)
+
       return acounts
 
 # main function to execute this script

rda_python_icoads/fillitable.py

@@ -233,11 +233,15 @@ class FillItable(PgIMMA):
             else:
                self.pglog("{}({}): index counted already between {} and {}".format(var, val, pgrec['imin'], pgrec['imax']), self.LOGWRN)
 
-      cnt = 0
+      cnt = ncnt = 0
       for val in records:
          pgrec = records[val]
+         if not pgrec:
+            ncnt += 1   # counted already; nothing to add
+            continue
          cnt += self.pgupdt(vtable, pgrec, "{} = {}".format(var, val), self.LGEREX)
 
+      if ncnt: self.pglog("{}: nothing to add for {} values counted already".format(vtable, ncnt), self.LOGWRN)
       self.pglog("{} of {} values recounted in table '{}'".format(cnt, vcnt, vtable), self.LOGWRN)
 
    #

rda_python_icoads/fillmonth.py

@@ -24,7 +24,7 @@ class FillMonth(PgUtil):
    def __init__(self):
       super().__init__()
       self.CMDS = {
-         'filename' : "IMMA1_R3.0.2_",
+         'filenames' : ["IMMA1_R3.0.2_", "IMMA1_R3.0.3_"],   # IMMA1_R3.0.3_ since 2025-08
          'fillicoads' : "fillicoads -i ",
          'fillitable' : "fillitable -t -v dck pt sid -r ",
          'cdmsmonth' : "cdmsmonth "
@@ -64,17 +64,31 @@ class FillMonth(PgUtil):
       self.cmdlog()
       sys.exit(0)
 
+   #
+   # locate the monthly IMMA1 file, gunzip it if only the gzipped one is on file
+   #
+   def get_monthly_file(self, smonth):
+
+      files = [fname + smonth for fname in self.CMDS['filenames']]
+
+      for file in files:
+         if op.isfile(file): return file
+
+      for file in files:
+         zfile = file + ".gz"
+         if op.isfile(zfile):
+            self.pgsystem("gunzip " + zfile, self.LGEREX, 5)
+            return file
+
+      self.pglog("Miss monthly IMMA1 file, gzipped or not, in {}: {}".format(
+                 op.abspath('.'), ', '.join(files)), self.LGEREX)
+
    #
    # fill monthly IMMA1 data to IVADDB
    #
    def fill_monthly_data(self, smonth, srange):
 
-      file = self.CMDS['filename'] + smonth
-
-      if not op.isfile(file):
-         # unzip file
-         cmd = "gunzip {}.gz".format(file)
-         self.pgsystem(cmd, self.LGWNEX, 5)
+      file = self.get_monthly_file(smonth)
 
       # fillicoads
       cmd = self.CMDS['fillicoads'] + file

rda_python_icoads/pg_imma.py

@@ -375,7 +375,7 @@ class PgIMMA(PgOPT):
          self.UIDOFFSET = 4
          self.UIDLENGTH = 15
          atti = line[15:17]
-         CURRN3 = int(line[12])
+         self.CURRN3 = int(line[12])
       else:
          atti = None
 
@@ -622,8 +622,28 @@ class PgIMMA(PgOPT):
    #
    # add multiple imma records into different tables in RDADB
    #
+   # all writes for one date are wrapped in a single transaction, so an interrupted
+   # fill can never leave an inventory record behind without its data; the date is
+   # then simply refilled when the file is processed again
+   #
    def add_imma_records(self, cdate, records):
 
+      self.starttran()
+      try:
+         acnts = self.add_date_records(cdate, records)
+      except BaseException:
+         self.aborttran()
+         self.pglog("{}: records rolled back for the date".format(cdate), self.LOGWRN)
+         raise
+      self.endtran()
+
+      return acnts
+
+   #
+   # add all records of one date into the attm and the control tables
+   #
+   def add_date_records(self, cdate, records):
+
       if self.INVENTORY and self.IMMA_NAMES[0] in records:   # add counting record into inventory table
          rcnt = len(records[self.IMMA_NAMES[0]]['iidx'])
          if rcnt > 0: self.INVENTORY = self.add_inventory_record(self.INVENTORY['fname'], cdate, rcnt, self.INVENTORY)

{rda_python_icoads-3.0.1.dist-info → rda_python_icoads-3.0.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_icoads
-Version: 3.0.1
+Version: 3.0.3
 Summary: RDA python package to manage RDA ICOADS datasets
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-icoads

{rda_python_icoads-3.0.1.dist-info → rda_python_icoads-3.0.3.dist-info}/RECORD

@@ -8,10 +8,10 @@ rda_python_icoads/countattm.py,sha256=HACWjl1J1gR-wObSAY_8WtJh6RGZG_b1dwE9NGr6ZF
 rda_python_icoads/countattmvar.py,sha256=5Xrg6zPxy0ckDUE0WYFVF9oDZRBOCTEQq0WQFd5VxRE,8571
 rda_python_icoads/counticoads.py,sha256=7SMu2HJBvUSmmThkNitAdvdyk7aO9dWKvdSv6VGBLng,5179
 rda_python_icoads/countsst.py,sha256=3jLrZUqlrZBVt9ib7nBO6n1D6jPL0T9Esu_elntHMDg,7170
-rda_python_icoads/fillicoads.py,sha256=DzveAwULO8dDnooSMmEiGiS8k0lIa1qW9G7kTmyeH-k,6082
+rda_python_icoads/fillicoads.py,sha256=kPk4SVaZhfA7DKSZEyiM0_ipsvq4Ct8ERN9J_e8KXxA,11146
 rda_python_icoads/fillinventory.py,sha256=cDcfjPAF2Ir991xO9PNMiGSB9boLCUj8txKAw3D_yzI,4617
-rda_python_icoads/fillitable.py,sha256=LRXXF0YbSKp40Yerwt_aIsDO7fsPQDvhARWoBJ6bHPI,10957
-rda_python_icoads/fillmonth.py,sha256=WDP6m6rWW59mCPnQ-ruAeLFnw2U8Dlm6OOFAGEnA1h0,2912
+rda_python_icoads/fillitable.py,sha256=QpatUrSbIHcs-BCL8dBfZCM4FnQWwhArWnENhmvBZUQ,11178
+rda_python_icoads/fillmonth.py,sha256=jeelzcSFic29-R5JoaKr87iDygha0b9AaHSRbxxPluY,3405
 rda_python_icoads/fixiidx.py,sha256=5k6Intm4Zw5mAik-W-lnj167_YloP5THqaY96TMP4Mw,2333
 rda_python_icoads/imma1_subset.py,sha256=IX8vAdViFtcG3ai0_FQSro9038D2cRDcQdezUrNA6Kg,27566
 rda_python_icoads/maxsst.py,sha256=TroXmOKvy2VS0l6ILPJmVfVvDTynma_yoilaVDMXCZ8,9108
@@ -20,12 +20,12 @@ rda_python_icoads/msg3.0_subset_readme.txt,sha256=Nb3rFq4lkxBDhSzS5XLCgCkZpozUGe
 rda_python_icoads/msg3_subset.py,sha256=MXnV-bFDFef5O2vafOxVcrt97wRGu4zmfy8EXbbwxS0,14629
 rda_python_icoads/msg_download.py,sha256=G_MawrQtpmsQOPTgDIcq94K4-BySj7xPTsQydd2UVYo,7636
 rda_python_icoads/msgsubset.f,sha256=KuM2WAk0ZFEH8LBhHKiGWt8zaW_0ARJXYV2kiD5w5xs,21253
-rda_python_icoads/pg_imma.py,sha256=JBNBcCCnZCOcozi0rok149ahY2j0I7pX2lzGbmcvVe4,65727
+rda_python_icoads/pg_imma.py,sha256=RDl2h-JScZxozVLHhebrEuAAUWtbU0_7BoJKZpxMhI8,66378
 rda_python_icoads/rdimma1_csv.f,sha256=_hOHvA0UiTykCIsj7ZYkmooNmUq29-p7ZnyjeSyvt8U,22443
 rda_python_icoads/writeicoads.py,sha256=HQqTp4m2erTV1bOkkBJZwDH6ispJPutPM3ZA8Y_giKI,5841
-rda_python_icoads-3.0.1.dist-info/licenses/LICENSE,sha256=1dck4EAQwv8QweDWCXDx-4Or0S8YwiCstaso_H57Pno,1097
-rda_python_icoads-3.0.1.dist-info/METADATA,sha256=OClItbEyC0u-NRGPY2O9T8a0u6E7ZLNwE0WykxXy6zw,3032
-rda_python_icoads-3.0.1.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
-rda_python_icoads-3.0.1.dist-info/entry_points.txt,sha256=RrLvvQNiBddv_LCYU9b49Zm3fpSl53DBOUXi22SVcE0,776
-rda_python_icoads-3.0.1.dist-info/top_level.txt,sha256=PjJe4PwUIiNxPlNg2d8JNPY1SLSu-_GHAdXe-hUVU1U,18
-rda_python_icoads-3.0.1.dist-info/RECORD,,
+rda_python_icoads-3.0.3.dist-info/licenses/LICENSE,sha256=1dck4EAQwv8QweDWCXDx-4Or0S8YwiCstaso_H57Pno,1097
+rda_python_icoads-3.0.3.dist-info/METADATA,sha256=nZybIybbtjmcxeiFOE1J48lhKGrsiISeWik6X23GnKI,3032
+rda_python_icoads-3.0.3.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+rda_python_icoads-3.0.3.dist-info/entry_points.txt,sha256=RrLvvQNiBddv_LCYU9b49Zm3fpSl53DBOUXi22SVcE0,776
+rda_python_icoads-3.0.3.dist-info/top_level.txt,sha256=PjJe4PwUIiNxPlNg2d8JNPY1SLSu-_GHAdXe-hUVU1U,18
+rda_python_icoads-3.0.3.dist-info/RECORD,,