rda-python-icoads 1.0.6__py3-none-any.whl → 1.0.9__py3-none-any.whl

rda_python_icoads/writeicoads.py

@@ -0,0 +1,169 @@
+#!/usr/bin/env python3
+#
+##################################################################################
+#
+#     Title : writeicoads
+#    Author : Zaihua Ji, zji@ucar.edu
+#      Date : 01/05/2021
+#             2025-03-03 transferred to package rda_python_icoads from
+#             https://github.com/NCAR/rda-icoads.git
+#   Purpose : read ICOADS data from IVADDB and write out monthly files in IMMA format
+#
+#    Github : https://github.com/NCAR/rda-python-icoads.git
+#
+##################################################################################
+
+import sys
+import os
+import re
+from os import path as op
+from rda_python_common import PgLOG
+from rda_python_common import PgDBI
+from rda_python_common import PgSIG
+from rda_python_common import PgUtil
+from rda_python_common import PgFile
+from . import PgIMMA
+
+PVALS = {
+   'bdate' : None,
+   'edate' : None,
+   'month' : [],
+   'bmdate' : [],
+   'emdate' : [],
+   'fnroot' : "IMMA1_R3.0.0",
+   'names' : None,
+   'mporc' : 10,
+   'dumpall' : 0
+}
+
+#
+# main function to run dsarch
+#
+def main():
+
+   option = ''
+   argv = sys.argv[1:]
+   
+   for arg in argv:
+      if arg == "-b":
+         PgLOG.PGLOG['BCKGRND'] = 1
+      elif arg == "-a":
+         PVALS['dumpall'] = 1
+      elif arg == "-f":
+         option = 'f'
+      elif arg == "-m":
+         option = 'm'
+      elif re.match(r'^-', arg):
+        PgLOG.pglog(arg + ": Invalid Option", PgLOG.LGWNEX)
+      elif option:
+         if option == 'f':
+           PVALS['fnroot'] = arg
+         elif option == 'm':
+            PVALS['mproc'] = arg
+         option = ''
+      elif not PVALS['bdate']:
+         PVALS['bdate'] = arg
+      elif not PVALS['edate']:
+         PVALS['edate'] = arg
+      else:
+        PgLOG.pglog(arg + ": Invalid parameter", PgLOG.LGWNEX)
+
+   PgDBI.ivaddb_dbname()
+   
+   if not (PVALS['bdate'] and PVALS['edate']):
+      pgrec = PgDBI.pgget("cntldb.inventory", "min(date) bdate, max(date) edate", '', PgLOG.LGEREX)
+      print("Usage: writeicoads [-a] [-m mproc] [-f RootFileName] BeginDate EndDate")
+      print("   Default RootFileName = {}".format(PVALS['fnroot']))
+      print("   Option -a - dump all attms, including multi-line ones, such as IVAD and REANQC")
+      print("   Option -m - start up to given number of processes, one for each file dump (Default to 10)")
+      print("   Set BeginDate and EndDate between '{}' and '{}'".format(pgrec['bdate'], pgrec['edate']))
+      sys.exit(0)
+
+   if PgUtil.diffdate(PVALS['bdate'], PVALS['edate']) > 0:
+      tmpdate = PVALS['bdate']
+      PVALS['bdate'] = PVALS['edate']
+      PVALS['edate'] = tmpdate
+   
+   PgLOG.PGLOG['LOGFILE'] = "icoads.log"
+   PgLOG.cmdlog("writeicoads {}".format(' '.join(argv)))
+   PVALS['names'] = '/'.join(PgIMMA.IMMA_NAMES)
+   write_imma_data()
+   PgLOG.cmdlog()
+   sys.exit(0)
+
+#
+# read imma data from IVADB and dump into files
+#
+def write_imma_data():
+   
+   mcnt = init_months()
+
+   if mcnt == 1: PVALS['mproc'] = 1
+   if PVALS['mproc'] > 1: PgSIG.start_none_daemon('writeicoads', '', PgLOG.PGLOG['CURUID'], PVALS['mproc'], 300, 1)
+
+   for midx in range(mcnt):
+      if PVALS['mproc'] > 1:
+         stat = PgSIG.start_child("writeicoads_{}".format(midx), PgLOG.LOGWRN, 1)  # try to start a child process
+         if stat <= 0:
+            sys.exit(1)   # something wrong
+         elif PgSIG.PGSIG['PPID'] > 1:
+            write_monthly_imma_file(midx)
+            sys.exit(0)  # stop child process
+         else:
+            PgDBI.pgdisconnect(0)  # disconnect database for reconnection
+            continue  # continue for next midx
+      else:
+         write_monthly_imma_file(midx)
+
+   if PVALS['mproc'] > 1: # quit parent without waiting
+      PgLOG.pglog("Started {} child processes to write icoads files".format(mcnt), PgLOG.LOGWRN)
+
+#
+# read icoads record from given file name and save them into RDADB
+#
+def write_monthly_imma_file(midx):
+
+   fname = "{}_{}".format(PVALS['fnroot'], PVALS['month'][midx])
+   PgLOG.pglog("write IMMA1 records into File '{}' from IVADDB".format(fname), PgLOG.WARNLG)
+   opened = 0
+   acounts = [0]*PgIMMA.TABLECOUNT
+   IMMA = open(fname, 'w')
+   cdate = PVALS['bmdate'][midx]
+   while cdate <= PVALS['emdate'][midx]:
+      acnts = PgIMMA.write_imma_records(IMMA, cdate, 0, PVALS['dumpall'])
+      if acnts:
+         for i in range(PgIMMA.TABLECOUNT): acounts[i] += acnts[i]
+      cdate = PgUtil.adddate(cdate, 0, 0, 1)
+
+   IMMA.close()
+   if acounts[0] == 0: PgFile.delete_local_file(fname)
+
+   PgLOG.pglog("{}({}) written into {}".format('/'.join(map(str, acounts)), PVALS['names'], fname), PgLOG.LOGWRN)
+
+#
+# intialize month arrays
+#
+def init_months():
+
+   bdate = PVALS['bdate']
+   table = "cntldb.inventory"
+   mcnt = done = 0
+   while True:
+      edate = PgUtil.enddate(bdate, 0, 'M')
+      if PgUtil.diffdate(PVALS['edate'], edate) <= 0:
+         edate = PVALS['edate']
+         done = 1
+      if PgDBI.pgget(table, "date", "date BETWEEN '{}' AND '{}'".format(bdate, edate), PgLOG.LGEREX):
+         PVALS['bmdate'].append(bdate)
+         PVALS['month'].append(PgUtil.format_date(bdate, "YYYY-MM"))
+         PVALS['emdate'].append(edate)
+         mcnt += 1
+      if done: break
+      bdate = PgUtil.adddate(edate, 0, 0, 1)
+
+   return mcnt
+
+#
+# call main() to start program
+#
+if __name__ == "__main__": main()

{rda_python_icoads-1.0.6.dist-info → rda_python_icoads-1.0.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rda_python_icoads
-Version: 1.0.6
+Version: 1.0.9
 Summary: RDA python package to manage RDA ICOADS datasets
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-icoads

rda_python_icoads-1.0.9.dist-info/RECORD

@@ -0,0 +1,25 @@
+rda_python_icoads/PgIMMA.py,sha256=VJVFXUzOpGhlm-oHV35zkX_I-MMo-pJQt31ZOIlIDeo,58125
+rda_python_icoads/R3.0-stat_doc.pdf,sha256=7f9wKlaTV7ZWXCqyZ4MZ3QcL5pw-Hz623A3xe8QCoFM,59515
+rda_python_icoads/README_R3.0_Subset.html,sha256=tf5Om69KsZZ9ksIjIwtGQUtU8O4U-fqOrIxrl_4tRPo,6878
+rda_python_icoads/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+rda_python_icoads/checkicoads.py,sha256=nbA09uqaVBf9BHPWWeLFke3sRAGgV7CyOG4VIKI33wY,6599
+rda_python_icoads/cleanicoads.py,sha256=A7JuwYbaVHjVpZFuQZBd9YMQTdMnu7YiqSmbnq058BI,4936
+rda_python_icoads/counticoads.py,sha256=GB7mRXXQ80Ty_1UF4hyYxrO0IPOeUCq5JUhoxkuvXZs,4555
+rda_python_icoads/fillicoads.py,sha256=xIj9vm3LdhjwQar28B77vfpe7gJDGHCUZ_08zzRnTuo,4239
+rda_python_icoads/fillinventory.py,sha256=eRNw6laHsF_Ry5SVOJCulDNa4T-9FTT2UObt__Pr3IM,4144
+rda_python_icoads/fillitable.py,sha256=_zCUuYZifMZUoBO8JSY0DcoQOM33smQ1WtWJWnlH3ks,9941
+rda_python_icoads/fillmonth.py,sha256=Abo_5KoX4Zs-vlYcEvovo14xEiO_BNgUxV5fFxT-L3M,2546
+rda_python_icoads/imma1_subset.py,sha256=8r130jzm4b9IPjMJBwwRvWrdgxDnXC8KbQjFwiE7LPA,23704
+rda_python_icoads/msg,sha256=J5lkNELaJke718h8VJgqbR0v_nkivStfW0xKz30sVJw,27153
+rda_python_icoads/msg3.0_subset_readme.txt,sha256=Nb3rFq4lkxBDhSzS5XLCgCkZpozUGefy2JkUh1gk0Wo,4144
+rda_python_icoads/msg3_subset.py,sha256=piTGWpsUjbCD2J6q3SylQRlzux5RqW4-ifoSMbvFK_s,13026
+rda_python_icoads/msg_download.py,sha256=SqJVdVs6kN-ZLh1d4rSMS4IkRdRrYf5c4Y45zV7pJVQ,6704
+rda_python_icoads/msgsubset.f,sha256=KuM2WAk0ZFEH8LBhHKiGWt8zaW_0ARJXYV2kiD5w5xs,21253
+rda_python_icoads/rdimma1_csv.f,sha256=_hOHvA0UiTykCIsj7ZYkmooNmUq29-p7ZnyjeSyvt8U,22443
+rda_python_icoads/writeicoads.py,sha256=nNgcPTFw957qsBkMVXgrPAkj3qmG8g5b6NeePp4NkNk,5270
+rda_python_icoads-1.0.9.dist-info/LICENSE,sha256=1dck4EAQwv8QweDWCXDx-4Or0S8YwiCstaso_H57Pno,1097
+rda_python_icoads-1.0.9.dist-info/METADATA,sha256=gEgX9ihZd2B002Z7_knUJWgqybCtUa9Fqp6mHcdJjyk,772
+rda_python_icoads-1.0.9.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+rda_python_icoads-1.0.9.dist-info/entry_points.txt,sha256=AQnzGqKNOCg8X7dRieCscw3tH8_Y05RCw9_ij4hlIOY,557
+rda_python_icoads-1.0.9.dist-info/top_level.txt,sha256=PjJe4PwUIiNxPlNg2d8JNPY1SLSu-_GHAdXe-hUVU1U,18
+rda_python_icoads-1.0.9.dist-info/RECORD,,

rda_python_icoads-1.0.9.dist-info/entry_points.txt

@@ -0,0 +1,12 @@
+[console_scripts]
+checkicoads = rda_python_icoads.checkicoads:main
+cleanicoads = rda_python_icoads.cleanicoads:main
+counticoads = rda_python_icoads.counticoads:main
+fillicoads = rda_python_icoads.fillicoads:main
+fillinventory = rda_python_icoads.fillinventory:main
+fillitable = rda_python_icoads.fillitable:main
+fillmonth = rda_python_icoads.fillmonth:main
+imma1_subset = rda_python_icoads.imma1_subset:main
+msg3_subset = rda_python_icoads.msg3_subset:main
+msg_download = rda_python_icoads.msg_download:main
+writeicoads = rda_python_icoads.writeicoads:main

rda_python_icoads-1.0.6.dist-info/RECORD

@@ -1,11 +0,0 @@
-rda_python_icoads/PgIMMA.py,sha256=VJVFXUzOpGhlm-oHV35zkX_I-MMo-pJQt31ZOIlIDeo,58125
-rda_python_icoads/README_R3.0_Subset.html,sha256=tf5Om69KsZZ9ksIjIwtGQUtU8O4U-fqOrIxrl_4tRPo,6878
-rda_python_icoads/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-rda_python_icoads/imma1_subset.py,sha256=vPZ-pHoE1pY0_KCEERx5-cvD6N0Drwvk0GLPa3i5fxo,23680
-rda_python_icoads/rdimma1_csv.f,sha256=_hOHvA0UiTykCIsj7ZYkmooNmUq29-p7ZnyjeSyvt8U,22443
-rda_python_icoads-1.0.6.dist-info/LICENSE,sha256=1dck4EAQwv8QweDWCXDx-4Or0S8YwiCstaso_H57Pno,1097
-rda_python_icoads-1.0.6.dist-info/METADATA,sha256=5c3M34YkDs1ljjEOxUbUbGnwDj-M9cQsHDJJbQsJXDM,772
-rda_python_icoads-1.0.6.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
-rda_python_icoads-1.0.6.dist-info/entry_points.txt,sha256=q-IUAuLdRph_A67-ktgEyowjGyOSRovQTzQWPf_Qtwk,69
-rda_python_icoads-1.0.6.dist-info/top_level.txt,sha256=PjJe4PwUIiNxPlNg2d8JNPY1SLSu-_GHAdXe-hUVU1U,18
-rda_python_icoads-1.0.6.dist-info/RECORD,,

rda_python_icoads-1.0.6.dist-info/entry_points.txt

@@ -1,2 +0,0 @@
-[console_scripts]
-imma1_subset = rda_python_icoads.imma1_subset:main