rda-python-metrics 1.0.40__py3-none-any.whl → 1.0.42__py3-none-any.whl

rda_python_metrics/fillcdgusage.py

@@ -31,29 +31,28 @@ USAGE = {
 }
 
 DSIDS = {
-#   'ucar.cgd.cesm2.cam6.prescribed_sst_amip' : ['d651010'],
-#   'ucar.cgd.ccsm4.CLM_LAND_ONLY' : ['d651011'],
-#   'ucar.cgd.artmip' : ['d651012', 'd651016', 'd651017', 'd651018'],
-#   'tamip' : ['d651013'],
-#   'ucar.cgd.ccsm4.CLIVAR_LE' : ['d651014'],
-#   'ucar.cgd.cesm2.Gettelman_CESM2_ECS' : ['d651015'],
-#   'ucar.cgd.ccsm4.geomip.ssp5' : ['d651024'],
-#   'ucar.cgd.ccsm4.IOD-PACEMAKER' : ['d651021'],
-#   'ucar.cgd.ccsm4.past2k_transient' : ['651023'],
-#   'ucar.cgd.ccsm4.lowwarming' : ['d651025'],
-#   'ucar.cgd.ccsm4.CESM_CAM5_BGC_ME' : ['d651000'],
-#   'ucar.cgd.ccsm4.iTRACE' : ['d651022'],
-#   'ucar.cgd.ccsm4.so2_geoeng' : ['d651026'],
-#   'ucar.cgd.ccsm4.cesmLE' : ['d651027'],
-#   'ucar.cgd.ccsm4.CESM1-CAM5-DP' : ['d651028'],
-#   'ucar.cgd.ccsm4.amv_lens' : ['d651031'],
-#   'ucar.cgd.ccsm4.ATL-PACEMAKER' : ['d651032'],
-#   'ucar.cgd.ccsm4.pac-pacemaker' : ['d651033'],
-#   'ucar.cgd.ccsm4.SD-WACCM-X_v2.1' : ['d651034'],
-#   'ucar.cgd.ccsm4.amv_lens' : ['d651035'],
-#   'ucar.cgd.cesm2.cism_ismip6' : ['d651036'],
-#   'ucar.cgd.ccsm4.pliomip2' : ['d651037'],
-   # new added
+   'ucar.cgd.cesm2.cam6.prescribed_sst_amip' : ['d651010'],
+   'ucar.cgd.ccsm4.CLM_LAND_ONLY' : ['d651011'],
+   'ucar.cgd.artmip' : ['d651012', 'd651016', 'd651017', 'd651018'],
+   'tamip' : ['d651013'],
+   'ucar.cgd.ccsm4.CLIVAR_LE' : ['d651014'],
+   'ucar.cgd.cesm2.Gettelman_CESM2_ECS' : ['d651015'],
+   'ucar.cgd.ccsm4.geomip.ssp5' : ['d651024'],
+   'ucar.cgd.ccsm4.IOD-PACEMAKER' : ['d651021'],
+   'ucar.cgd.ccsm4.past2k_transient' : ['651023'],
+   'ucar.cgd.ccsm4.lowwarming' : ['d651025'],
+   'ucar.cgd.ccsm4.CESM_CAM5_BGC_ME' : ['d651000'],
+   'ucar.cgd.ccsm4.iTRACE' : ['d651022'],
+   'ucar.cgd.ccsm4.so2_geoeng' : ['d651026'],
+   'ucar.cgd.ccsm4.cesmLE' : ['d651027'],
+   'ucar.cgd.ccsm4.CESM1-CAM5-DP' : ['d651028'],
+   'ucar.cgd.ccsm4.amv_lens' : ['d651031'],
+   'ucar.cgd.ccsm4.ATL-PACEMAKER' : ['d651032'],
+   'ucar.cgd.ccsm4.pac-pacemaker' : ['d651033'],
+   'ucar.cgd.ccsm4.SD-WACCM-X_v2.1' : ['d651034'],
+   'ucar.cgd.ccsm4.amv_lens' : ['d651035'],
+   'ucar.cgd.cesm2.cism_ismip6' : ['d651036'],
+   'ucar.cgd.ccsm4.pliomip2' : ['d651037'],
    'ucar.cgd.cesm2-waccm.s2s_hindcasts': ['d651040'],
    'ucar.cgd.CESM1.3_SH_storm_tracks': ['d651044'],
    'ucar.cgd.cesm2.waccm6.ssp245': ['d651045'],
@@ -67,13 +66,12 @@ DSIDS = {
    'ucar.cgd.ccsm4.PaleoIF' : ['d651052'],
    'ucar.cgd.ccsm4.b.e11.B20LE_fixedO3' : ['d651053'],
    'ucar.cgd.cesm2.single.forcing.large.ensemble' : ['d651055'],
-   'ucar.cgd.cesm2le.output': ['d651056'],    # 2025-01 done
+   'ucar.cgd.cesm2le.output': ['d651056'],
    'ucar.cgd.ccsm4.ARISE-SAI-1.5' : ['d651059'],
    'ucar.cgd.cesm2.s2s_hindcasts': ['d651060'],
    'ucar.cgd.cesm2.s2s_hindcasts.mjo': ['d651061'],
    'ucar.cgd.cesm2.s2s_hindcasts.tc_tracks': ['d651062'],
    'ucar.cgd.cesm2.s2s_hindcasts.cesm2.climo': ['d651063'],
-   #add 2025-08-06
    'ucar.cgd.ccsm4.cesmLME' : ['d651058'],
    'ucar.cgd.ccsm4.GLENS' : ['d651064'],
    'ucar.cgd.ccsm4.CESM2-CISM2-LIGtransient' : ['d651066'],
@@ -85,8 +83,8 @@ DSIDS = {
    'ucar.cgd.cesm1.cldmod': ['d651069'],
    'ucar.cgd.cesm2.marine.biogeochemistry': ['d651071'],
    'ucar.cgd.nw2.mom6': ['d651072'],
-#   'ucar.cgd.cesm2.smyle': ['d651065'],  # not done util the end of Aug.
-   
+   'ucar.cgd.cesm2.cam6.ppe': ['d651076'],
+   'ucar.cgd.cesm2.smyle': ['d651065']
 }
 
 ALLIDS = list(DSIDS.keys())

rda_python_metrics/fillgdexusage.py

@@ -30,44 +30,58 @@ USAGE = {
 }
 
 DSIDS = {
+   "14_schuster" : "d583138",
+   "384_duvivier" : "d583139",
+   "model_data_for_17_march_2015_storm_event" : "d583140",
+   "326_qingyu" : "d583141",
+   "354_fasullo" : "d583142",
+   "157_gaubert" : "d583143",
+   "370_gaubert" : "d583144",
+   "459" : "d583145",
+   "108_kristenk" : "d583146",
+   "82_abaker" : "d583147",
+   "215_grabow" : "d583148",
+   "222_duvivier" : "d583149",
+   "366_rberrios" : "d583150",
+   "210_dxyang" : "d583100",
+   "238_junkyung" : "d583101",
+   "361_islas" : "d583102",
+   "448" : "d583103",
+   "187_jaredlee" : "d583104",
+   "273_kdagon" : "d583105",
+   "149_liuh" : "d583106",
+   "279_schwantes" : "d583107",
+   "60_pinto" : "d583108",
+   "WACC_Dependence_on_Solar_Activity" : "d583109",
+   "275_domingom" : "d583110",
+   "445" : "d583111",
+   "TIE-GCM_2017_Solar_Storm" : "d583112",
+   "123_andrew" : "d583113",
+   "453" : "d583114",
+   "367_mclong" : "d583115",
+   "258_jiangzhu" : "d583116",
+   "136_hannay" : "d583117",
+   "173B_dcherian" : "d583118",
+   "407_islas" : "d583119",
+   "189_islas" : "d583120",
+   "289_shields" : "d583121",
+   "415_mcinerney" : "d583122",
+   "151_dll" : "d583123",
+   "460" : "d583124",
+   "117_phamkh" : "d583125",
+   "109_erichend" : "d583126",
+   "412" : "d583127",
+   "390_knocasio" : "d583128",
+   "382_kumar" : "d583129",
+   "470" : "d583130",
+   "371_abaker" : "d583131",
+   "CESM-cocco_CO2_experiments" : "d583132",
+   "fuel_moisture_content" : "d583133",
+   "81_rberrios" : "d583134",
+   "286_knocasio" : "d583135",
+   "383_stebbins" : "d583136",
+   "256_tilmes" : "d583137",
    '371_abaker' : 'd583131',
-   'CESM-cocco_CO2_experiments' : 'd583132',
-   '256_tilmes' : 'd583137',
-   'fuel_moisture_content' : 'd583133',
-   '81_rberrios' : 'd583134',
-   '210_dxyang' : 'd583100',
-   '286_knocasio' : 'd583135',
-   '238_junkyung' : 'd583101',
-   '361_islas' : 'd583102',
-   '448' : 'd583103',
-   '383_stebbins' : 'd583136',
-   '187_jaredlee' : 'd583104',
-   '273_kdagon' : 'd583105',
-   '149_liuh' : 'd583106',
-   '279_schwantes' : 'd583107',
-   '60_pinto' : 'd583108',
-   'WACC_Dependence_on_Solar_Activity' : 'd583109',
-   '275_domingom' : 'd583110',
-   '445' : 'd583111',
-   'TIE-GCM_2017_Solar_Storm' : 'd583112',
-   '123_andrew' : 'd583113',
-   '453' : 'd583114',
-   '367_mclong' : 'd583115',
-   '258_jiangzhu' : 'd583116',
-   '136_hannay' : 'd583117',
-   '173B_dcherian' : 'd583118',
-   '407_islas' : 'd583119',
-   '189_islas' : 'd583120',
-   '289_shields' : 'd583121',
-   '415_mcinerney' : 'd583122',
-   '151_dll' : 'd583123',
-   '460' : 'd583124',
-   '117_phamkh' : 'd583125',
-   '109_erichend' : 'd583126',
-   '412' : 'd583127',
-   '390_knocasio' : 'd583128',
-   '382_kumar' : 'd583129',
-   '470' : 'd583130',
    # icarus
    'icarus.experiment.403' : 'd789010',
    'icarus.experiment.404' : 'd789011',

rda_python_metrics/fillrdadb.py

@@ -144,8 +144,7 @@ def send_email_notice():
          if receiver: receiver += ', '
          receiver += (logname + "@ucar.edu")
 
-      ret = PgLOG.send_email("RDADB Weekly Data Usage Gathered on " + PgUtil.curdate(), receiver, msg)
-      if ret: PgLOG.pglog(ret, PgLOG.LOGWRN)
+      PgLOG.send_email("RDADB Weekly Data Usage Gathered on " + PgUtil.curdate(), receiver, msg)
 
 #
 # call main() to start program

rda_python_metrics/fillzenodousage.py

@@ -0,0 +1,408 @@
+#!/usr/bin/env python3
+#
+###############################################################################
+#
+#     Title : fillgdexusage
+#    Author : Zaihua Ji,  zji@ucar.edu
+#      Date : 2025-04-14
+#   Purpose : python program to retrieve info from ZENODO Postgres database for GDS 
+#             file accesses and backup fill table tdsusage in PostgreSQL database dssdb.
+# 
+#    Github : https://github.com/NCAR/rda-python-metrics.git
+#
+###############################################################################
+#
+import sys
+import re
+import glob
+from os import path as op
+from time import time as tm
+from rda_python_common import PgLOG
+from rda_python_common import PgUtil
+from rda_python_common import PgFile
+from rda_python_common import PgDBI
+from rda_python_common import PgSplit
+from . import PgIPInfo
+
+USAGE = {
+   'ZNDTBL'  : "gdexzenodo",
+   'CDATE' : PgUtil.curdate(),
+}
+
+DSIDS = {
+   "498" : "15760289",
+   "386_ahijevych" : "15501153",
+   "66_oclifton" : "15501155",
+   "196_mayernik" : "15501157",
+   "83_heyms1" : "15501159",
+   "405_qwu" : "15501163",
+   "WACCMX_DART_Hindcast" : "15501165",
+   "234_buchholz" : "15501167",
+   "365_barthm" : "15501171",
+   "Polar_Cap_HIWIND_observation_and_TIEGCM_simulations" : "15501173",
+   "266_xuguang" : "15501175",
+   "277_lqian" : "15501177",
+   "Ocean_MHT_Values" : "15501181",
+   "92_xuguang" : "15501184",
+   "118_nickp" : "15501186",
+   "398_kunwu" : "15501188",
+   "107_siyuan" : "15501190",
+   "274_gaubert" : "15501192",
+   "301_ldong" : "15501194",
+   "GRL_Kurowski_al_2019" : "15501196",
+   "112_fjudt" : "15501201",
+   "14_schuster" : "15501203",
+   "102_abaker" : "15501205",
+   "402_qwu" : "15501207",
+   "324_lqian" : "15501209",
+   "Response_of_the_F2-region_Ionosphere_to_the_21_August_2017_Solar_Eclipse_at_Millstone_Hill" : "15501211",
+   "362_kunwu" : "15501213",
+   "LFM-TIEGCM-RCM_Simulation" : "15501216",
+   "140_xuguang" : "15501221",
+   "240_ldong" : "15501223",
+   "93_sallyz" : "15501229",
+   "452" : "15501231",
+   "318_ldong" : "15501233",
+   "161_ldong" : "15501237",
+   "288_voemel" : "15501239",
+   "160_xuguang" : "15501241",
+   "CO_Flux_Inversion_Attribution" : "15501243",
+   "404_buchholz" : "15501245",
+   "193_wily" : "15501247",
+   "471" : "15501249",
+   "323_grabow" : "15501251",
+   "2019_GrandChallengeIT_paper" : "15501253",
+   "203_patton" : "15501255",
+   "450" : "15501259",
+   "359_wwieder" : "15501261",
+   "205_xuguang" : "15501265",
+   "218_xuguang" : "15501267",
+   "236_xuguang" : "15501269",
+   "211_fjudt" : "15501273",
+   "2018_Schmidt_JGR_Volcanic_RF" : "15501277",
+   "438" : "15501279",
+   "267_fredc" : "15501282",
+   "321_patton" : "15501286",
+   "467" : "15501294",
+   "145_lqian" : "15501296",
+   "317_jaredlee" : "15501298",
+   "WACCMX_2005_Solar_Flare" : "15501305",
+   "369_buchholz" : "15501313",
+   "148_bjohns" : "15501319",
+   "158_asphilli" : "15501326",
+   "VAPOR_Sample_Data" : "15501333",
+   "188b_oleson" : "15501339",
+   "206_grabow" : "15501349",
+   "127_cweeks" : "15501358",
+   "2019_Ionosphere_Thermosphere_Qian" : "15501375",
+   "391_kim" : "15501388",
+   "200_fredc" : "15501403",
+   "147_miesch" : "15501412",
+   "141_maute" : "15501419",
+   "147b_jcsda" : "15501432",
+   "283_dll" : "15501444",
+   "78_clyne_wrf" : "15501451",
+   "345_wwieder" : "15501470",
+   "221_mclong" : "15501488",
+   "257_ottobli" : "15501516",
+   "287_qwu" : "15501551",
+   "400_qwu" : "15501569",
+   "184_jjang" : "15501603",
+   "197_islas" : "15501620",
+   "ftir" : "15501660",
+   "282_fasullo" : "15501697",
+   "139_oclifton" : "15501795",
+   "348_wwieder" : "15501824",
+   "451" : "15501860",
+   "188a_oleson" : "15501878",
+   "173A_dcherian" : "15501913",
+   "297_qingyu" : "15501946",
+   "523" : "15501974",
+   "472" : "15502021",
+   "138_maute" : "15502102",
+   "387_jay" : "15502178",
+   "jedi-skylab" : "15502247",
+   "CLM5_Sensitivity_Analyses" : "15502308",
+   "531" : "15502409",
+   "327_qwu" : "15502530",
+   "94_nickp" : "15502666",
+   "camels" : "15529996",
+   "68_rschwant" : "15502739",
+   "WACCM_VolcAerProp" : "15503247",
+   "213_fasullo" : "15503556",
+   "263_nystrom" : "15504044",
+   "239_fasullo" : "15504166",
+   "292_qingyu" : "15504276",
+   "294_maute" : "15504555",
+   "209_yeager" : "15504665",
+   "440" : "15504780",
+   "207_gunterl" : "15504857",
+   "172_cdswk" : "15505157",
+   "285_qingyu" : "15505262",
+   "422" : "15505355",
+   "120_maute" : "15505595",
+   "69_maute" : "15507025",
+   "63_maute" : "15507365",
+   "280_islas" : "15507449",
+   "JAMES_2019MS001833" : "15507514",
+   "165_nickp" : "15507754",
+   "510" : "15507947",
+   "96_morrison" : "15508039",
+   "164_bukovsky" : "15508226",
+   "378_caron" : "15508458",
+   "483" : "15508746",
+   "144_grabow" : "15508914",
+   "188c_oleson" : "15509127",
+   "204_ajahn" : "15510016",
+   "72_kdagon" : "15511424",
+   "302_yeager" : "15511789",
+   "86_morrison" : "15513137",
+   "259_jimenez" : "15514297"
+}
+
+ALLIDS = list(DSIDS.keys())
+
+WFILES = {}
+
+#
+# main function to run this program
+#
+def main():
+
+   params = {}  # array of input values
+   argv = sys.argv[1:]
+   opt = None
+   
+   for arg in argv:
+      if arg == "-b":
+         PgLOG.PGLOG['BCKGRND'] = 1
+      elif re.match(r'^-[msNy]$', arg):
+         opt = arg[1]
+         params[opt] = []
+      elif re.match(r'^-', arg):
+         PgLOG.pglog(arg + ": Invalid Option", PgLOG.LGWNEX)
+      elif opt:
+         params[opt].append(arg)
+      else:
+         PgLOG.pglog(arg + ": Value passed in without leading option", PgLOG.LGWNEX)
+
+   if not opt:
+      PgLOG.show_usage('fillgdexusage')
+   elif 's' not in params:
+      PgLOG.pglog("-s: Missing dataset short name to gather ZENODO metrics", PgLOG.LGWNEX)
+   elif len(params) < 2:
+      PgLOG.pglog("-(m|N|y): Missing Month, NumberDays or Year to gather ZENODO metrics", PgLOG.LGWNEX)
+      
+   
+   PgLOG.cmdlog("fillgdexusage {}".format(' '.join(argv)))
+   dranges = get_date_ranges(params)
+   dsids = get_dataset_ids(params['s'])
+   if dranges and dsids: fill_gdex_usages(dsids, dranges)
+   PgLOG.pglog(None, PgLOG.LOGWRN|PgLOG.SNDEML)  # send email out if any
+
+   sys.exit(0)
+
+#
+# connect to the gdex database esg-production
+#
+def gdex_dbname():
+   PgDBI.set_scname('gdex-production', 'metrics', 'gateway-reader', None, 'sagedbprodalma.ucar.edu')
+
+#
+# get datasets
+#
+def get_dataset_ids(dsnames):
+
+   gdex_dbname()
+   dsids = []
+   tbname = 'metadata.dataset'
+   for dsname in dsnames:
+      if re.match(r'^all$', dsname, re.I): return get_dataset_ids(ALLIDS)
+      if dsname not in DSIDS:
+         PgLOG.pglog(dsname + ": Unknown ZENODO dataset short name", PgLOG.LOGWRN)
+         continue
+      bt = tm()
+      pgrec = PgDBI.pgget(tbname, 'id', "short_name = '{}'".format(dsname))
+      if not (pgrec and pgrec['id']): continue
+      zndid = DSIDS[dsname]
+      strids = "{}-{}".format(dsname, zndid)
+      gdexid = pgrec['id']
+      gdexids = [gdexid]
+      ccnt = 1
+      ccnt += recursive_dataset_ids(gdexid, gdexids)
+      dsids.append([dsname, zndid, gdexids, strids])
+      rmsg = PgLOG.seconds_to_string_time(tm() - bt)
+      PgLOG.pglog("{}: Found {} GDEX dsid/subdsids in {} at {}".format(strids, ccnt, rmsg, PgLOG.current_datetime()), PgLOG.LOGWRN)
+
+   if not dsids: PgLOG.pglog("No Dataset Id identified to gather GDEX metrics", PgLOG.LOGWRN)
+
+   return dsids
+
+#
+# get gdexids recursivley
+#
+def recursive_dataset_ids(pgdexid, gdexids):
+
+   tbname = 'metadata.dataset'
+   pgrecs = PgDBI.pgmget(tbname, 'id', "parent_dataset_id = '{}'".format(pgdexid))
+   if not pgrecs: return 0
+
+   ccnt = 0
+   for gdexid in pgrecs['id']:
+      if gdexid in gdexids: continue
+      gdexids.append(gdexid)
+      ccnt += 1
+      ccnt += recursive_dataset_ids(gdexid, gdexids)
+
+   return ccnt
+
+#
+# get the date ranges for given condition
+#
+def get_date_ranges(inputs):
+
+   dranges = []
+   for opt in inputs:
+      for input in inputs[opt]:
+         # get date range
+         dates = []
+         if opt == 'N':
+            dates.append(PgUtil.adddate(USAGE['CDATE'], 0, 0, -int(input)))
+            dates.append(USAGE['CDATE'])
+         elif opt == 'm':
+            tms = input.split('-')
+            dates.append(PgUtil.fmtdate(int(tms[0]), int(tms[1]), 1))
+            dates.append(PgUtil.enddate(dates[0], 0, 'M'))
+         elif opt == 'y':
+            dates.append(input + "-01-01")
+            dates.append(input + "-12-31")
+         if dates: dranges.append(dates)
+
+   return dranges
+
+#
+# get file download records for given dsid
+#
+def get_dsid_records(gdexids, dates, strids):
+
+   gdex_dbname()
+   tbname = 'metrics.file_download'
+   fields = ('date_completed, remote_address, logical_file_size, logical_file_name, file_access_point_uri, user_agent_name, bytes_sent, '
+             'subset_file_size, range_request, dataset_file_size, dataset_file_name, dataset_file_file_access_point_uri')
+   dscnt = len(gdexids)
+   dscnd = "dataset_id "
+   if dscnt == 1:
+      dscnd += "= '{}'".format(gdexids[0])
+   else:
+      dscnd += "IN ('" + "','".join(gdexids) + "')"
+   dtcnd = "date_completed BETWEEN '{} 00:00:00' AND '{} 23:59:59'".format(dates[0], dates[1])
+   cond = "{} AND {} ORDER BY date_completed".format(dscnd, dtcnd)
+   PgLOG.pglog("{}: Query for {} GDEX dsid/subdsids between {} and {} at {}".format(strids, dscnt, dates[0], dates[1], PgLOG.current_datetime()), PgLOG.LOGWRN)
+   pgrecs = PgDBI.pgmget(tbname, fields, cond)
+   PgDBI.dssdb_dbname()
+
+   return pgrecs
+
+#
+# Fill ZND usages into table dssdb.tdsusage from gdex access records
+#
+def fill_gdex_usages(dsids, dranges):
+
+   allcnt = awcnt = azcnt = lcnt = 0
+   for dates in dranges:
+      for dsid in dsids:
+         lcnt += 1
+         dsname = dsid[0]
+         zndid = dsid[1]
+         gdexids = dsid[2]
+         strids = dsid[3]
+         bt = tm()
+         pgrecs = get_dsid_records(gdexids, dates, strids)
+         pgcnt = len(pgrecs['dataset_file_name']) if pgrecs else 0
+         if pgcnt == 0:
+            PgLOG.pglog("{}: No record found to gather GDEX usage between {} and {}".format(strids, dates[0], dates[1]), PgLOG.LOGWRN)
+            continue
+         rmsg = PgLOG.seconds_to_string_time(tm() - bt)
+         PgLOG.pglog("{}: Got {} records in {} for processing GDEX usage at {}".format(strids, pgcnt, rmsg, PgLOG.current_datetime()), PgLOG.LOGWRN)
+         zcnt = 0
+         pwkey = wrec = cdate = None
+         zrecs = {}
+         bt = tm()
+         for i in range(pgcnt):
+            if (i+1)%20000 == 0:
+               PgLOG.pglog("{}/{} GDEX/ZND records processed to add".format(i, zcnt), PgLOG.WARNLG)
+
+            pgrec = PgUtil.onerecord(pgrecs, i)
+            dsize = pgrec['bytes_sent']
+            if not dsize: continue
+            (year, quarter, date, time) = get_record_date_time(pgrec['date_completed'])
+            url = pgrec['dataset_file_file_access_point_uri']
+            if not url: url = pgrec['file_access_point_uri']
+            ip = pgrec['remote_address']
+            engine = pgrec['user_agent_name']
+            ms = re.search(r'^https*://tds.ucar.edu/thredds/(\w+)/', url)
+            if ms:
+               # tds usage
+               if pgrec['subset_file_size']:
+                  etype = 'S'
+               elif pgrec['range_request']:
+                  etype = 'R'
+               else:
+                  etype = 'F'
+               method = 'TDS-' + etype
+            else:
+               # web usage
+               method = 'WEB'
+
+            if date != cdate:
+               if zrecs:
+                  zcnt += add_zusage_records(zrecs, cdate)
+                  zrecs = {}
+               cdate = date
+            zkey = "{}:{}:{}:{}".format(ip, zndid, method)
+            if zkey in zrecs:
+               zrecs[zkey]['size'] += dsize
+               zrecs[zkey]['fcount'] += 1
+            else:
+               iprec =  PgIPInfo.get_missing_ipinfo(ip)
+               if not iprec: continue
+               zrecs[zkey] = {'ip' : ip, 'zdsid' : zndid, 'date' : cdate, 'time' : time, 'quarter' : quarter,
+                              'size' : dsize, 'fcount' : 1, 'method' : method}
+
+         if zrecs: zcnt += add_zusage_records(zrecs, cdate)
+         azcnt += zcnt
+         allcnt += pgcnt
+         rmsg = PgLOG.seconds_to_string_time(tm() - bt)
+         PgLOG.pglog("{}: {} usage records added for {} ZENODO entries in {}".format(strids, azcnt, allcnt, rmsg), PgLOG.LOGWRN)
+
+def get_record_date_time(ctime):
+
+   ms = re.search(r'^(\d+)-(\d+)-(\d+) (\d\d:\d\d:\d\d)', str(ctime))
+   if ms:
+      y = ms.group(1)
+      m = int(ms.group(2))
+      d = ms.group(3)
+      q = 1 + int((m-1)/3)
+      t = ms.group(4)
+      return (y, q, "{}-{:02}-{}".format(y, m, d), t)
+   else:
+      PgLOG.pglog(str(ctime) + ": Invalid time format", PgLOG.LGEREX)
+
+def add_zusage_records(records, date):
+
+   cnt = 0
+   for key in records:
+      record = records[key]
+      cond = "date = '{}' AND time = '{}' AND ip = '{}'".format(date, record['time'], record['ip'])
+      if PgDBI.pgget(USAGE['ZNDTBL'], '', cond, PgLOG.LGEREX): continue
+
+      cnt += PgDBI.pgadd(USAGE['ZNDTBL'], record, PgLOG.LOGWRN)
+
+   PgLOG.pglog("{}: {} ZND usage records added at {}".format(date, cnt, PgLOG.current_datetime()), PgLOG.LOGWRN)
+
+   return cnt
+
+#
+# call main() to start program
+#
+if __name__ == "__main__": main()

{rda_python_metrics-1.0.40.dist-info → rda_python_metrics-1.0.42.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_metrics
-Version: 1.0.40
+Version: 1.0.42
 Summary: RDA Python Package to gather and view data usage metrics
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-metrics

{rda_python_metrics-1.0.40.dist-info → rda_python_metrics-1.0.42.dist-info}/RECORD

@@ -3,13 +3,13 @@ rda_python_metrics/PgView.py,sha256=r6otb3DjfiaQJdg0z8bZQAOlhr4JnrXJzp9wgWh_8qQ,
 rda_python_metrics/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
 rda_python_metrics/fillawsusage.py,sha256=bylk7m8VJyXMZMZwyPkU3rQqIdttjkeo8aTYeneqMDQ,7073
 rda_python_metrics/fillawsusage.usg,sha256=-lLTRLTaEPL2rSQ4mhpIekhyYrLGahy7NWtaq_8PCDM,611
-rda_python_metrics/fillcdgusage.py,sha256=jJVxQ4d5at0lXXKmV7TVf2GrPWSrJv5XBiHGMc3uxEA,16706
+rda_python_metrics/fillcdgusage.py,sha256=CSfyefeUJjGGLUlsH5H8C9d14WWtUJ_jPkyS7AFmL3o,16636
 rda_python_metrics/fillcdgusage.usg,sha256=5lkd4Zdi72nQcha-JtbaLnxl66V4wBDykKwQtUOtMrw,667
 rda_python_metrics/fillcodusage.py,sha256=_RUcgY1Cf17dSjidhQCMOwVaPO1VL26s3bYGT8oSPWU,8322
 rda_python_metrics/fillcodusage.usg,sha256=3B5IkQ4uwylqY8uEfUdnZ_MXqhYudeylMp5ulhUGXH8,678
 rda_python_metrics/fillcountry.py,sha256=7i5LNi3scRoyRCT6t7aeNTGKOpxzJ2mA9tnvUqje2AU,2314
 rda_python_metrics/fillendtime.py,sha256=skZttlpoY19g0dGwqGQI8t_1YPPTPEXwg3EfNlfL90I,2533
-rda_python_metrics/fillgdexusage.py,sha256=8KR5Lt30VCTxWOtc9EStLFzq5qa2di8RRQ3qMuIvSYY,37818
+rda_python_metrics/fillgdexusage.py,sha256=9U8k2WykGAmnyKvY6ZiYqzpw9MXpE1fpyJUjFDMb_Fc,38258
 rda_python_metrics/fillgdexusage.usg,sha256=mVYtK0pIYmvma0skT-wXM-NOEpkN_i3E61UdWgQWLfs,648
 rda_python_metrics/fillglobususage.py,sha256=zuxzoeV9BAMoVUu_VCYECPddYNrBWpV74kkYpyqIQhg,8443
 rda_python_metrics/fillglobususage.usg,sha256=1GgmCP22IQZdADwL5Mmkz3v8Ws-G7U3teQ1AxRJfV_4,637
@@ -19,12 +19,13 @@ rda_python_metrics/filloneorder.py,sha256=ADHbcKCDh9bJunnxYbkbjwU2QpC43hvGlLWaUR
 rda_python_metrics/filloneorder.usg,sha256=mtOySKx6-D4k2bbTcmi6cSYtINiycRyHQkHozi0CQu0,1466
 rda_python_metrics/fillosdfusage.py,sha256=--c6PzeZ6EZXpb0zxy4lZCDjwwIBb7c6xgP7QCKLoMQ,7087
 rda_python_metrics/fillosdfusage.usg,sha256=Qc5YdUuOiAH74FfVdkwkrQRDYXcASVbxMdBsVQj4X1k,635
-rda_python_metrics/fillrdadb.py,sha256=cb6upPApAZloOouUSzZZmjGvOsYT_Nzh9Lz926WE3ZQ,5333
+rda_python_metrics/fillrdadb.py,sha256=5OIzLUo-MHWVIA3sQE7CRMQSW8tX9PH7DocDy4rvbYA,5282
 rda_python_metrics/fillrdadb.usg,sha256=E_Bf4G8yVABogjRmIOaIbTGgnII2W6RltaFad2XEV2Q,1228
 rda_python_metrics/filltdsusage.py,sha256=vOwVzAtWUHO4O-FCSJMg0GKxw-Xc5AzSbfqVFktUAlA,10201
 rda_python_metrics/filltdsusage.usg,sha256=yqTHRe8WpZWpu3gso_obgt6LO41-JC27rTIDfdZcISo,538
 rda_python_metrics/filluser.py,sha256=CvaMRaUPaR9nxJAExkLTb3Ci4sD7RQMOpWhWJdbyMF0,8907
 rda_python_metrics/filluser.usg,sha256=Xgqi0QwA9-4jpYj3L4Q4TISpVwRlsomt2G7T0oYAFak,520
+rda_python_metrics/fillzenodousage.py,sha256=BWm1SLNNX7nhO6he9Ccl6lH9jUY9G83AmrusDMY6Dc8,13204
 rda_python_metrics/logarch.py,sha256=8OFirXeO24dljXQPbNJqGDvP9F0qWK11VHSw38CdUuM,13896
 rda_python_metrics/logarch.usg,sha256=6ksFxu_-lzhRpQE2YFAqgGVhwFP4PTweCbFBuGhhICU,960
 rda_python_metrics/pgperson.py,sha256=4XTDnVBFyF9drzx2I_DGCxQ-8I1bpclqTuQvOx3wST4,2088
@@ -50,9 +51,9 @@ rda_python_metrics/viewwebfile.py,sha256=HSMNkQQawonu6W3blV7g9UbJuNy9VAOn9COqgmj
 rda_python_metrics/viewwebfile.usg,sha256=lTNi8Yu8BUJuExEDJX-vsJyWUSUIQTS-DiiBEVFo33s,10054
 rda_python_metrics/viewwebusage.py,sha256=ES2lI8NaCeCpTGi94HU-cDRBxHMiUBbplyYsZf2KqF0,16650
 rda_python_metrics/viewwebusage.usg,sha256=OVDZ78p87E3HLW34ZhasNJ7Zmw8XXjmZPPWZfRhPLXo,9936
-rda_python_metrics-1.0.40.dist-info/licenses/LICENSE,sha256=1dck4EAQwv8QweDWCXDx-4Or0S8YwiCstaso_H57Pno,1097
-rda_python_metrics-1.0.40.dist-info/METADATA,sha256=Cp0ty6JjaTV6osYkeuO7L1j0X2zXZnphZxV3rETxD4o,761
-rda_python_metrics-1.0.40.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-rda_python_metrics-1.0.40.dist-info/entry_points.txt,sha256=_Cw8KvtYwxLPt3kLJHwQ1r2swsKKbXk-2zYPZAg2czc,1345
-rda_python_metrics-1.0.40.dist-info/top_level.txt,sha256=aoBgbR_o70TP0QmMW0U6inRHYtfKld47OBmnWnLnDOs,19
-rda_python_metrics-1.0.40.dist-info/RECORD,,
+rda_python_metrics-1.0.42.dist-info/licenses/LICENSE,sha256=1dck4EAQwv8QweDWCXDx-4Or0S8YwiCstaso_H57Pno,1097
+rda_python_metrics-1.0.42.dist-info/METADATA,sha256=X6jSYha4z0rwsLp1cFusK5TL2fAFzQbHNqEQt0ocJlw,761
+rda_python_metrics-1.0.42.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+rda_python_metrics-1.0.42.dist-info/entry_points.txt,sha256=fv4wjyEdiLFYNaJzhmet0SxZcF_J6M42koft2Na1XMs,1403
+rda_python_metrics-1.0.42.dist-info/top_level.txt,sha256=aoBgbR_o70TP0QmMW0U6inRHYtfKld47OBmnWnLnDOs,19
+rda_python_metrics-1.0.42.dist-info/RECORD,,

{rda_python_metrics-1.0.40.dist-info → rda_python_metrics-1.0.42.dist-info}/entry_points.txt

@@ -12,6 +12,7 @@ fillosdfusage = rda_python_metrics.fillosdfusage:main
 fillrdadb = rda_python_metrics.fillrdadb:main
 filltdsusage = rda_python_metrics.filltdsusage:main
 filluser = rda_python_metrics.filluser:main
+fillzenodousage = rda_python_metrics.fillzenodousage:main
 logarch.py = rda_python_metrics.logarch:main
 pgperson = rda_python_metrics.pgperson:main
 pgusername = rda_python_metrics.pgusername:main