rda-python-metrics 1.0.30__tar.gz → 1.0.32__tar.gz

{rda_python_metrics-1.0.30/src/rda_python_metrics.egg-info → rda_python_metrics-1.0.32}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_metrics
-Version: 1.0.30
+Version: 1.0.32
 Summary: RDA Python Package to gather and view data usage metrics
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-metrics

{rda_python_metrics-1.0.30 → rda_python_metrics-1.0.32}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "rda_python_metrics"
-version = "1.0.30"
+version = "1.0.32"
 authors = [
   { name="Zaihua Ji",  email="zji@ucar.edu" },
 ]
@@ -44,6 +44,7 @@ pythonpath = [
 "fillcountry" = "rda_python_metrics.fillcountry:main"
 "fillendtime" = "rda_python_metrics.fillendtime:main"
 "fillglobususage" = "rda_python_metrics.fillglobususage:main"
+"fillgdexusage" = "rda_python_metrics.fillgdexusage:main"
 "fillipinfo" = "rda_python_metrics.fillipinfo:main"
 "filloneorder" = "rda_python_metrics.filloneorder:main"
 "fillosdfusage" = "rda_python_metrics.fillosdfusage:main"

{rda_python_metrics-1.0.30 → rda_python_metrics-1.0.32}/src/rda_python_metrics/PgIPInfo.py

@@ -237,12 +237,9 @@ def update_ipinfo_record(record, pgrec = None):
 #
 def set_ipinfo(ip, ipopt = True):
 
-   if ip in IPRECS:
-      pgrec = IPRECS[ip]
-      if pgrec or not ipopt: return pgrec
-   else:
-      pgrec = PgDBI.pgget('ipinfo', '*', "ip = '{}'".format(ip))
+   if ip in IPRECS: return IPRECS[ip]
 
+   pgrec = PgDBI.pgget('ipinfo', '*', "ip = '{}'".format(ip))
    if not pgrec or ipopt and pgrec['stat_flag'] == 'M':
       record = get_ipinfo_record(ip) if ipopt else None
       if not record: record = get_geoip2_record(ip)

{rda_python_metrics-1.0.30 → rda_python_metrics-1.0.32}/src/rda_python_metrics/fillcdgusage.py

@@ -31,28 +31,28 @@ USAGE = {
 }
 
 DSIDS = {
-   'ucar.cgd.cesm2.cam6.prescribed_sst_amip' : ['d651010'],
-   'ucar.cgd.ccsm4.CLM_LAND_ONLY' : ['d651011'],
-   'ucar.cgd.artmip' : ['d651012', 'd651016', 'd651017', 'd651018'],
-   'tamip' : ['d651013'],
-   'ucar.cgd.ccsm4.CLIVAR_LE' : ['d651014'],
-   'ucar.cgd.cesm2.Gettelman_CESM2_ECS' : ['d651015'],
-   'ucar.cgd.ccsm4.geomip.ssp5' : ['d651024'],
-   'ucar.cgd.ccsm4.IOD-PACEMAKER' : ['d651021'],
-   'ucar.cgd.ccsm4.past2k_transient' : ['651023'],
-   'ucar.cgd.ccsm4.lowwarming' : ['d651025'],
-   'ucar.cgd.ccsm4.CESM_CAM5_BGC_ME' : ['d651000'],
-   'ucar.cgd.ccsm4.iTRACE' : ['d651022'],
-   'ucar.cgd.ccsm4.so2_geoeng' : ['d651026'],
-   'ucar.cgd.ccsm4.cesmLE' : ['d651027'],
-   'ucar.cgd.ccsm4.CESM1-CAM5-DP' : ['d651028'],
-   'ucar.cgd.ccsm4.amv_lens' : ['d651031'],
-   'ucar.cgd.ccsm4.ATL-PACEMAKER' : ['d651032'],
-   'ucar.cgd.ccsm4.pac-pacemaker' : ['d651033'],
-   'ucar.cgd.ccsm4.SD-WACCM-X_v2.1' : ['d651034'],
-   'ucar.cgd.ccsm4.amv_lens' : ['d651035'],
-   'ucar.cgd.cesm2.cism_ismip6' : ['d651036'],
-   'ucar.cgd.ccsm4.pliomip2' : ['d651037'],
+#   'ucar.cgd.cesm2.cam6.prescribed_sst_amip' : ['d651010'],
+#   'ucar.cgd.ccsm4.CLM_LAND_ONLY' : ['d651011'],
+#   'ucar.cgd.artmip' : ['d651012', 'd651016', 'd651017', 'd651018'],
+#   'tamip' : ['d651013'],
+#   'ucar.cgd.ccsm4.CLIVAR_LE' : ['d651014'],
+#   'ucar.cgd.cesm2.Gettelman_CESM2_ECS' : ['d651015'],
+#   'ucar.cgd.ccsm4.geomip.ssp5' : ['d651024'],
+#   'ucar.cgd.ccsm4.IOD-PACEMAKER' : ['d651021'],
+#   'ucar.cgd.ccsm4.past2k_transient' : ['651023'],
+#   'ucar.cgd.ccsm4.lowwarming' : ['d651025'],
+#   'ucar.cgd.ccsm4.CESM_CAM5_BGC_ME' : ['d651000'],
+#   'ucar.cgd.ccsm4.iTRACE' : ['d651022'],
+#   'ucar.cgd.ccsm4.so2_geoeng' : ['d651026'],
+#   'ucar.cgd.ccsm4.cesmLE' : ['d651027'],
+#   'ucar.cgd.ccsm4.CESM1-CAM5-DP' : ['d651028'],
+#   'ucar.cgd.ccsm4.amv_lens' : ['d651031'],
+#   'ucar.cgd.ccsm4.ATL-PACEMAKER' : ['d651032'],
+#   'ucar.cgd.ccsm4.pac-pacemaker' : ['d651033'],
+#   'ucar.cgd.ccsm4.SD-WACCM-X_v2.1' : ['d651034'],
+#   'ucar.cgd.ccsm4.amv_lens' : ['d651035'],
+#   'ucar.cgd.cesm2.cism_ismip6' : ['d651036'],
+#   'ucar.cgd.ccsm4.pliomip2' : ['d651037'],
    # new added
    'ucar.cgd.cesm2-waccm.s2s_hindcasts': ['d651040'],
    'ucar.cgd.CESM1.3_SH_storm_tracks': ['d651044'],
@@ -61,18 +61,32 @@ DSIDS = {
    'ucar.cgd.ccsm4.TC-CESM': ['d651047'],
    'ucar.cgd.cesm2.ISSI_OSSE': ['d651048'],
    'ucar.cgd.ccsm4.SOcean_Eddies_mclong': ['d651049'],
-   'trace': ['d651050'],
+   'ucar.cgd.ccsm.trace': ['d651050'],
    'ucar.cgd.cesm2.waccm.solar': ['d651051'],
    'ucar.cgd.ccsm4.CESM1-CCSM4_mid-Pliocene' : ['d651042'],
    'ucar.cgd.ccsm4.PaleoIF' : ['d651052'],
    'ucar.cgd.ccsm4.b.e11.B20LE_fixedO3' : ['d651053'],
    'ucar.cgd.cesm2.single.forcing.large.ensemble' : ['d651055'],
-   'ucar.cgd.cesm2le.output': ['d651056'],
+   'ucar.cgd.cesm2le.output': ['d651056'],    # 2025-01 done
+   'ucar.cgd.ccsm4.ARISE-SAI-1.5' : ['d651059'],
    'ucar.cgd.cesm2.s2s_hindcasts': ['d651060'],
    'ucar.cgd.cesm2.s2s_hindcasts.mjo': ['d651061'],
    'ucar.cgd.cesm2.s2s_hindcasts.tc_tracks': ['d651062'],
    'ucar.cgd.cesm2.s2s_hindcasts.cesm2.climo': ['d651063'],
-   'ucar.cgd.ccsm4.ARISE-SAI-1.5' : ['d651059']
+   #add 2025-08-06
+   'ucar.cgd.ccsm4.cesmLME' : ['d651058'],
+   'ucar.cgd.ccsm4.GLENS' : ['d651064'],
+   'ucar.cgd.ccsm4.CESM2-CISM2-LIGtransient' : ['d651066'],
+   'ucar.cgd.cesm2.pacific.pacemaker' : ['d651068'],
+   'ucar.cgd.cesm2.tuned.sea.ice.albedo' : ['d651070'],
+   'ucar.cgd.cesm2.cmip5.forcing' : ['d651075'],
+   'ucar.cgd.cesm2.ssp245.biomass.burning' : ['d651073'],
+   'ucar.cgd.cesm2.ssp585.biomass.burning' : ['d651067'],
+   'ucar.cgd.cesm1.cldmod': ['d651069'],
+   'ucar.cgd.cesm2.marine.biogeochemistry': ['d651071'],
+   'ucar.cgd.nw2.mom6': ['d651072'],
+#   'ucar.cgd.cesm2.smyle': ['d651065'],  # not done util the end of Aug.
+   
 }
 
 ALLIDS = list(DSIDS.keys())
@@ -225,12 +239,14 @@ def fill_cdg_usages(dsids, dranges):
 
    allcnt = awcnt = atcnt = lcnt = 0
    for dates in dranges:
-      for dsid in dsids:
+      for adsid in dsids:
          lcnt += 1
-         dsname = dsid[0]
-         rdaids = dsid[1]
-         cdgids = dsid[2]
-         strids = dsid[3]
+         dsname = adsid[0]
+         rdaids = adsid[1]
+         getdsid = False if len(rdaids) == 1 else True
+         dsid = rdaids[0]
+         cdgids = adsid[2]
+         strids = adsid[3]
          bt = tm()
          pgrecs = get_dsid_records(cdgids, dates, strids)
          pgcnt = len(pgrecs['dataset_file_name']) if pgrecs else 0
@@ -248,6 +264,10 @@ def fill_cdg_usages(dsids, dranges):
                PgLOG.pglog("{}/{}/{} CDG/TDS/WEB records processed to add".format(i, tcnt, wcnt), PgLOG.WARNLG)
 
             pgrec = PgUtil.onerecord(pgrecs, i)
+            wfile = pgrec['dataset_file_name']
+            if not wfile:
+               wfile = pgrec['logic_file_name']
+               if not wfile: continue
             dsize = pgrec['bytes_sent']
             if not dsize: continue
             (year, quarter, date, time) = get_record_date_time(pgrec['date_completed'])
@@ -255,14 +275,13 @@ def fill_cdg_usages(dsids, dranges):
             if not url: url = pgrec['file_access_point_uri']
             ip = pgrec['remote_address']
             engine = pgrec['user_agent_name']
-            wfile = pgrec['dataset_file_name']
-            if not wfile: wfile = pgrec['logic_file_name']
-            wfrec = get_wfile_record(rdaids, wfile)
-            if not wfrec: continue
-            dsid = wfrec['dsid']
             ms = re.search(r'^https*://tds.ucar.edu/thredds/(\w+)/', url)
             if ms:
                # tds usage
+               if getdsid:
+                  wfrec = get_wfile_record(rdaids, wfile)
+                  if not wfrec: continue
+                  dsid = wfrec['dsid']
                method = ms.group(1)
                if pgrec['subset_file_size']:
                   etype = 'S'
@@ -289,6 +308,9 @@ def fill_cdg_usages(dsids, dranges):
                                  'region' : iprec['region'], 'email' : iprec['email']}
             else:
                # web usage
+               wfrec = get_wfile_record(rdaids, wfile)
+               if not wfrec: continue
+               if getdsid: dsid = wfrec['dsid']
                fsize = pgrec['dataset_file_size']
                if not fsize: fsize = pgrec['logic_file_size']
                method = 'CDG'