rda-python-metrics 1.0.25__tar.gz → 1.0.27__tar.gz

{rda_python_metrics-1.0.25/src/rda_python_metrics.egg-info → rda_python_metrics-1.0.27}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_metrics
-Version: 1.0.25
+Version: 1.0.27
 Summary: RDA Python Package to gather and view data usage metrics
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-metrics

{rda_python_metrics-1.0.25 → rda_python_metrics-1.0.27}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "rda_python_metrics"
-version = "1.0.25"
+version = "1.0.27"
 authors = [
   { name="Zaihua Ji",  email="zji@ucar.edu" },
 ]

{rda_python_metrics-1.0.25 → rda_python_metrics-1.0.27}/src/rda_python_metrics/PgIPInfo.py

@@ -109,16 +109,16 @@ def domain_ipinfo_record(dmname):
 #
 def get_ip_hostname(ip, iprec, record):
 
+   record['hostname'] = ip
    if iprec:
       if 'hostname' in iprec and iprec['hostname']:
          record['hostname'] = iprec['hostname']         
          record['org_type'] = PgDBI.get_org_type(None, record['hostname'])
          return
-   record['hostname'] = ip
-   if 'domain' in iprec and iprec['domain']:
-      record['hostname'] += '.' + iprec['domain']
-      record['org_type'] = PgDBI.get_org_type(None, record['hostname'])
-      return
+      if 'asn' in iprec and iprec['asn'] and 'domain' in iprec['asn'] and iprec['asn']['domain']:
+         record['hostname'] += '.' + iprec['asn']['domain']
+         record['org_type'] = PgDBI.get_org_type(None, record['hostname'])
+         return
 
    try:
       hostrec = socket.gethostbyaddr(ip)

{rda_python_metrics-1.0.25 → rda_python_metrics-1.0.27}/src/rda_python_metrics/fillcdgusage.py

@@ -66,7 +66,13 @@ DSIDS = {
    'ucar.cgd.ccsm4.CESM1-CCSM4_mid-Pliocene' : ['d651042'],
    'ucar.cgd.ccsm4.PaleoIF' : ['d651052'],
    'ucar.cgd.ccsm4.b.e11.B20LE_fixedO3' : ['d651053'],
-   'ucar.cgd.cesm2.single.forcing.large.ensemble' : ['d651055']
+   'ucar.cgd.cesm2.single.forcing.large.ensemble' : ['d651055'],
+   'ucar.cgd.cesm2le.output': ['d651056'],
+   'ucar.cgd.cesm2.s2s_hindcasts': ['d651060'],
+   'ucar.cgd.cesm2.s2s_hindcasts.mjo': ['d651061'],
+   'ucar.cgd.cesm2.s2s_hindcasts.tc_tracks': ['d651062'],
+   'ucar.cgd.cesm2.s2s_hindcasts.cesm2.climo': ['d651063'],
+   'ucar.cgd.ccsm4.ARISE-SAI-1.5' : ['d651059']
 }
 
 ALLIDS = list(DSIDS.keys())

{rda_python_metrics-1.0.25 → rda_python_metrics-1.0.27}/src/rda_python_metrics/fillipinfo.py

@@ -113,7 +113,7 @@ def fill_ip_info(option, inputs, table):
 def get_next_date(date, edate):
 
    if date < edate:
-      ndate = PgUtil.enddate(date, 'M')
+      ndate = PgUtil.enddate(date, 0, 'M')
       if ndate < edate: edate = ndate
    if date < edate:
       cond = f"BETWEEN '{date}' AND '{edate}'"
@@ -186,14 +186,13 @@ def fix_wuser_records(date, cnd):
 
    table = 'wuser'
    cond = f"start_date {cnd} AND region IS NULL"
-   pgrecs = PgDBI.pgmget(table, 'wuid, email, ip', cond, PgLOG.LGEREX)
+   pgrecs = PgDBI.pgmget(table, 'wuid, email', cond, PgLOG.LGEREX)
    if not pgrecs: return 0
-   cnt = len(pgrecs['ip']) if pgrecs else 0
+   cnt = len(pgrecs['wuid']) if pgrecs else 0
    mcnt = 0
    for i in range(cnt):
-      ip = pgrecs['ip'][i]
       email = pgrecs['email'][i]
-      record = PgIPInfo.get_missing_ipinfo(ip, email)
+      record = PgIPInfo.get_missing_ipinfo(None, email)
       if record:
          mcnt += PgDBI.pgupdt(table, record, "wuid = '{}'".format(pgrecs['wuid'][i]))
 

{rda_python_metrics-1.0.25 → rda_python_metrics-1.0.27}/src/rda_python_metrics/fillosdfusage.py

@@ -25,7 +25,7 @@ USAGE = {
    'OSDFTBL'  : "wusage",
    'OSDFDIR' : PgLOG.PGLOG["DSSDATA"] + "/work/zji/osdflogs/",
    'OSDFGET' : 'wget -m -nH -np -nd https://pelicanplatform.org/pelican-access-logs/ncar-access-log/',
-   'OSDFLOG' : "{}.log",   # YYYY-MM-DD.log
+   'OSDFLOG' : "{}-cache.log",   # YYYY-MM-DD-cache.log
 }
 
 #
@@ -37,7 +37,6 @@ def main():
    argv = sys.argv[1:]
    option = None
    datelimits = [None, None]
-   
 
    for arg in argv:
       ms = re.match(r'^-(b|d|p|N)$', arg)
@@ -64,7 +63,7 @@ def main():
    PgFile.change_local_directory(USAGE['OSDFDIR'])
    filenames = get_log_file_names(option, params, datelimits)
    if filenames:
-      fill_osdf_usages(filenames, datelimits)
+      fill_osdf_usages(filenames)
    else:
       PgLOG.pglog("No log file found for given command: " + cmdstr, PgLOG.LOGWRN)
 
@@ -79,10 +78,6 @@ def get_log_file_names(option, params, datelimits):
    filenames = []
    if option == 'd':
       for pdate in params:
-         pdays = PgUtil.get_weekday(pdate)
-         if pdays > 0:
-            PgLOG.pglog(pdate + ": Skip a Non-Sunday date", PgLOG.LOGWRN)
-            continue
          filenames.append(USAGE['OSDFLOG'].format(pdate))
    else:
       if option == 'N':
@@ -94,32 +89,34 @@ def get_log_file_names(option, params, datelimits):
             edate = datelimits[1] = params[1]
          else:
             edate = PgUtil.curdate()
-      pdays = PgUtil.get_weekday(pdate)
-      if pdays > 0: pdate = PgUtil.adddate(pdate, 0, 0, 7-pdays)
       while pdate <= edate:
          filenames.append(USAGE['OSDFLOG'].format(pdate))
-         pdate = PgUtil.adddate(pdate, 0, 0, 7)
+         pdate = PgUtil.adddate(pdate, 0, 0, 1)
 
    return filenames
 
 #
 # Fill OSDF usages into table dssdb.osdfusage of DSS PgSQL database from osdf access logs
 #
-def fill_osdf_usages(fnames, datelimits):
+def fill_osdf_usages(fnames):
 
    cntall = addall = 0
 
    fcnt = len(fnames)
    for logfile in fnames:
-      PgLOG.pgsystem(USAGE['OSDFGET'] + logfile, 5, PgLOG.LOGWRN)
       linfo = PgFile.check_local_file(logfile)
       if not linfo:
-         PgLOG.pglog("{}: Not exists for Gathering OSDF usage".format(logfile), PgLOG.LOGWRN)
-         continue
-      if linfo['data_size'] == 0:
-         PgLOG.pglog("{}: Empty log for Gathering OSDF usage".format(logfile), PgLOG.LOGWRN)
-         continue
-      PgLOG.pglog("Gathering usage info from {} at {}".format(logfile, PgLOG.current_datetime()), PgLOG.LOGWRN)
+         xzfile = logfile + '.xz'
+         PgLOG.pgsystem(USAGE['OSDFGET'] + xzfile, 5, PgLOG.LOGWRN)
+         linfo = PgFile.check_local_file(xzfile)
+         if not linfo:
+            PgLOG.pglog("{}: Not exists for Gathering OSDF usage".format(xzfile), PgLOG.LOGWRN)
+            continue
+         PgFile.compress_local_file(xzfile)
+         linfo = PgFile.check_local_file(logfile)
+         if not linfo:
+            PgLOG.pglog("{}: Error unxz OSDF usage".format(xzfile), PgLOG.LGEREX)
+      PgLOG.pglog("{}: Gathering OSDF usage at {}".format(logfile, PgLOG.current_datetime()), PgLOG.LOGWRN)
       osdf = PgFile.open_local_file(logfile)
       if not osdf: continue
       cntadd = entcnt = 0
@@ -131,19 +128,26 @@ def fill_osdf_usages(fnames, datelimits):
          if entcnt%10000 == 0:
             PgLOG.pglog("{}: {}/{} OSDF log entries processed/records added".format(logfile, entcnt, cntadd), PgLOG.WARNLG)
 
-         ms = re.match(r'^\[(\S+)\] \[Objectname:\/ncar\/rda\/([a-z]\d{6})\/(\S+)\] \[Host:(\S+)\] \[Server:(\S+)\] \[Read:(\d+)\]', line)
+         ms = re.match(r'^\[(\S+)\] \[Objectname:\/ncar\/rda\/([a-z]\d{6})\/(\S+)\].* \[Host:(\S+)\].* \[AppInfo:(\S+)\].* \[Read:(\d+)\]', line)
          if not ms: continue
-         size = int(ms.group(6))
-         if size < 100: continue  # ignore small files
-         ip = ms.group(4)
-         dsid = PgUtil.format_dataset_id(ms.group(2))
+         dt = ms.group(1)
+         dsid = ms.group(2)
          wfile = ms.group(3)
+         ip = ms.group(4)
          engine = ms.group(5)
-         
-         (year, quarter, date, time) = get_record_date_time(ms.group(1))
-         if datelimits[0] and date < datelimits[0]: continue
-         if datelimits[1] and date > datelimits[1]: continue
+         size = int(ms.group(6))
+         (year, quarter, date, time) = get_record_date_time(dt)
          locflag = 'C'
+         if re.match(r'^curl', engine, re.I):
+            method = "CURL"
+         elif re.match(r'^wget', engine, re.I):
+            method = "WGET"
+         elif re.match(r'^python', engine, re.I):
+            method = "PYTHN"
+         elif re.match(r'^N/A', engine, re.I):
+            method = "N/A"
+         else:
+            method = "WEB"
          method = "OSDF"
 
          record = {'ip' : ip, 'dsid' : dsid, 'wfile' : wfile, 'date' : date,
@@ -209,7 +213,7 @@ def add_to_allusage(year, logrec, wurec):
    pgrec['size'] = logrec['size']
    pgrec['method'] = logrec['method']
    pgrec['ip'] = logrec['ip']
-   pgrec['source'] = 'W'
+   pgrec['source'] = 'P'
    return PgDBI.add_yearly_allusage(year, pgrec)
 
 #

{rda_python_metrics-1.0.25 → rda_python_metrics-1.0.27}/src/rda_python_metrics/viewallusage.usg

@@ -53,7 +53,8 @@
               E*- EMAIL        - user email address
               F*- FORMAT       - dataset default data format
               G*- GROUP        - data usage source group: W - Web log, G - Globus,
-                                 T - Thredds, O - Order, and A - AWS log
+                                 T - Thredds, O - Request Order, A - AWS log,
+                                 C - CDG GDEX DB, and P - OSDF Pelican Platform
               K*- REGION       - Region Name users from
               M*- MONTH        - format as YYYY-MM, for example 2004-04
               N*- COUNTRY      - country codes users from
@@ -135,7 +136,8 @@
       - Option -f, for data in datasets with specified data format;
 
       - Option -g, for data usage source groups, W - Web log, G - Globus,
-           T - Thredds, O - Order, and A - AWS log;
+           T - Thredds, O - Request Order, A - AWS log, C - CDG GDEX DB,
+           and P - OSDF logs on Pelican Platform;
 
       - Option -h, works with Option -e to include historical user emails
            registered before;

{rda_python_metrics-1.0.25 → rda_python_metrics-1.0.27/src/rda_python_metrics.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_metrics
-Version: 1.0.25
+Version: 1.0.27
 Summary: RDA Python Package to gather and view data usage metrics
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-metrics