rda-python-metrics 1.0.5__tar.gz → 1.0.7__tar.gz

{rda_python_metrics-1.0.5 → rda_python_metrics-1.0.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_metrics
-Version: 1.0.5
+Version: 1.0.7
 Summary: RDA Python Package to gather and view data usage metrics
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-metrics

{rda_python_metrics-1.0.5 → rda_python_metrics-1.0.7}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "rda_python_metrics"
-version = "1.0.5"
+version = "1.0.7"
 authors = [
   { name="Zaihua Ji", email="zji@ucar.edu" },
 ]

{rda_python_metrics-1.0.5 → rda_python_metrics-1.0.7}/src/rda_python_metrics/fillosdfusage.py

@@ -23,7 +23,7 @@ from . import PgIPInfo
 
 USAGE = {
    'OSDFTBL'  : "wusage",
-   'OSDFDIR' : PgLOG.PGLOG["DSSDATA"] + "/work/logs/gridftp/",
+   'OSDFDIR' : PgLOG.PGLOG["DSSDATA"] + "/work/zji/osdflogs/",
    'OSDFGET' : 'wget -m -nH -np -nd https://pelicanplatform.org/pelican-access-logs/ncar-access-log/',
    'OSDFLOG' : "{}.log",   # YYYY-MM-DD.log
 }
@@ -95,7 +95,7 @@ def get_log_file_names(option, params, datelimits):
          else:
             edate = PgUtil.curdate()
       pdays = PgUtil.get_weekday(pdate)
-      if pdays > 0: pdate = PgUtil.adddate(edate, 0, 0, 7-pdays)
+      if pdays > 0: pdate = PgUtil.adddate(pdate, 0, 0, 7-pdays)
       while pdate <= edate:
          filenames.append(USAGE['OSDFLOG'].format(pdate))
          pdate = PgUtil.adddate(pdate, 0, 0, 7)
@@ -131,7 +131,7 @@ def fill_osdf_usages(fnames, datelimits):
          if entcnt%10000 == 0:
             PgLOG.pglog("{}: {}/{} OSDF log entries processed/records added".format(logfile, entcnt, cntadd), PgLOG.WARNLG)
 
-         ms = re.match(r'^\[(\S+)\] \[Objectname:(\/ncar\/rda\/([a-z]\d{6})\/(\S+)\] \[Host:(\S+)\] \[server:(\S+)\] \[Read:(\d+)\]', line)
+         ms = re.match(r'^\[(\S+)\] \[Objectname:\/ncar\/rda\/([a-z]\d{6})\/(\S+)\] \[Host:(\S+)\] \[Server:(\S+)\] \[Read:(\d+)\]', line)
          if not ms: continue
          size = int(ms.group(6))
          if size < 100: continue  # ignore small files

{rda_python_metrics-1.0.5 → rda_python_metrics-1.0.7}/src/rda_python_metrics/fillrdadb.py

@@ -101,6 +101,8 @@ def fill_rdadb(option, params):
    PgLOG.pgsystem("fillglobususage {} {}".format(RDADB['BCKGRND'], filecond), PgLOG.LGWNEM, 5)
    # fill available AWS web data usages
    PgLOG.pgsystem("fillawsusage {} {}".format(RDADB['BCKGRND'], filecond), PgLOG.LGWNEM, 5)
+   # fill available OSDF web data usages
+   PgLOG.pgsystem("fillosdfusage {} {}".format(RDADB['BCKGRND'], filecond), PgLOG.LGWNEM, 5)
 
    if RDADB['DOMAIL']: send_email_notice()
    PgLOG.pglog("End Filling RDADB info at {}".format(PgLOG.current_datetime()), PgLOG.LGWNEM)

{rda_python_metrics-1.0.5 → rda_python_metrics-1.0.7}/src/rda_python_metrics/logarch.py

@@ -23,16 +23,18 @@ from rda_python_common import PgFile
 from rda_python_common import PgSIG
 
 # the defined options for archiving different logs
-WLOG = 0x01  # archive web log
+WLOG = 0x21  # archive web log
 TLOG = 0x02  # archive tds log
 DLOG = 0x04  # archive dssdb logs
 SLOG = 0x08  # append dssdb sub batch logs
 ALOG = 0x10  # archive AWS web log
+OLOG = 0x20  # archive OSDF web log
 
 LOGS = {
    'OPTION' : 0,
    'AWSLOG' : PgLOG.PGLOG["TRANSFER"] + "/AWSera5log",
    'WEBLOG' : PgLOG.PGLOG["DSSDATA"] + "/work/logs/gridftp",
+   'OSDFLOG' : PgLOG.PGLOG["DSSDATA"] + "/zji/osdflogs",
    'MGTLOG' : "/data/logs",
    'TDSLOG' : "/data/logs/nginx",
    'RDALOG' : PgLOG.PGLOG['LOGPATH'],
@@ -70,6 +72,8 @@ def main():
             LOGS['OPTION'] |= WLOG
          elif option == "a":
             LOGS['OPTION'] |= ALOG
+         elif option == "o":
+            LOGS['OPTION'] |= OLOG
          elif option == "s":
             LOGS['OPTION'] |= SLOG
          elif option == "t":
@@ -91,6 +95,7 @@ def main():
    if LOGS['OPTION']&SLOG: append_dssdb_sublog()
    if LOGS['OPTION']&DLOG: archive_dssdb_log()
    if LOGS['OPTION']&WLOG: archive_web_log(smonth)
+   if LOGS['OPTION']&OLOG: archive_osdf_log(smonth)
    if LOGS['OPTION']&ALOG: archive_aws_log(smonth)
    if LOGS['OPTION']&TLOG: archive_tds_log(smonth)
 
@@ -142,6 +147,38 @@ def archive_web_log(smonth):
       s = 's' if tcnt > 1 else ''
       PgLOG.pglog("{}: {} globus log{} tarred, gzipped and archived at {}".format(afile, tcnt, s, PgLOG.current_datetime()), PgLOG.LGWNEM)
 
+#
+# Archive OSDF web log files to LOGS['DECSLOGS']
+#
+def archive_osdf_log(smonth):
+
+   (yr, mn) = get_year_month(smonth)
+   PgFile.change_local_directory(LOGS['DECSLOGS'], PgLOG.LGEREM)
+   logpath = LOGS['LOGPATH'] if LOGS['LOGPATH'] else LOGS['OSDFLOG']
+   afile = "osdfweb{}-{}.log.tar".format(yr, mn)
+   dfile = "./OSDFLOG/{}.gz".format(afile)
+   if op.exists(dfile):
+      PgLOG.pglog("{}: file exists already under {}, remove it before backup again".format(dfile, LOGS['DECSLOGS']), PgLOG.LGWNEM)
+      return
+
+   if op.exists(afile): PgFile.delete_local_file(afile)
+
+   logfiles = sorted(glob.glob("{}/{}-{}-??.log".format(logpath, yr, mn)))
+   topt = '-cvf'
+   tcnt = 0
+   for logfile in logfiles:
+      lfile = op.basename(logfile)
+      tcmd = "tar {} {} -C {} {}".format(topt, afile, logpath, lfile)
+      tcnt += PgLOG.pgsystem(tcmd, PgLOG.LGWNEM, 5)
+      topt = '-uvf'
+   
+   if tcnt > 0:
+      PgLOG.pgsystem("gzip " + afile, PgLOG.LGWNEM, 5)
+      afile += '.gz'
+      PgFile.move_local_file(dfile, afile, PgLOG.LGWNEM)
+      s = 's' if tcnt > 1 else ''
+      PgLOG.pglog("{}: {} globus log{} tarred, gzipped and archived at {}".format(afile, tcnt, s, PgLOG.current_datetime()), PgLOG.LGWNEM)
+
 #
 # Archive AWS web log files to LOGS['DECSLOGS']
 #

{rda_python_metrics-1.0.5 → rda_python_metrics-1.0.7}/src/rda_python_metrics/logarch.usg

@@ -3,7 +3,7 @@
    at any time. It checks if the given log files are archived or not and will
    ignore the ones that are backed up already.
 
- Usage: logarch [-b] [-a] [-d] [-w] [-s] [-t] [-p LogPath]  \
+ Usage: logarch [-b] [-a] [-d] [-o] [-w] [-s] [-t] [-p LogPath]  \
                 [-m YearMonth] [-n]
 
       - Option -b, do not display processing info on screen;
@@ -12,6 +12,8 @@
 
       - Option -d, archive dssdb log files;
       
+      - Option -o, archive OSDF log files;
+
       - Option -p, provide a LogPath if not the fefault one;
 
       - Option -s, append PBS sublog files to common log files;

{rda_python_metrics-1.0.5 → rda_python_metrics-1.0.7}/src/rda_python_metrics.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_metrics
-Version: 1.0.5
+Version: 1.0.7
 Summary: RDA Python Package to gather and view data usage metrics
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-metrics