rda-python-metrics 1.0.32__tar.gz → 1.0.46__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {rda_python_metrics-1.0.32/src/rda_python_metrics.egg-info → rda_python_metrics-1.0.46}/PKG-INFO +2 -1
- {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/pyproject.toml +7 -3
- rda_python_metrics-1.0.46/src/rda_python_metrics/fillawsusage.py +210 -0
- {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/fillawsusage.usg +1 -1
- {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/fillcdgusage.py +32 -27
- {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/fillgdexusage.py +51 -37
- {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/fillglobususage.py +9 -11
- {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/fillglobususage.usg +4 -2
- {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/fillosdfusage.py +57 -89
- {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/fillosdfusage.usg +1 -1
- {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/fillrdadb.py +1 -2
- rda_python_metrics-1.0.46/src/rda_python_metrics/fillzenodousage.py +408 -0
- {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/logarch.py +15 -8
- rda_python_metrics-1.0.46/src/rda_python_metrics/viewawsusage.py +321 -0
- rda_python_metrics-1.0.46/src/rda_python_metrics/viewawsusage.usg +190 -0
- rda_python_metrics-1.0.46/src/rda_python_metrics/viewosdfusage.py +321 -0
- rda_python_metrics-1.0.46/src/rda_python_metrics/viewosdfusage.usg +190 -0
- {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46/src/rda_python_metrics.egg-info}/PKG-INFO +2 -1
- {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics.egg-info/SOURCES.txt +5 -0
- {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics.egg-info/entry_points.txt +4 -1
- {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics.egg-info/requires.txt +1 -0
- rda_python_metrics-1.0.32/src/rda_python_metrics/fillawsusage.py +0 -254
- {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/LICENSE +0 -0
- {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/MANIFEST.in +0 -0
- {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/README.md +0 -0
- {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/setup.cfg +0 -0
- {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/PgIPInfo.py +0 -0
- {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/PgView.py +0 -0
- {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/__init__.py +0 -0
- {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/fillcdgusage.usg +0 -0
- {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/fillcodusage.py +0 -0
- {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/fillcodusage.usg +0 -0
- {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/fillcountry.py +0 -0
- {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/fillendtime.py +0 -0
- {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/fillgdexusage.usg +0 -0
- {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/fillipinfo.py +0 -0
- {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/fillipinfo.usg +0 -0
- {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/filloneorder.py +0 -0
- {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/filloneorder.usg +0 -0
- {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/fillrdadb.usg +0 -0
- {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/filltdsusage.py +0 -0
- {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/filltdsusage.usg +0 -0
- {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/filluser.py +0 -0
- {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/filluser.usg +0 -0
- {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/logarch.usg +0 -0
- {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/pgperson.py +0 -0
- {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/pgsyspath.py +0 -0
- {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/pgusername.py +0 -0
- {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/viewallusage.py +0 -0
- {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/viewallusage.usg +0 -0
- {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/viewcheckusage.py +0 -0
- {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/viewcheckusage.usg +0 -0
- {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/viewcodusage.py +0 -0
- {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/viewcodusage.usg +0 -0
- {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/viewordusage.py +0 -0
- {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/viewordusage.usg +0 -0
- {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/viewrqstusage.py +0 -0
- {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/viewrqstusage.usg +0 -0
- {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/viewtdsusage.py +0 -0
- {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/viewtdsusage.usg +0 -0
- {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/viewwebfile.py +0 -0
- {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/viewwebfile.usg +0 -0
- {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/viewwebusage.py +0 -0
- {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/viewwebusage.usg +0 -0
- {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics.egg-info/dependency_links.txt +0 -0
- {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics.egg-info/top_level.txt +0 -0
- {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/tests/test_metrics.py +0 -0
{rda_python_metrics-1.0.32/src/rda_python_metrics.egg-info → rda_python_metrics-1.0.46}/PKG-INFO
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: rda_python_metrics
|
|
3
|
-
Version: 1.0.
|
|
3
|
+
Version: 1.0.46
|
|
4
4
|
Summary: RDA Python Package to gather and view data usage metrics
|
|
5
5
|
Author-email: Zaihua Ji <zji@ucar.edu>
|
|
6
6
|
Project-URL: Homepage, https://github.com/NCAR/rda-python-metrics
|
|
@@ -17,6 +17,7 @@ Requires-Dist: geoip2
|
|
|
17
17
|
Requires-Dist: ipinfo
|
|
18
18
|
Requires-Dist: httplib2
|
|
19
19
|
Requires-Dist: dnspython
|
|
20
|
+
Requires-Dist: unidecode
|
|
20
21
|
Dynamic: license-file
|
|
21
22
|
|
|
22
23
|
RDA Python Package to gather and view data usage metrics.
|
|
@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
|
|
|
6
6
|
|
|
7
7
|
[project]
|
|
8
8
|
name = "rda_python_metrics"
|
|
9
|
-
version = "1.0.
|
|
9
|
+
version = "1.0.46"
|
|
10
10
|
authors = [
|
|
11
11
|
{ name="Zaihua Ji", email="zji@ucar.edu" },
|
|
12
12
|
]
|
|
@@ -25,7 +25,8 @@ dependencies = [
|
|
|
25
25
|
"geoip2",
|
|
26
26
|
"ipinfo",
|
|
27
27
|
"httplib2",
|
|
28
|
-
"dnspython"
|
|
28
|
+
"dnspython",
|
|
29
|
+
"unidecode"
|
|
29
30
|
]
|
|
30
31
|
|
|
31
32
|
[tool.pytest.ini_options]
|
|
@@ -38,13 +39,14 @@ pythonpath = [
|
|
|
38
39
|
|
|
39
40
|
[project.scripts]
|
|
40
41
|
"logarch.py" = "rda_python_metrics.logarch:main"
|
|
41
|
-
"fillawsusage" = "rda_python_metrics.
|
|
42
|
+
"fillawsusage" = "rda_python_metrics.fillawsusage:main"
|
|
42
43
|
"fillcdgusage" = "rda_python_metrics.fillcdgusage:main"
|
|
43
44
|
"fillcodusage" = "rda_python_metrics.fillcodusage:main"
|
|
44
45
|
"fillcountry" = "rda_python_metrics.fillcountry:main"
|
|
45
46
|
"fillendtime" = "rda_python_metrics.fillendtime:main"
|
|
46
47
|
"fillglobususage" = "rda_python_metrics.fillglobususage:main"
|
|
47
48
|
"fillgdexusage" = "rda_python_metrics.fillgdexusage:main"
|
|
49
|
+
"fillzenodousage" = "rda_python_metrics.fillzenodousage:main"
|
|
48
50
|
"fillipinfo" = "rda_python_metrics.fillipinfo:main"
|
|
49
51
|
"filloneorder" = "rda_python_metrics.filloneorder:main"
|
|
50
52
|
"fillosdfusage" = "rda_python_metrics.fillosdfusage:main"
|
|
@@ -54,9 +56,11 @@ pythonpath = [
|
|
|
54
56
|
"pgperson" = "rda_python_metrics.pgperson:main"
|
|
55
57
|
"pgusername" = "rda_python_metrics.pgusername:main"
|
|
56
58
|
"viewallusage" = "rda_python_metrics.viewallusage:main"
|
|
59
|
+
"viewawsusage" = "rda_python_metrics.viewawsusage:main"
|
|
57
60
|
"viewcheckusage" = "rda_python_metrics.viewcheckusage:main"
|
|
58
61
|
"viewcodusage" = "rda_python_metrics.viewcodusage:main"
|
|
59
62
|
"viewordusage" = "rda_python_metrics.viewordusage:main"
|
|
63
|
+
"viewosdfusage" = "rda_python_metrics.viewosdfusage:main"
|
|
60
64
|
"viewrqstusage" = "rda_python_metrics.viewrqstusage:main"
|
|
61
65
|
"viewtdsusage" = "rda_python_metrics.viewtdsusage:main"
|
|
62
66
|
"viewwebfile" = "rda_python_metrics.viewwebfile:main"
|
|
@@ -0,0 +1,210 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
#
|
|
3
|
+
###############################################################################
|
|
4
|
+
#
|
|
5
|
+
# Title : fillawsusage
|
|
6
|
+
# Author : Zaihua Ji, zji@ucar.edu
|
|
7
|
+
# Date : 03/11/2022
|
|
8
|
+
# 2025-03-26 transferred to package rda_python_metrics from
|
|
9
|
+
# https://github.com/NCAR/rda-database.git
|
|
10
|
+
# Purpose : python program to retrieve info from AWS logs
|
|
11
|
+
# and fill table wusages in PgSQL database dssdb.
|
|
12
|
+
#
|
|
13
|
+
# Github : https://github.com/NCAR/rda-pythn-metrics.git
|
|
14
|
+
#
|
|
15
|
+
###############################################################################
|
|
16
|
+
#
|
|
17
|
+
import sys
|
|
18
|
+
import re
|
|
19
|
+
import glob
|
|
20
|
+
from os import path as op
|
|
21
|
+
from rda_python_common import PgLOG
|
|
22
|
+
from rda_python_common import PgUtil
|
|
23
|
+
from rda_python_common import PgFile
|
|
24
|
+
from rda_python_common import PgDBI
|
|
25
|
+
from . import PgIPInfo
|
|
26
|
+
|
|
27
|
+
USAGE = {
|
|
28
|
+
'PGTBL' : "awsusage",
|
|
29
|
+
'AWSDIR' : PgLOG.PGLOG["TRANSFER"] + "/AWSera5log",
|
|
30
|
+
'AWSLOG' : "{}/{}-00-00-00-*",
|
|
31
|
+
'PFMT' : "YYYY/MM/DD"
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
DSIDS = {'nsf-ncar-era5' : 'd633000'}
|
|
35
|
+
|
|
36
|
+
#
|
|
37
|
+
# main function to run this program
|
|
38
|
+
#
|
|
39
|
+
def main():
|
|
40
|
+
|
|
41
|
+
params = [] # array of input values
|
|
42
|
+
argv = sys.argv[1:]
|
|
43
|
+
option = None
|
|
44
|
+
|
|
45
|
+
for arg in argv:
|
|
46
|
+
ms = re.match(r'^-(b|d|p|N)$', arg)
|
|
47
|
+
if ms:
|
|
48
|
+
opt = ms.group(1)
|
|
49
|
+
if opt == 'b':
|
|
50
|
+
PgLOG.PGLOG['BCKGRND'] = 1
|
|
51
|
+
elif option:
|
|
52
|
+
PgLOG.pglog("{}: Option -{} is present already".format(arg, option), PgLOG.LGWNEX)
|
|
53
|
+
else:
|
|
54
|
+
option = opt
|
|
55
|
+
elif re.match(r'^-', arg):
|
|
56
|
+
PgLOG.pglog(arg + ": Invalid Option", PgLOG.LGWNEX)
|
|
57
|
+
elif option:
|
|
58
|
+
params.append(arg)
|
|
59
|
+
else:
|
|
60
|
+
PgLOG.pglog(arg + ": Invalid Parameter", PgLOG.LGWNEX)
|
|
61
|
+
|
|
62
|
+
if not (option and params): PgLOG.show_usage('fillawsusage')
|
|
63
|
+
|
|
64
|
+
PgDBI.dssdb_dbname()
|
|
65
|
+
cmdstr = "fillawsusage {}".format(' '.join(argv))
|
|
66
|
+
PgLOG.cmdlog(cmdstr)
|
|
67
|
+
PgFile.change_local_directory(USAGE['AWSDIR'])
|
|
68
|
+
filenames = get_log_file_names(option, params)
|
|
69
|
+
if filenames:
|
|
70
|
+
fill_aws_usages(filenames)
|
|
71
|
+
else:
|
|
72
|
+
PgLOG.pglog("No log file found for given command: " + cmdstr, PgLOG.LOGWRN)
|
|
73
|
+
|
|
74
|
+
PgLOG.pglog(None, PgLOG.LOGWRN)
|
|
75
|
+
sys.exit(0)
|
|
76
|
+
|
|
77
|
+
#
|
|
78
|
+
# get the log file dates
|
|
79
|
+
#
|
|
80
|
+
def get_log_file_names(option, params):
|
|
81
|
+
|
|
82
|
+
filenames = {}
|
|
83
|
+
if option == 'd':
|
|
84
|
+
for dt in params:
|
|
85
|
+
pdate = PgUtil.format_date(dt)
|
|
86
|
+
pd = PgUtil.format_date(pdate, USAGE['PFMT'])
|
|
87
|
+
fname = USAGE['AWSLOG'].format(pd, pdate)
|
|
88
|
+
fnames = glob.glob(fname)
|
|
89
|
+
if fnames: filenames[pdate] = sorted(fnames)
|
|
90
|
+
else:
|
|
91
|
+
if option == 'N':
|
|
92
|
+
edate = PgUtil.curdate()
|
|
93
|
+
pdate = PgUtil.adddate(edate, 0, 0, -int(params[0]))
|
|
94
|
+
else:
|
|
95
|
+
pdate = PgUtil.format_date(params[0])
|
|
96
|
+
if len(params) > 1:
|
|
97
|
+
edate = PgUtil.adddate(PgUtil.format_date(params[1]), 0, 0, 1)
|
|
98
|
+
else:
|
|
99
|
+
edate = PgUtil.curdate()
|
|
100
|
+
while pdate < edate:
|
|
101
|
+
pd = PgUtil.format_date(pdate, USAGE['PFMT'])
|
|
102
|
+
fname = USAGE['AWSLOG'].format(pd, pdate)
|
|
103
|
+
fnames = glob.glob(fname)
|
|
104
|
+
if fnames: filenames[pdate] = sorted(fnames)
|
|
105
|
+
pdate = PgUtil.adddate(pdate, 0, 0, 1)
|
|
106
|
+
|
|
107
|
+
return filenames
|
|
108
|
+
|
|
109
|
+
#
|
|
110
|
+
# Fill AWS usages into table dssdb.awsusage of DSS PgSQL database from aws access logs
|
|
111
|
+
#
|
|
112
|
+
def fill_aws_usages(filenames):
|
|
113
|
+
|
|
114
|
+
year = cntall = addall = 0
|
|
115
|
+
for pdate in filenames:
|
|
116
|
+
fnames = filenames[pdate]
|
|
117
|
+
fcnt = len(fnames)
|
|
118
|
+
PgLOG.pglog("{}: Gathering AWS usage info from {} log files at {}".format(pdate, fcnt, PgLOG.current_datetime()), PgLOG.LOGWRN)
|
|
119
|
+
records = {}
|
|
120
|
+
cntadd = entcnt = 0
|
|
121
|
+
for logfile in fnames:
|
|
122
|
+
if not op.isfile(logfile):
|
|
123
|
+
PgLOG.pglog("{}: Not exists for Gathering AWS usage".format(logfile), PgLOG.LOGWRN)
|
|
124
|
+
continue
|
|
125
|
+
aws = PgFile.open_local_file(logfile)
|
|
126
|
+
if not aws: continue
|
|
127
|
+
while True:
|
|
128
|
+
line = aws.readline()
|
|
129
|
+
if not line: break
|
|
130
|
+
entcnt += 1
|
|
131
|
+
if entcnt%20000 == 0:
|
|
132
|
+
dcnt = len(records)
|
|
133
|
+
PgLOG.pglog("{}: {}/{} AWS log entries processed/records to add".format(pdate, entcnt, dcnt), PgLOG.WARNLG)
|
|
134
|
+
|
|
135
|
+
ms = re.match(r'^\w+ ([\w-]+) \[(\S+).*\] ([\d\.]+) .+ REST\.GET\.OBJECT \S+ "GET.+" \d+ - (\d+) \d+ .* ".+" "(.+)" ', line)
|
|
136
|
+
if not ms: continue
|
|
137
|
+
values = list(ms.groups())
|
|
138
|
+
if values[0] not in DSIDS: continue
|
|
139
|
+
dsid = DSIDS[values[0]]
|
|
140
|
+
size = int(values[3])
|
|
141
|
+
ip = values[2]
|
|
142
|
+
engine = values[4]
|
|
143
|
+
moff = engine.find('/')
|
|
144
|
+
if moff > 0:
|
|
145
|
+
if moff > 20: moff = 20
|
|
146
|
+
method = engine[0:moff].upper()
|
|
147
|
+
else:
|
|
148
|
+
method = "AWS"
|
|
149
|
+
key = "{}:{}:{}".format(ip, dsid, method)
|
|
150
|
+
if key in records:
|
|
151
|
+
records[key]['size'] += size
|
|
152
|
+
records[key]['fcount'] += 1
|
|
153
|
+
else:
|
|
154
|
+
(year, quarter, date, time) = get_record_date_time(values[1])
|
|
155
|
+
iprec = PgIPInfo.get_missing_ipinfo(ip)
|
|
156
|
+
if not iprec: continue
|
|
157
|
+
records[key] = {'ip' : ip, 'dsid' : dsid, 'date' : date, 'time' : time, 'quarter' : quarter,
|
|
158
|
+
'size' : size, 'fcount' : 1, 'method' : method, 'engine' : engine,
|
|
159
|
+
'org_type' : iprec['org_type'], 'country' : iprec['country'],
|
|
160
|
+
'region' : iprec['region'], 'email' : iprec['email']}
|
|
161
|
+
aws.close()
|
|
162
|
+
if records: cntadd = add_usage_records(records, year)
|
|
163
|
+
PgLOG.pglog("{}: {} AWS usage records added for {} entries at {}".format(pdate, cntadd, entcnt, PgLOG.current_datetime()), PgLOG.LOGWRN)
|
|
164
|
+
cntall += entcnt
|
|
165
|
+
if cntadd:
|
|
166
|
+
addall += cntadd
|
|
167
|
+
if addall > cntadd:
|
|
168
|
+
PgLOG.pglog("{} AWS usage records added for {} entries at {}".format(addall, cntall, PgLOG.current_datetime()), PgLOG.LOGWRN)
|
|
169
|
+
|
|
170
|
+
def get_record_date_time(ctime):
|
|
171
|
+
|
|
172
|
+
ms = re.search(r'^(\d+)/(\w+)/(\d+):(\d+:\d+:\d+)$', ctime)
|
|
173
|
+
if ms:
|
|
174
|
+
d = int(ms.group(1))
|
|
175
|
+
m = PgUtil.get_month(ms.group(2))
|
|
176
|
+
y = ms.group(3)
|
|
177
|
+
t = ms.group(4)
|
|
178
|
+
q = 1 + int((m-1)/3)
|
|
179
|
+
return (y, q, "{}-{:02}-{:02}".format(y, m, d), t)
|
|
180
|
+
else:
|
|
181
|
+
PgLOG.pglog(ctime + ": Invalid date/time format", PgLOG.LGEREX)
|
|
182
|
+
|
|
183
|
+
def add_usage_records(records, year):
|
|
184
|
+
|
|
185
|
+
cnt = 0
|
|
186
|
+
for key in records:
|
|
187
|
+
record = records[key]
|
|
188
|
+
cond = "date = '{}' AND time = '{}' AND ip = '{}'".format(record['date'], record['time'], record['ip'])
|
|
189
|
+
if PgDBI.pgget(USAGE['PGTBL'], '', cond, PgLOG.LGEREX): continue
|
|
190
|
+
if add_to_allusage(year, record):
|
|
191
|
+
cnt += PgDBI.pgadd(USAGE['PGTBL'], record, PgLOG.LOGWRN)
|
|
192
|
+
|
|
193
|
+
return cnt
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def add_to_allusage(year, pgrec):
|
|
197
|
+
|
|
198
|
+
record = {'source' : 'A'}
|
|
199
|
+
flds = ['ip', 'dsid', 'date', 'time', 'quarter', 'size', 'method',
|
|
200
|
+
'org_type', 'country', 'region', 'email']
|
|
201
|
+
|
|
202
|
+
for fld in flds:
|
|
203
|
+
record[fld] = pgrec[fld]
|
|
204
|
+
|
|
205
|
+
return PgDBI.add_yearly_allusage(year, record)
|
|
206
|
+
|
|
207
|
+
#
|
|
208
|
+
# call main() to start program
|
|
209
|
+
#
|
|
210
|
+
if __name__ == "__main__": main()
|
{rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/fillawsusage.usg
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
|
|
2
2
|
Retrieves usage information from AWS Server logs under directory
|
|
3
|
-
../rda/transer/AWSera5log/ to fill table '
|
|
3
|
+
../rda/transer/AWSera5log/ to fill table 'awsusage' in database 'rdadb'.
|
|
4
4
|
|
|
5
5
|
Usage: fillawsusage [-b] [-d LogFileDates] [-N NumberDay] [-p BeginDate [Enddate]]
|
|
6
6
|
|
{rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/fillcdgusage.py
RENAMED
|
@@ -31,29 +31,30 @@ USAGE = {
|
|
|
31
31
|
}
|
|
32
32
|
|
|
33
33
|
DSIDS = {
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
34
|
+
'pi_cesm2_atm_river_analysis' : ['d010073'],
|
|
35
|
+
'na-cordex' : ['d316009'],
|
|
36
|
+
'ucar.cgd.cesm2.cam6.prescribed_sst_amip' : ['d651010'],
|
|
37
|
+
'ucar.cgd.ccsm4.CLM_LAND_ONLY' : ['d651011'],
|
|
38
|
+
'ucar.cgd.artmip' : ['d651012', 'd651016', 'd651017', 'd651018'],
|
|
39
|
+
'tamip' : ['d651013'],
|
|
40
|
+
'ucar.cgd.ccsm4.CLIVAR_LE' : ['d651014'],
|
|
41
|
+
'ucar.cgd.cesm2.Gettelman_CESM2_ECS' : ['d651015'],
|
|
42
|
+
'ucar.cgd.ccsm4.geomip.ssp5' : ['d651024'],
|
|
43
|
+
'ucar.cgd.ccsm4.IOD-PACEMAKER' : ['d651021'],
|
|
44
|
+
'ucar.cgd.ccsm4.past2k_transient' : ['651023'],
|
|
45
|
+
'ucar.cgd.ccsm4.lowwarming' : ['d651025'],
|
|
46
|
+
'ucar.cgd.ccsm4.CESM_CAM5_BGC_ME' : ['d651000'],
|
|
47
|
+
'ucar.cgd.ccsm4.iTRACE' : ['d651022'],
|
|
48
|
+
'ucar.cgd.ccsm4.so2_geoeng' : ['d651026'],
|
|
49
|
+
'ucar.cgd.ccsm4.cesmLE' : ['d651027'],
|
|
50
|
+
'ucar.cgd.ccsm4.CESM1-CAM5-DP' : ['d651028'],
|
|
51
|
+
'ucar.cgd.ccsm4.amv_lens' : ['d651031'],
|
|
52
|
+
'ucar.cgd.ccsm4.ATL-PACEMAKER' : ['d651032'],
|
|
53
|
+
'ucar.cgd.ccsm4.pac-pacemaker' : ['d651033'],
|
|
54
|
+
'ucar.cgd.ccsm4.SD-WACCM-X_v2.1' : ['d651034'],
|
|
55
|
+
'ucar.cgd.ccsm4.amv_lens' : ['d651035'],
|
|
56
|
+
'ucar.cgd.cesm2.cism_ismip6' : ['d651036'],
|
|
57
|
+
'ucar.cgd.ccsm4.pliomip2' : ['d651037'],
|
|
57
58
|
'ucar.cgd.cesm2-waccm.s2s_hindcasts': ['d651040'],
|
|
58
59
|
'ucar.cgd.CESM1.3_SH_storm_tracks': ['d651044'],
|
|
59
60
|
'ucar.cgd.cesm2.waccm6.ssp245': ['d651045'],
|
|
@@ -67,13 +68,12 @@ DSIDS = {
|
|
|
67
68
|
'ucar.cgd.ccsm4.PaleoIF' : ['d651052'],
|
|
68
69
|
'ucar.cgd.ccsm4.b.e11.B20LE_fixedO3' : ['d651053'],
|
|
69
70
|
'ucar.cgd.cesm2.single.forcing.large.ensemble' : ['d651055'],
|
|
70
|
-
'ucar.cgd.cesm2le.output': ['d651056'],
|
|
71
|
+
'ucar.cgd.cesm2le.output': ['d651056'],
|
|
71
72
|
'ucar.cgd.ccsm4.ARISE-SAI-1.5' : ['d651059'],
|
|
72
73
|
'ucar.cgd.cesm2.s2s_hindcasts': ['d651060'],
|
|
73
74
|
'ucar.cgd.cesm2.s2s_hindcasts.mjo': ['d651061'],
|
|
74
75
|
'ucar.cgd.cesm2.s2s_hindcasts.tc_tracks': ['d651062'],
|
|
75
76
|
'ucar.cgd.cesm2.s2s_hindcasts.cesm2.climo': ['d651063'],
|
|
76
|
-
#add 2025-08-06
|
|
77
77
|
'ucar.cgd.ccsm4.cesmLME' : ['d651058'],
|
|
78
78
|
'ucar.cgd.ccsm4.GLENS' : ['d651064'],
|
|
79
79
|
'ucar.cgd.ccsm4.CESM2-CISM2-LIGtransient' : ['d651066'],
|
|
@@ -85,8 +85,13 @@ DSIDS = {
|
|
|
85
85
|
'ucar.cgd.cesm1.cldmod': ['d651069'],
|
|
86
86
|
'ucar.cgd.cesm2.marine.biogeochemistry': ['d651071'],
|
|
87
87
|
'ucar.cgd.nw2.mom6': ['d651072'],
|
|
88
|
-
|
|
89
|
-
|
|
88
|
+
'ucar.cgd.cesm2.cam6.ppe': ['d651076'],
|
|
89
|
+
'ucar.cgd.cesm2.smyle': ['d651065'],
|
|
90
|
+
# new added
|
|
91
|
+
'gridded_precip_and_temp' : ['d010078'],
|
|
92
|
+
'29_newman' : ['d010079'],
|
|
93
|
+
'waccm-x.ion.asymmetry' : ['d010081'],
|
|
94
|
+
'NARCCAP' : ['d316015']
|
|
90
95
|
}
|
|
91
96
|
|
|
92
97
|
ALLIDS = list(DSIDS.keys())
|
{rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/fillgdexusage.py
RENAMED
|
@@ -30,44 +30,58 @@ USAGE = {
|
|
|
30
30
|
}
|
|
31
31
|
|
|
32
32
|
DSIDS = {
|
|
33
|
+
"14_schuster" : "d583138",
|
|
34
|
+
"384_duvivier" : "d583139",
|
|
35
|
+
"model_data_for_17_march_2015_storm_event" : "d583140",
|
|
36
|
+
"326_qingyu" : "d583141",
|
|
37
|
+
"354_fasullo" : "d583142",
|
|
38
|
+
"157_gaubert" : "d583143",
|
|
39
|
+
"370_gaubert" : "d583144",
|
|
40
|
+
"459" : "d583145",
|
|
41
|
+
"108_kristenk" : "d583146",
|
|
42
|
+
"82_abaker" : "d583147",
|
|
43
|
+
"215_grabow" : "d583148",
|
|
44
|
+
"222_duvivier" : "d583149",
|
|
45
|
+
"366_rberrios" : "d583150",
|
|
46
|
+
"210_dxyang" : "d583100",
|
|
47
|
+
"238_junkyung" : "d583101",
|
|
48
|
+
"361_islas" : "d583102",
|
|
49
|
+
"448" : "d583103",
|
|
50
|
+
"187_jaredlee" : "d583104",
|
|
51
|
+
"273_kdagon" : "d583105",
|
|
52
|
+
"149_liuh" : "d583106",
|
|
53
|
+
"279_schwantes" : "d583107",
|
|
54
|
+
"60_pinto" : "d583108",
|
|
55
|
+
"WACC_Dependence_on_Solar_Activity" : "d583109",
|
|
56
|
+
"275_domingom" : "d583110",
|
|
57
|
+
"445" : "d583111",
|
|
58
|
+
"TIE-GCM_2017_Solar_Storm" : "d583112",
|
|
59
|
+
"123_andrew" : "d583113",
|
|
60
|
+
"453" : "d583114",
|
|
61
|
+
"367_mclong" : "d583115",
|
|
62
|
+
"258_jiangzhu" : "d583116",
|
|
63
|
+
"136_hannay" : "d583117",
|
|
64
|
+
"173B_dcherian" : "d583118",
|
|
65
|
+
"407_islas" : "d583119",
|
|
66
|
+
"189_islas" : "d583120",
|
|
67
|
+
"289_shields" : "d583121",
|
|
68
|
+
"415_mcinerney" : "d583122",
|
|
69
|
+
"151_dll" : "d583123",
|
|
70
|
+
"460" : "d583124",
|
|
71
|
+
"117_phamkh" : "d583125",
|
|
72
|
+
"109_erichend" : "d583126",
|
|
73
|
+
"412" : "d583127",
|
|
74
|
+
"390_knocasio" : "d583128",
|
|
75
|
+
"382_kumar" : "d583129",
|
|
76
|
+
"470" : "d583130",
|
|
77
|
+
"371_abaker" : "d583131",
|
|
78
|
+
"CESM-cocco_CO2_experiments" : "d583132",
|
|
79
|
+
"fuel_moisture_content" : "d583133",
|
|
80
|
+
"81_rberrios" : "d583134",
|
|
81
|
+
"286_knocasio" : "d583135",
|
|
82
|
+
"383_stebbins" : "d583136",
|
|
83
|
+
"256_tilmes" : "d583137",
|
|
33
84
|
'371_abaker' : 'd583131',
|
|
34
|
-
'CESM-cocco_CO2_experiments' : 'd583132',
|
|
35
|
-
'256_tilmes' : 'd583137',
|
|
36
|
-
'fuel_moisture_content' : 'd583133',
|
|
37
|
-
'81_rberrios' : 'd583134',
|
|
38
|
-
'210_dxyang' : 'd583100',
|
|
39
|
-
'286_knocasio' : 'd583135',
|
|
40
|
-
'238_junkyung' : 'd583101',
|
|
41
|
-
'361_islas' : 'd583102',
|
|
42
|
-
'448' : 'd583103',
|
|
43
|
-
'383_stebbins' : 'd583136',
|
|
44
|
-
'187_jaredlee' : 'd583104',
|
|
45
|
-
'273_kdagon' : 'd583105',
|
|
46
|
-
'149_liuh' : 'd583106',
|
|
47
|
-
'279_schwantes' : 'd583107',
|
|
48
|
-
'60_pinto' : 'd583108',
|
|
49
|
-
'WACC_Dependence_on_Solar_Activity' : 'd583109',
|
|
50
|
-
'275_domingom' : 'd583110',
|
|
51
|
-
'445' : 'd583111',
|
|
52
|
-
'TIE-GCM_2017_Solar_Storm' : 'd583112',
|
|
53
|
-
'123_andrew' : 'd583113',
|
|
54
|
-
'453' : 'd583114',
|
|
55
|
-
'367_mclong' : 'd583115',
|
|
56
|
-
'258_jiangzhu' : 'd583116',
|
|
57
|
-
'136_hannay' : 'd583117',
|
|
58
|
-
'173B_dcherian' : 'd583118',
|
|
59
|
-
'407_islas' : 'd583119',
|
|
60
|
-
'189_islas' : 'd583120',
|
|
61
|
-
'289_shields' : 'd583121',
|
|
62
|
-
'415_mcinerney' : 'd583122',
|
|
63
|
-
'151_dll' : 'd583123',
|
|
64
|
-
'460' : 'd583124',
|
|
65
|
-
'117_phamkh' : 'd583125',
|
|
66
|
-
'109_erichend' : 'd583126',
|
|
67
|
-
'412' : 'd583127',
|
|
68
|
-
'390_knocasio' : 'd583128',
|
|
69
|
-
'382_kumar' : 'd583129',
|
|
70
|
-
'470' : 'd583130',
|
|
71
85
|
# icarus
|
|
72
86
|
'icarus.experiment.403' : 'd789010',
|
|
73
87
|
'icarus.experiment.404' : 'd789011',
|
{rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/fillglobususage.py
RENAMED
|
@@ -27,7 +27,7 @@ from . import PgIPInfo
|
|
|
27
27
|
|
|
28
28
|
USAGE = {
|
|
29
29
|
'PGTBL' : "wusage",
|
|
30
|
-
'GBSDIR' : PgLOG.PGLOG["
|
|
30
|
+
'GBSDIR' : PgLOG.PGLOG["GDEXWORK"] + "/logs/gridftp/",
|
|
31
31
|
'GBSLOG' : "access_log_gridftp0{}_{}",
|
|
32
32
|
}
|
|
33
33
|
|
|
@@ -43,7 +43,7 @@ def main():
|
|
|
43
43
|
|
|
44
44
|
|
|
45
45
|
for arg in argv:
|
|
46
|
-
ms = re.match(r'^-(b|d|p|N)$', arg)
|
|
46
|
+
ms = re.match(r'^-(b|d|f|p|N)$', arg)
|
|
47
47
|
if ms:
|
|
48
48
|
opt = ms.group(1)
|
|
49
49
|
if opt == 'b':
|
|
@@ -80,7 +80,9 @@ def main():
|
|
|
80
80
|
def get_log_file_names(option, params, datelimits):
|
|
81
81
|
|
|
82
82
|
filenames = []
|
|
83
|
-
if option == '
|
|
83
|
+
if option == 'f':
|
|
84
|
+
filenames = params
|
|
85
|
+
elif option == 'd':
|
|
84
86
|
for pdate in params:
|
|
85
87
|
fdate = PgUtil.format_date(pdate, 'MMDDYYYY')
|
|
86
88
|
fname = USAGE['GBSLOG'].format('?', fdate)
|
|
@@ -147,18 +149,14 @@ def fill_globus_usages(fnames, datelimits):
|
|
|
147
149
|
locflag = 'O' if re.match(r'^https://stratus\.', sline) else 'G'
|
|
148
150
|
idx = wfile.find('?')
|
|
149
151
|
if idx > -1: wfile = wfile[:idx]
|
|
150
|
-
|
|
151
|
-
if
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
method = "WGET"
|
|
155
|
-
elif re.match(r'^python', engine, re.I):
|
|
156
|
-
method = "PYTHN"
|
|
152
|
+
moff = engine.find('/')
|
|
153
|
+
if moff > 0:
|
|
154
|
+
if moff > 20: moff = 20
|
|
155
|
+
method = engine[0:moff].upper()
|
|
157
156
|
else:
|
|
158
157
|
method = "WEB"
|
|
159
158
|
|
|
160
159
|
key = "{}:{}:{}".format(ip, dsid, wfile) if stat == '206' else None
|
|
161
|
-
|
|
162
160
|
if record:
|
|
163
161
|
if key == pkey:
|
|
164
162
|
record['size'] += size
|
{rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/fillglobususage.usg
RENAMED
|
@@ -3,14 +3,16 @@
|
|
|
3
3
|
/gpfs/fs1/collections/rda/work/logs/gridftp/ to fill table 'wusage' in
|
|
4
4
|
database 'rdadb'.
|
|
5
5
|
|
|
6
|
-
Usage: fillglobususage [-b] [-d LogFileDates] [-N NumberDay] [-p BeginDate [Enddate]]
|
|
6
|
+
Usage: fillglobususage [-b] [-d LogFileDates] [-f LogFileNames] [-N NumberDay] [-p BeginDate [Enddate]]
|
|
7
7
|
|
|
8
|
-
select option, -d, -N or -p to run this application.
|
|
8
|
+
select option, -d, -f, -N or -p to run this application.
|
|
9
9
|
|
|
10
10
|
- Option -b, log process information into logfile only;
|
|
11
11
|
|
|
12
12
|
- Option -d, retrieve usage info from given log file dates;
|
|
13
13
|
|
|
14
|
+
- Option -f, retrieve usage info from given log file names;
|
|
15
|
+
|
|
14
16
|
- Option -N, retrieve usage info in recent NumberDay days;
|
|
15
17
|
|
|
16
18
|
- Option -p, retrieve usage info between given period. For missing EndDate,
|