rda-python-metrics 1.0.33__py3-none-any.whl → 1.0.35__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rda_python_metrics/fillawsusage.py +79 -124
- rda_python_metrics/fillawsusage.usg +1 -1
- rda_python_metrics/fillosdfusage.py +54 -87
- rda_python_metrics/fillosdfusage.usg +1 -1
- {rda_python_metrics-1.0.33.dist-info → rda_python_metrics-1.0.35.dist-info}/METADATA +1 -1
- {rda_python_metrics-1.0.33.dist-info → rda_python_metrics-1.0.35.dist-info}/RECORD +10 -10
- {rda_python_metrics-1.0.33.dist-info → rda_python_metrics-1.0.35.dist-info}/WHEEL +0 -0
- {rda_python_metrics-1.0.33.dist-info → rda_python_metrics-1.0.35.dist-info}/entry_points.txt +0 -0
- {rda_python_metrics-1.0.33.dist-info → rda_python_metrics-1.0.35.dist-info}/licenses/LICENSE +0 -0
- {rda_python_metrics-1.0.33.dist-info → rda_python_metrics-1.0.35.dist-info}/top_level.txt +0 -0
|
@@ -25,13 +25,13 @@ from rda_python_common import PgDBI
|
|
|
25
25
|
from . import PgIPInfo
|
|
26
26
|
|
|
27
27
|
USAGE = {
|
|
28
|
-
'PGTBL' : "
|
|
28
|
+
'PGTBL' : "awsusage",
|
|
29
29
|
'AWSDIR' : PgLOG.PGLOG["TRANSFER"] + "/AWSera5log",
|
|
30
30
|
'AWSLOG' : "{}/{}-00-00-00-*",
|
|
31
31
|
'PFMT' : "YYYY/MM/DD"
|
|
32
32
|
}
|
|
33
33
|
|
|
34
|
-
DSIDS = {'nsf-ncar-era5' :
|
|
34
|
+
DSIDS = {'nsf-ncar-era5' : 'd633000'}
|
|
35
35
|
|
|
36
36
|
#
|
|
37
37
|
# main function to run this program
|
|
@@ -79,14 +79,14 @@ def main():
|
|
|
79
79
|
#
|
|
80
80
|
def get_log_file_names(option, params):
|
|
81
81
|
|
|
82
|
-
filenames =
|
|
82
|
+
filenames = {}
|
|
83
83
|
if option == 'd':
|
|
84
84
|
for dt in params:
|
|
85
85
|
pdate = PgUtil.format_date(dt)
|
|
86
86
|
pd = PgUtil.format_date(pdate, USAGE['PFMT'])
|
|
87
87
|
fname = USAGE['AWSLOG'].format(pd, pdate)
|
|
88
88
|
fnames = glob.glob(fname)
|
|
89
|
-
if fnames: filenames
|
|
89
|
+
if fnames: filenames[pdate] = sorted(fnames)
|
|
90
90
|
else:
|
|
91
91
|
if option == 'N':
|
|
92
92
|
edate = PgUtil.curdate()
|
|
@@ -97,11 +97,11 @@ def get_log_file_names(option, params):
|
|
|
97
97
|
edate = PgUtil.format_date(params[1])
|
|
98
98
|
else:
|
|
99
99
|
edate = PgUtil.curdate()
|
|
100
|
-
while pdate
|
|
100
|
+
while pdate < edate:
|
|
101
101
|
pd = PgUtil.format_date(pdate, USAGE['PFMT'])
|
|
102
102
|
fname = USAGE['AWSLOG'].format(pd, pdate)
|
|
103
103
|
fnames = glob.glob(fname)
|
|
104
|
-
if fnames: filenames
|
|
104
|
+
if fnames: filenames[pdate] = sorted(fnames)
|
|
105
105
|
pdate = PgUtil.adddate(pdate, 0, 0, 1)
|
|
106
106
|
|
|
107
107
|
return filenames
|
|
@@ -109,72 +109,62 @@ def get_log_file_names(option, params):
|
|
|
109
109
|
#
|
|
110
110
|
# Fill AWS usages into table dssdb.awsusage of DSS PgSQL database from aws access logs
|
|
111
111
|
#
|
|
112
|
-
def fill_aws_usages(
|
|
113
|
-
|
|
114
|
-
cntall = addall = 0
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
PgLOG.pglog("{}: Not exists for Gathering AWS usage".format(logfile), PgLOG.LOGWRN)
|
|
119
|
-
continue
|
|
120
|
-
PgLOG.pglog("Gathering usage info from {} at {}".format(logfile, PgLOG.current_datetime()), PgLOG.LOGWRN)
|
|
121
|
-
aws = PgFile.open_local_file(logfile)
|
|
122
|
-
if not aws: continue
|
|
123
|
-
ptime = ''
|
|
124
|
-
record = {}
|
|
112
|
+
def fill_aws_usages(filenames):
|
|
113
|
+
|
|
114
|
+
year = cntall = addall = 0
|
|
115
|
+
for pdate in filenames:
|
|
116
|
+
fnames = filenames[pdate]
|
|
117
|
+
records = {}
|
|
125
118
|
cntadd = entcnt = 0
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
cntadd += add_file_usage(year, record)
|
|
171
|
-
record = None
|
|
172
|
-
if record: cntadd += add_file_usage(year, record)
|
|
173
|
-
aws.close()
|
|
119
|
+
for logfile in fnames:
|
|
120
|
+
if not op.isfile(logfile):
|
|
121
|
+
PgLOG.pglog("{}: Not exists for Gathering AWS usage".format(logfile), PgLOG.LOGWRN)
|
|
122
|
+
continue
|
|
123
|
+
PgLOG.pglog("Gathering AWS usage info from {} at {}".format(logfile, PgLOG.current_datetime()), PgLOG.LOGWRN)
|
|
124
|
+
aws = PgFile.open_local_file(logfile)
|
|
125
|
+
if not aws: continue
|
|
126
|
+
while True:
|
|
127
|
+
line = aws.readline()
|
|
128
|
+
if not line: break
|
|
129
|
+
entcnt += 1
|
|
130
|
+
if entcnt%20000 == 0:
|
|
131
|
+
dcnt = len(records)
|
|
132
|
+
PgLOG.pglog("{}: {}/{} AWS log entries processed/records to add".format(pdate, entcnt, dcnt), PgLOG.WARNLG)
|
|
133
|
+
|
|
134
|
+
ms = re.match(r'^\w+ ([\w-]+) \[(\S+).*\] ([\d\.]+) .+ REST\.GET\.OBJECT \S+ "GET.+" \d+ - (\d+) \d+ .* ".+" "(.+)" ', line)
|
|
135
|
+
if not ms: continue
|
|
136
|
+
values = list(ms.groups())
|
|
137
|
+
if values[0] not in DSIDS: continue
|
|
138
|
+
dsid = DSIDS[values[0]]
|
|
139
|
+
size = int(values[3])
|
|
140
|
+
ip = values[2]
|
|
141
|
+
engine = values[4]
|
|
142
|
+
moff = engine.find('/')
|
|
143
|
+
if moff > 0:
|
|
144
|
+
if moff > 20: moff = 20
|
|
145
|
+
method = engine[0:moff].upper()
|
|
146
|
+
else:
|
|
147
|
+
method = "AWS"
|
|
148
|
+
key = "{}:{}:{}".format(ip, dsid, method)
|
|
149
|
+
if key in records:
|
|
150
|
+
records[key]['size'] += size
|
|
151
|
+
records[key]['fcount'] += 1
|
|
152
|
+
else:
|
|
153
|
+
(year, quarter, date, time) = get_record_date_time(values[1])
|
|
154
|
+
iprec = PgIPInfo.get_missing_ipinfo(ip)
|
|
155
|
+
if not iprec: continue
|
|
156
|
+
records[key] = {'ip' : ip, 'dsid' : dsid, 'date' : date, 'time' : time, 'quarter' : quarter,
|
|
157
|
+
'size' : size, 'fcount' : 1, 'method' : method, 'engine' : engine,
|
|
158
|
+
'org_type' : iprec['org_type'], 'country' : iprec['country'],
|
|
159
|
+
'region' : iprec['region'], 'email' : iprec['email']}
|
|
160
|
+
aws.close()
|
|
161
|
+
if records: cntadd = add_usage_records(records, year)
|
|
162
|
+
PgLOG.pglog("{}: {} AWS usage records added for {} entries at {}".format(pdate, cntadd, entcnt, PgLOG.current_datetime()), PgLOG.LOGWRN)
|
|
174
163
|
cntall += entcnt
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
164
|
+
if cntadd:
|
|
165
|
+
addall += cntadd
|
|
166
|
+
if addall > cntadd:
|
|
167
|
+
PgLOG.pglog("{} AWS usage records added for {} entries at {}".format(addall, cntall, PgLOG.current_datetime()), PgLOG.LOGWRN)
|
|
178
168
|
|
|
179
169
|
def get_record_date_time(ctime):
|
|
180
170
|
|
|
@@ -189,64 +179,29 @@ def get_record_date_time(ctime):
|
|
|
189
179
|
else:
|
|
190
180
|
PgLOG.pglog(ctime + ": Invalid date/time format", PgLOG.LGEREX)
|
|
191
181
|
|
|
192
|
-
|
|
193
|
-
# Fill usage of a single online data file into table dssdb.wusage of DSS PgSQL database
|
|
194
|
-
#
|
|
195
|
-
def add_file_usage(year, logrec):
|
|
196
|
-
|
|
197
|
-
pgrec = get_wfile_wid(logrec['dsid'], logrec['wfile'])
|
|
198
|
-
if not pgrec: return 0
|
|
199
|
-
|
|
200
|
-
table = "{}_{}".format(USAGE['PGTBL'], year)
|
|
201
|
-
cond = "wid = {} AND method = '{}' AND date_read = '{}' AND time_read = '{}'".format(pgrec['wid'], logrec['method'], logrec['date'], logrec['time'])
|
|
202
|
-
if PgDBI.pgget(table, "", cond, PgLOG.LOGWRN): return 0
|
|
203
|
-
|
|
204
|
-
wurec = PgIPInfo.get_wuser_record(logrec['ip'], logrec['date'])
|
|
205
|
-
if not wurec: return 0
|
|
206
|
-
record = {'wid' : pgrec['wid'], 'dsid' : pgrec['dsid']}
|
|
207
|
-
record['wuid_read'] = wurec['wuid']
|
|
208
|
-
record['date_read'] = logrec['date']
|
|
209
|
-
record['time_read'] = logrec['time']
|
|
210
|
-
record['size_read'] = logrec['size']
|
|
211
|
-
record['method'] = logrec['method']
|
|
212
|
-
record['locflag'] = logrec['locflag']
|
|
213
|
-
record['ip'] = logrec['ip']
|
|
214
|
-
record['quarter'] = logrec['quarter']
|
|
215
|
-
|
|
216
|
-
if add_to_allusage(year, logrec, wurec):
|
|
217
|
-
return PgDBI.add_yearly_wusage(year, record)
|
|
218
|
-
else:
|
|
219
|
-
return 0
|
|
220
|
-
|
|
221
|
-
def add_to_allusage(year, logrec, wurec):
|
|
222
|
-
|
|
223
|
-
pgrec = {'email' : wurec['email'], 'org_type' : wurec['org_type'],
|
|
224
|
-
'country' : wurec['country'], 'region' : wurec['region']}
|
|
225
|
-
pgrec['dsid'] = logrec['dsid']
|
|
226
|
-
pgrec['date'] = logrec['date']
|
|
227
|
-
pgrec['quarter'] = logrec['quarter']
|
|
228
|
-
pgrec['time'] = logrec['time']
|
|
229
|
-
pgrec['size'] = logrec['size']
|
|
230
|
-
pgrec['method'] = logrec['method']
|
|
231
|
-
pgrec['ip'] = logrec['ip']
|
|
232
|
-
pgrec['source'] = 'A'
|
|
233
|
-
return PgDBI.add_yearly_allusage(year, pgrec)
|
|
182
|
+
def add_usage_records(records, year):
|
|
234
183
|
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
184
|
+
cnt = 0
|
|
185
|
+
for key in records:
|
|
186
|
+
record = records[key]
|
|
187
|
+
cond = "date = '{}' AND time = '{}' AND ip = '{}'".format(record['date'], record['time'], record['ip'])
|
|
188
|
+
if PgDBI.pgget(USAGE['PGTBL'], '', cond, PgLOG.LGEREX): continue
|
|
189
|
+
if add_to_allusage(year, record):
|
|
190
|
+
cnt += PgDBI.pgadd(USAGE['PGTBL'], record, PgLOG.LOGWRN)
|
|
191
|
+
|
|
192
|
+
return cnt
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def add_to_allusage(year, pgrec):
|
|
239
196
|
|
|
240
|
-
|
|
241
|
-
|
|
197
|
+
record = {'source' : 'A'}
|
|
198
|
+
flds = ['ip', 'dsid', 'date', 'time', 'quarter', 'size', 'method',
|
|
199
|
+
'org_type', 'country', 'region', 'email']
|
|
242
200
|
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
if pgrec:
|
|
246
|
-
pgrec = PgDBI.pgget("wfile", "*", "wid = {}".format(pgrec['wid']))
|
|
247
|
-
if pgrec: pgrec['dsid'] = dsid
|
|
201
|
+
for fld in flds:
|
|
202
|
+
record[fld] = pgrec[fld]
|
|
248
203
|
|
|
249
|
-
return
|
|
204
|
+
return PgDBI.add_yearly_allusage(year, record)
|
|
250
205
|
|
|
251
206
|
#
|
|
252
207
|
# call main() to start program
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
|
|
2
2
|
Retrieves usage information from AWS Server logs under directory
|
|
3
|
-
../rda/transer/AWSera5log/ to fill table '
|
|
3
|
+
../rda/transer/AWSera5log/ to fill table 'awsusage' in database 'rdadb'.
|
|
4
4
|
|
|
5
5
|
Usage: fillawsusage [-b] [-d LogFileDates] [-N NumberDay] [-p BeginDate [Enddate]]
|
|
6
6
|
|
|
@@ -22,7 +22,7 @@ from rda_python_common import PgSplit
|
|
|
22
22
|
from . import PgIPInfo
|
|
23
23
|
|
|
24
24
|
USAGE = {
|
|
25
|
-
'OSDFTBL' : "
|
|
25
|
+
'OSDFTBL' : "osdfusage",
|
|
26
26
|
'OSDFDIR' : PgLOG.PGLOG["DSSDATA"] + "/work/zji/osdflogs/",
|
|
27
27
|
'OSDFGET' : 'wget -m -nH -np -nd https://pelicanplatform.org/pelican-access-logs/ncar-access-log/',
|
|
28
28
|
'OSDFLOG' : "{}-cache.log", # YYYY-MM-DD-cache.log
|
|
@@ -100,9 +100,7 @@ def get_log_file_names(option, params, datelimits):
|
|
|
100
100
|
#
|
|
101
101
|
def fill_osdf_usages(fnames):
|
|
102
102
|
|
|
103
|
-
cntall = addall = 0
|
|
104
|
-
|
|
105
|
-
fcnt = len(fnames)
|
|
103
|
+
year = cntall = addall = 0
|
|
106
104
|
for logfile in fnames:
|
|
107
105
|
linfo = PgFile.check_local_file(logfile)
|
|
108
106
|
if not linfo:
|
|
@@ -119,46 +117,54 @@ def fill_osdf_usages(fnames):
|
|
|
119
117
|
PgLOG.pglog("{}: Gathering OSDF usage at {}".format(logfile, PgLOG.current_datetime()), PgLOG.LOGWRN)
|
|
120
118
|
osdf = PgFile.open_local_file(logfile)
|
|
121
119
|
if not osdf: continue
|
|
120
|
+
records = {}
|
|
122
121
|
cntadd = entcnt = 0
|
|
123
|
-
pkey = None
|
|
124
122
|
while True:
|
|
125
123
|
line = osdf.readline()
|
|
126
124
|
if not line: break
|
|
127
125
|
entcnt += 1
|
|
128
|
-
if entcnt%
|
|
129
|
-
|
|
126
|
+
if entcnt%20000 == 0:
|
|
127
|
+
dcnt = len(records)
|
|
128
|
+
PgLOG.pglog("{}: {}/{} OSDF log entries processed/records added".format(logfile, entcnt, dcnt), PgLOG.WARNLG)
|
|
130
129
|
|
|
131
|
-
ms = re.match(r'^\[(\S+)\] \[Objectname:\/ncar\/rda\/([a-z]\d{6})
|
|
130
|
+
ms = re.match(r'^\[(\S+)\] \[Objectname:\/ncar\/rda\/([a-z]\d{6})\/\S+\].* \[Site:(\S+)\].* \[Host:(\S+)\].* \[AppInfo:(\S+)\].* \[Read:(\d+)\]', line)
|
|
132
131
|
if not ms: continue
|
|
133
132
|
dt = ms.group(1)
|
|
134
133
|
dsid = ms.group(2)
|
|
135
|
-
|
|
134
|
+
site = ms.group(3)
|
|
136
135
|
ip = ms.group(4)
|
|
137
136
|
if ip == 'N/A': ip = '0.0.0.0'
|
|
138
137
|
engine = ms.group(5)
|
|
139
138
|
size = int(ms.group(6))
|
|
140
|
-
(
|
|
141
|
-
|
|
142
|
-
if re.match(r'^curl', engine, re.I):
|
|
143
|
-
method = "CURL"
|
|
144
|
-
elif re.match(r'^wget', engine, re.I):
|
|
145
|
-
method = "WGET"
|
|
146
|
-
elif re.match(r'^python', engine, re.I):
|
|
147
|
-
method = "PYTHN"
|
|
148
|
-
elif re.match(r'^N/A', engine, re.I):
|
|
149
|
-
method = "N/A"
|
|
139
|
+
if re.match(r'^N/A', engine, re.I):
|
|
140
|
+
method = "OSDF"
|
|
150
141
|
else:
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
142
|
+
moff = engine.find('/')
|
|
143
|
+
if moff > 0:
|
|
144
|
+
if moff > 20: moff = 20
|
|
145
|
+
method = engine[0:moff].upper()
|
|
146
|
+
else:
|
|
147
|
+
method = "OSDF"
|
|
148
|
+
key = "{}:{}:{}".format(ip, dsid, method)
|
|
149
|
+
if key in records:
|
|
150
|
+
records[key]['size'] += size
|
|
151
|
+
records[key]['fcount'] += 1
|
|
152
|
+
else:
|
|
153
|
+
(year, quarter, date, time) = get_record_date_time(dt)
|
|
154
|
+
iprec = PgIPInfo.get_missing_ipinfo(ip)
|
|
155
|
+
if not iprec: continue
|
|
156
|
+
records[key] = {'ip' : ip, 'dsid' : dsid, 'date' : date, 'time' : time, 'quarter' : quarter,
|
|
157
|
+
'size' : size, 'fcount' : 1, 'method' : method, 'engine' : engine,
|
|
158
|
+
'org_type' : iprec['org_type'], 'country' : iprec['country'],
|
|
159
|
+
'region' : iprec['region'], 'email' : iprec['email'], 'site' : site}
|
|
158
160
|
osdf.close()
|
|
161
|
+
if records: cntadd = add_usage_records(records, year)
|
|
162
|
+
PgLOG.pglog("{}: {} OSDF usage records added for {} entries at {}".format(logfile, cntadd, entcnt, PgLOG.current_datetime()), PgLOG.LOGWRN)
|
|
159
163
|
cntall += entcnt
|
|
160
|
-
|
|
161
|
-
|
|
164
|
+
if cntadd:
|
|
165
|
+
addall += cntadd
|
|
166
|
+
if addall > cntadd:
|
|
167
|
+
PgLOG.pglog("{} OSDF usage records added for {} entries at {}".format(addall, cntall, PgLOG.current_datetime()), PgLOG.LOGWRN)
|
|
162
168
|
|
|
163
169
|
|
|
164
170
|
def get_record_date_time(ctime):
|
|
@@ -174,67 +180,28 @@ def get_record_date_time(ctime):
|
|
|
174
180
|
else:
|
|
175
181
|
PgLOG.pglog(ctime + ": Invalid date/time format", PgLOG.LGEREX)
|
|
176
182
|
|
|
177
|
-
|
|
178
|
-
# Fill usage of a single online data file into table dssdb.wusage of DSS PgSQL database
|
|
179
|
-
#
|
|
180
|
-
def add_file_usage(year, logrec):
|
|
181
|
-
|
|
182
|
-
pgrec = get_wfile_wid(logrec['dsid'], logrec['wfile'])
|
|
183
|
-
if not pgrec: return 0
|
|
184
|
-
|
|
185
|
-
table = "{}_{}".format(USAGE['OSDFTBL'], year)
|
|
186
|
-
cond = "wid = {} AND method = '{}' AND date_read = '{}' AND time_read = '{}'".format(pgrec['wid'], logrec['method'], logrec['date'], logrec['time'])
|
|
187
|
-
if PgDBI.pgget(table, "", cond, PgLOG.LOGWRN): return 0
|
|
188
|
-
|
|
189
|
-
wurec = PgIPInfo.get_wuser_record(logrec['ip'], logrec['date'])
|
|
190
|
-
if not wurec: return 0
|
|
191
|
-
record = {'wid' : pgrec['wid'], 'dsid' : pgrec['dsid']}
|
|
192
|
-
record['wuid_read'] = wurec['wuid']
|
|
193
|
-
record['date_read'] = logrec['date']
|
|
194
|
-
record['time_read'] = logrec['time']
|
|
195
|
-
record['size_read'] = logrec['size']
|
|
196
|
-
record['method'] = logrec['method']
|
|
197
|
-
record['locflag'] = logrec['locflag']
|
|
198
|
-
record['ip'] = logrec['ip']
|
|
199
|
-
record['quarter'] = logrec['quarter']
|
|
200
|
-
|
|
201
|
-
if add_to_allusage(year, logrec, wurec):
|
|
202
|
-
return PgDBI.add_yearly_wusage(year, record)
|
|
203
|
-
else:
|
|
204
|
-
return 0
|
|
205
|
-
|
|
206
|
-
def add_to_allusage(year, logrec, wurec):
|
|
207
|
-
|
|
208
|
-
pgrec = {'email' : wurec['email'], 'org_type' : wurec['org_type'],
|
|
209
|
-
'country' : wurec['country'], 'region' : wurec['region']}
|
|
210
|
-
pgrec['dsid'] = logrec['dsid']
|
|
211
|
-
pgrec['date'] = logrec['date']
|
|
212
|
-
pgrec['quarter'] = logrec['quarter']
|
|
213
|
-
pgrec['time'] = logrec['time']
|
|
214
|
-
pgrec['size'] = logrec['size']
|
|
215
|
-
pgrec['method'] = logrec['method']
|
|
216
|
-
pgrec['ip'] = logrec['ip']
|
|
217
|
-
pgrec['source'] = 'P'
|
|
218
|
-
return PgDBI.add_yearly_allusage(year, pgrec)
|
|
183
|
+
def add_usage_records(records, year):
|
|
219
184
|
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
185
|
+
cnt = 0
|
|
186
|
+
for key in records:
|
|
187
|
+
record = records[key]
|
|
188
|
+
cond = "date = '{}' AND time = '{}' AND ip = '{}'".format(record['date'], record['time'], record['ip'])
|
|
189
|
+
if PgDBI.pgget(USAGE['OSDFTBL'], '', cond, PgLOG.LGEREX): continue
|
|
190
|
+
if add_to_allusage(year, record):
|
|
191
|
+
cnt += PgDBI.pgadd(USAGE['OSDFTBL'], record, PgLOG.LOGWRN)
|
|
224
192
|
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
return pgrec
|
|
193
|
+
return cnt
|
|
194
|
+
|
|
195
|
+
def add_to_allusage(year, pgrec):
|
|
196
|
+
|
|
197
|
+
record = {'source' : 'P'}
|
|
198
|
+
flds = ['ip', 'dsid', 'date', 'time', 'quarter', 'size', 'method',
|
|
199
|
+
'org_type', 'country', 'region', 'email']
|
|
200
|
+
|
|
201
|
+
for fld in flds:
|
|
202
|
+
record[fld] = pgrec[fld]
|
|
203
|
+
|
|
204
|
+
return PgDBI.add_yearly_allusage(year, record)
|
|
238
205
|
|
|
239
206
|
#
|
|
240
207
|
# call main() to start program
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
|
|
2
2
|
Retrieves usage information from OSDF Server logs under directory
|
|
3
|
-
/gpfs/fs1/collections/rda/work/zji/osdflogs/ to fill table '
|
|
3
|
+
/gpfs/fs1/collections/rda/work/zji/osdflogs/ to fill table 'osdfusage' in
|
|
4
4
|
database 'dssdb'.
|
|
5
5
|
|
|
6
6
|
Usage: fillosdfusage [-b] [-d LogFileDates] [-N NumberDay] [-p BeginDate [Enddate]]
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
rda_python_metrics/PgIPInfo.py,sha256=CfcnJxD2zHyAhemhTisdnPB72wHwE8MxS0EH4EAKnfE,9151
|
|
2
2
|
rda_python_metrics/PgView.py,sha256=r6otb3DjfiaQJdg0z8bZQAOlhr4JnrXJzp9wgWh_8qQ,24369
|
|
3
3
|
rda_python_metrics/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
|
4
|
-
rda_python_metrics/fillawsusage.py,sha256=
|
|
5
|
-
rda_python_metrics/fillawsusage.usg,sha256
|
|
4
|
+
rda_python_metrics/fillawsusage.py,sha256=8fJnUZuAWGs4dNSKNvEuqnthMJtgMm4WKidZlBzylIE,7008
|
|
5
|
+
rda_python_metrics/fillawsusage.usg,sha256=-lLTRLTaEPL2rSQ4mhpIekhyYrLGahy7NWtaq_8PCDM,611
|
|
6
6
|
rda_python_metrics/fillcdgusage.py,sha256=jJVxQ4d5at0lXXKmV7TVf2GrPWSrJv5XBiHGMc3uxEA,16706
|
|
7
7
|
rda_python_metrics/fillcdgusage.usg,sha256=5lkd4Zdi72nQcha-JtbaLnxl66V4wBDykKwQtUOtMrw,667
|
|
8
8
|
rda_python_metrics/fillcodusage.py,sha256=_RUcgY1Cf17dSjidhQCMOwVaPO1VL26s3bYGT8oSPWU,8322
|
|
@@ -17,8 +17,8 @@ rda_python_metrics/fillipinfo.py,sha256=BrboxAIs8Q5jhz_4GYW_GibMT5GyEbnQkYfFR8hc
|
|
|
17
17
|
rda_python_metrics/fillipinfo.usg,sha256=YeCR58xGv0emqHUZ_9R977HrqaeBwbd6j5QRF2Lc7TA,702
|
|
18
18
|
rda_python_metrics/filloneorder.py,sha256=ADHbcKCDh9bJunnxYbkbjwU2QpC43hvGlLWaURHNxkg,5433
|
|
19
19
|
rda_python_metrics/filloneorder.usg,sha256=mtOySKx6-D4k2bbTcmi6cSYtINiycRyHQkHozi0CQu0,1466
|
|
20
|
-
rda_python_metrics/fillosdfusage.py,sha256
|
|
21
|
-
rda_python_metrics/fillosdfusage.usg,sha256=
|
|
20
|
+
rda_python_metrics/fillosdfusage.py,sha256=--c6PzeZ6EZXpb0zxy4lZCDjwwIBb7c6xgP7QCKLoMQ,7087
|
|
21
|
+
rda_python_metrics/fillosdfusage.usg,sha256=Qc5YdUuOiAH74FfVdkwkrQRDYXcASVbxMdBsVQj4X1k,635
|
|
22
22
|
rda_python_metrics/fillrdadb.py,sha256=cb6upPApAZloOouUSzZZmjGvOsYT_Nzh9Lz926WE3ZQ,5333
|
|
23
23
|
rda_python_metrics/fillrdadb.usg,sha256=E_Bf4G8yVABogjRmIOaIbTGgnII2W6RltaFad2XEV2Q,1228
|
|
24
24
|
rda_python_metrics/filltdsusage.py,sha256=vOwVzAtWUHO4O-FCSJMg0GKxw-Xc5AzSbfqVFktUAlA,10201
|
|
@@ -46,9 +46,9 @@ rda_python_metrics/viewwebfile.py,sha256=HSMNkQQawonu6W3blV7g9UbJuNy9VAOn9COqgmj
|
|
|
46
46
|
rda_python_metrics/viewwebfile.usg,sha256=lTNi8Yu8BUJuExEDJX-vsJyWUSUIQTS-DiiBEVFo33s,10054
|
|
47
47
|
rda_python_metrics/viewwebusage.py,sha256=ES2lI8NaCeCpTGi94HU-cDRBxHMiUBbplyYsZf2KqF0,16650
|
|
48
48
|
rda_python_metrics/viewwebusage.usg,sha256=OVDZ78p87E3HLW34ZhasNJ7Zmw8XXjmZPPWZfRhPLXo,9936
|
|
49
|
-
rda_python_metrics-1.0.
|
|
50
|
-
rda_python_metrics-1.0.
|
|
51
|
-
rda_python_metrics-1.0.
|
|
52
|
-
rda_python_metrics-1.0.
|
|
53
|
-
rda_python_metrics-1.0.
|
|
54
|
-
rda_python_metrics-1.0.
|
|
49
|
+
rda_python_metrics-1.0.35.dist-info/licenses/LICENSE,sha256=1dck4EAQwv8QweDWCXDx-4Or0S8YwiCstaso_H57Pno,1097
|
|
50
|
+
rda_python_metrics-1.0.35.dist-info/METADATA,sha256=HMw2rrp4xzVvgk1iMC2286DfW6pyjn5KIf4GtkAuiUk,761
|
|
51
|
+
rda_python_metrics-1.0.35.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
52
|
+
rda_python_metrics-1.0.35.dist-info/entry_points.txt,sha256=ctvX0Gx9zdnKpHx5sjZdl7_sLSR80LKYhPky9qkEpug,1239
|
|
53
|
+
rda_python_metrics-1.0.35.dist-info/top_level.txt,sha256=aoBgbR_o70TP0QmMW0U6inRHYtfKld47OBmnWnLnDOs,19
|
|
54
|
+
rda_python_metrics-1.0.35.dist-info/RECORD,,
|
|
File without changes
|
{rda_python_metrics-1.0.33.dist-info → rda_python_metrics-1.0.35.dist-info}/entry_points.txt
RENAMED
|
File without changes
|
{rda_python_metrics-1.0.33.dist-info → rda_python_metrics-1.0.35.dist-info}/licenses/LICENSE
RENAMED
|
File without changes
|
|
File without changes
|