rda-python-metrics 1.0.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of rda-python-metrics might be problematic. Click here for more details.
- rda_python_metrics/PgIPInfo.py +188 -0
- rda_python_metrics/PgView.py +782 -0
- rda_python_metrics/__init__.py +1 -0
- rda_python_metrics/fillawsusage.py +282 -0
- rda_python_metrics/fillawsusage.usg +17 -0
- rda_python_metrics/fillcodusage.py +247 -0
- rda_python_metrics/fillcodusage.usg +21 -0
- rda_python_metrics/fillcountry.py +79 -0
- rda_python_metrics/fillendtime.py +93 -0
- rda_python_metrics/fillglobususage.py +287 -0
- rda_python_metrics/fillglobususage.usg +17 -0
- rda_python_metrics/fillipinfo.py +185 -0
- rda_python_metrics/fillipinfo.usg +18 -0
- rda_python_metrics/filloneorder.py +155 -0
- rda_python_metrics/filloneorder.usg +41 -0
- rda_python_metrics/fillrdadb.py +151 -0
- rda_python_metrics/fillrdadb.usg +32 -0
- rda_python_metrics/filltdsusage.py +289 -0
- rda_python_metrics/filltdsusage.usg +17 -0
- rda_python_metrics/filluser.py +216 -0
- rda_python_metrics/filluser.usg +16 -0
- rda_python_metrics/logarch.py +359 -0
- rda_python_metrics/logarch.usg +27 -0
- rda_python_metrics/pgperson.py +72 -0
- rda_python_metrics/pgusername.py +50 -0
- rda_python_metrics/viewallusage.py +350 -0
- rda_python_metrics/viewallusage.usg +198 -0
- rda_python_metrics/viewcheckusage.py +289 -0
- rda_python_metrics/viewcheckusage.usg +185 -0
- rda_python_metrics/viewcodusage.py +314 -0
- rda_python_metrics/viewcodusage.usg +184 -0
- rda_python_metrics/viewordusage.py +340 -0
- rda_python_metrics/viewordusage.usg +224 -0
- rda_python_metrics/viewrqstusage.py +362 -0
- rda_python_metrics/viewrqstusage.usg +217 -0
- rda_python_metrics/viewtdsusage.py +323 -0
- rda_python_metrics/viewtdsusage.usg +191 -0
- rda_python_metrics/viewwebfile.py +294 -0
- rda_python_metrics/viewwebfile.usg +212 -0
- rda_python_metrics/viewwebusage.py +371 -0
- rda_python_metrics/viewwebusage.usg +211 -0
- rda_python_metrics-1.0.4.dist-info/METADATA +18 -0
- rda_python_metrics-1.0.4.dist-info/RECORD +47 -0
- rda_python_metrics-1.0.4.dist-info/WHEEL +5 -0
- rda_python_metrics-1.0.4.dist-info/entry_points.txt +22 -0
- rda_python_metrics-1.0.4.dist-info/licenses/LICENSE +21 -0
- rda_python_metrics-1.0.4.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,362 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
#
|
|
3
|
+
###############################################################################
|
|
4
|
+
#
|
|
5
|
+
# Title : viewrqstusage
|
|
6
|
+
# Author : Zaihua Ji, zji@ucar.edu
|
|
7
|
+
# Date : 03/15/2022
|
|
8
|
+
# 2025-03-28 transferred to package rda_python_metrics from
|
|
9
|
+
# https://github.com/NCAR/rda-database.git
|
|
10
|
+
# Purpose : python program to view usage information for Web Online files
|
|
11
|
+
# staged by utility prgoram dsrqst.
|
|
12
|
+
#
|
|
13
|
+
# Github : https://github.com/NCAR/rda-python-metrics.git
|
|
14
|
+
#
|
|
15
|
+
###############################################################################
|
|
16
|
+
#
|
|
17
|
+
import os
|
|
18
|
+
import re
|
|
19
|
+
import sys
|
|
20
|
+
from rda_python_common import PgLOG
|
|
21
|
+
from rda_python_common import PgUtil
|
|
22
|
+
from rda_python_common import PgDBI
|
|
23
|
+
from . import PgView
|
|
24
|
+
|
|
25
|
+
VUSG = {
|
|
26
|
+
'SNMS' : "ABCDEFGHIJKLMNOPQRSTUVWYZ", # all available short field names in FLDS
|
|
27
|
+
'OPTS' : 'AabcCdDeEfFghHiIlLmnMNoOsStTUvwy', # all available options, used for params
|
|
28
|
+
'NOPT' : 'abhnwz', # stand alone option without inputs
|
|
29
|
+
'ACND' : 'cdefgilmMosStvy', # available array condition options
|
|
30
|
+
'RCND' : 'DFNT', # available range condition options
|
|
31
|
+
'CNDS' : 'acdDefFgilmMNosStTvy', # condition options, ACND, RCND and 'a'
|
|
32
|
+
'HCND' : 'N', # condition options for having clause
|
|
33
|
+
'ECND' : 'my', # condition options need evaluating
|
|
34
|
+
'SFLD' : 'DEGILNOPSTUW', # string fields, to be quoted in condition
|
|
35
|
+
'UFLD' : 'ILNOSW', # string fields must be in upper case
|
|
36
|
+
'LFLD' : 'EPT', # string fields must be in lower case
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
# keys FLDS - short field names
|
|
40
|
+
# column 0 - column title showing in usage view
|
|
41
|
+
# column 1 - field name in format as shown in select clauses
|
|
42
|
+
# column 2 - field name shown in where condition query string
|
|
43
|
+
# column 3 - table name that the field belongs to
|
|
44
|
+
# column 4 - output field length, the longer one of data size and comlun title, determine
|
|
45
|
+
# dynamically if it is 0. Negative values indicate right justification
|
|
46
|
+
# column 5 - precision for floating point value if positive and show total value if not zero
|
|
47
|
+
# column 6 - field flag to indicate it is a group, distinct or sum field
|
|
48
|
+
FLDS = {
|
|
49
|
+
# SHRTNM COLUMNNANE FIELDNAME CNDNAME TBLNAM Size Prc Grp/Sum
|
|
50
|
+
'D' : ['DATE', "date_rqst", 'date_rqst', 'dspurge', 10, 0, 'G'],
|
|
51
|
+
'E' : ['EMAIL', "dspurge.email", 'dspurge.email', 'dspurge', 0, 0, 'G'],
|
|
52
|
+
'U' : ['FILENAME', "wfile", 'wfile', 'wfpurge', 0, 0, 'G'],
|
|
53
|
+
'G' : ['ORGNAME', "org_name", 'org_name', 'wuser', 0, 0, 'G'],
|
|
54
|
+
'I' : ['FIRSTNAME', "fstname", 'fstname', 'wuser', 0, 0, 'G'],
|
|
55
|
+
'K' : ['TIME', "time_rqst", 'time_rqst', 'dspurge', 8, 0, 'G'],
|
|
56
|
+
'L' : ['LASTNAME', "lstname", 'lstname', 'wuser', 0, 0, 'G'],
|
|
57
|
+
'M' : ['MONTH', PgDBI.fmtym("date_rqst"), 'date_rqst', 'dspurge', 7, 0, 'G'],
|
|
58
|
+
'N' : ['COUNTRY', "country", 'country', 'wuser', 0, 0, 'G'],
|
|
59
|
+
'O' : ['ORGTYPE', "org_type", 'org_type', 'wuser', 7, 0, 'G'],
|
|
60
|
+
'P' : ['PROCESSBY', "specialist", 'specialist', 'dspurge', 9, 0, 'G'],
|
|
61
|
+
'Q' : ['QUARTER', "quarter", 'quarter', 'dspurge', 7, 0, 'G'],
|
|
62
|
+
'R' : ['RQSTDETAIL', "rinfo", 'rinfo', 'dspurge', 0, 0, 'G'],
|
|
63
|
+
'S' : ['FROM', "fromflag", 'fromflag', 'dspurge', 0, 0, 'G'],
|
|
64
|
+
'T' : ['DATASET', "dspurge.dsid", 'dspurge.dsid', 'dspurge', 0, 0, 'G'],
|
|
65
|
+
'V' : ['RINDEX', "dspurge.rindex", 'dspurge.rindex','dspurge', 0, 0, 'G'],
|
|
66
|
+
'W' : ['RTYPE', "rqsttype", 'rqsttype', 'dspurge', 5, 0, 'G'],
|
|
67
|
+
'Y' : ['YEAR', PgDBI.fmtyr("date_rqst"), 'date_rqst', 'dspurge', 4, 0, 'G'],
|
|
68
|
+
'A' : ['DSCOUNT', "dspurge.dsid", 'A', 'dspurge', -7, -1, 'D'],
|
|
69
|
+
'B' : ['MBYTERQST', "round(sum(size_request)/1000000, 4)", 'B', 'dspurge', -14, 3, 'S'],
|
|
70
|
+
'F' : ['MBINPUT', "round(sum(size_input)/1000000, 4)", 'F', 'dspurge', -14, 3, 'S'],
|
|
71
|
+
'C' : ['#UNIQUSER', "dspurge.email", 'C', 'dspurge', -9, -1, 'D'],
|
|
72
|
+
'H' : ['#FILERQST', "count(fcount)", 'H', 'dspurge', -9, -1, 'S'],
|
|
73
|
+
'J' : ['#UNIQFILE', "wfpurge.wfile", 'J', 'wfpurge', -9, -1, 'D'],
|
|
74
|
+
'Z' : ['#REQUEST', "dspurge.rindex", 'Z', 'dspurge', -8, -1, 'D'],
|
|
75
|
+
'X' : ['INDEX', "", 'X', '', -6, 0, ' ']
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
# keys EXPAND - short field names allow zero usage
|
|
79
|
+
# column 0 - expand ID for group of fields
|
|
80
|
+
# column 1 - field name shown in where condition query string
|
|
81
|
+
# column 2 - field name in format as shown in select clauses
|
|
82
|
+
# column 3 - table name that the field belongs to
|
|
83
|
+
EXPAND = {
|
|
84
|
+
# SHRTNM EXPID CNDSTR FIELDNAME TBLNAM
|
|
85
|
+
'D' : ["TIME", "dDmy"],
|
|
86
|
+
'M' : ["TIME", "dDmy"],
|
|
87
|
+
'Q' : ["TIME", "dDmy"],
|
|
88
|
+
'Y' : ["TIME", "dDmy"],
|
|
89
|
+
|
|
90
|
+
'E' : ["USER", "eilgco", "email", "wuser"],
|
|
91
|
+
'I' : ["USER", "eilgco", "fstname", "wuser"],
|
|
92
|
+
'L' : ["USER", "eilgco", "lstname", "wuser"],
|
|
93
|
+
'G' : ["USER", "eilgco", "org_name", "wuser"],
|
|
94
|
+
'O' : ["USER", "eilgco", "org_type", "wuser"],
|
|
95
|
+
'N' : ["USER", "eilgco", "country", "wuser"],
|
|
96
|
+
|
|
97
|
+
'T' : ["FILE", "StT", "rcrqst.dsid", "rcrqst"],
|
|
98
|
+
'U' : ["FILE", "StT", "dsowner.specialist", "dsowner"],
|
|
99
|
+
|
|
100
|
+
'S' : ["FROM", "s", "fromflag", "dspurge"],
|
|
101
|
+
|
|
102
|
+
'W' : ["RTYPE", "M", "rqsttype", "dspurge"],
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
# valid options for params, a hash array of command line parameters
|
|
106
|
+
# a -- 1 to view all usage info available
|
|
107
|
+
# A -- number or records to return
|
|
108
|
+
# c -- array of specified country codes
|
|
109
|
+
# C -- a string of short field names for viewing usages
|
|
110
|
+
# d -- array of specified dates
|
|
111
|
+
# D -- dates range, array of 1 or 2 dates in format of YYYY-MM-DD
|
|
112
|
+
# e -- array of specified email addresses
|
|
113
|
+
# E -- use given date or date range for email notice of data update
|
|
114
|
+
# f -- array of specified WEB Online file names
|
|
115
|
+
# F -- file name range, array of 1 or 2 file names
|
|
116
|
+
# g -- array of specified orginization names
|
|
117
|
+
# h -- for give emails, include their histical emails registered before
|
|
118
|
+
# H -- a string of report title to replace the default one
|
|
119
|
+
# i -- array of specified first names
|
|
120
|
+
# I -- use given email IDs for email notice of data update
|
|
121
|
+
# l -- array of specified last names
|
|
122
|
+
# L -- column delimiter for output
|
|
123
|
+
# m -- array of specified months
|
|
124
|
+
# M -- array of specified request types
|
|
125
|
+
# N -- number read range, arrage of 1 or 2 integers
|
|
126
|
+
# o -- array of specified orginization types
|
|
127
|
+
# O -- a string of short field names for sorting on
|
|
128
|
+
# q -- array of the specified quarters, normally combined with years
|
|
129
|
+
# s -- array of from-flags the requests are initiated
|
|
130
|
+
# S -- array of login names of specialists who own the requests
|
|
131
|
+
# t -- array of specified dataset names
|
|
132
|
+
# T -- dataset range, array of 1 or 2 dataset names
|
|
133
|
+
# U -- use given unit for file or data sizes
|
|
134
|
+
# v -- aray of specified request indices
|
|
135
|
+
# w -- generate view without totals
|
|
136
|
+
# y -- array of specified years
|
|
137
|
+
# z -- generate view including entries with zero usage
|
|
138
|
+
params = {}
|
|
139
|
+
|
|
140
|
+
# relationship between parameter options and short field names, A option is not
|
|
141
|
+
# related to a field name if it is not in keys SNS
|
|
142
|
+
SNS = {
|
|
143
|
+
'c' : 'N', 'd' : 'D', 'D' : 'D', 'e' : 'E', 'f' : 'U', 'F' : 'U',
|
|
144
|
+
'g' : 'G', 'i' : 'I', 'l' : 'L', 'm' : 'M', 'M' : 'W', 'N' : 'H',
|
|
145
|
+
'o' : 'O', 'q' : 'Q', 's' : 'S', 'S' : 'P', 't' : 'T', 'T' : 'T',
|
|
146
|
+
'v' : 'V', 'y' : 'Y'
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
tablenames = fieldnames = condition = ''
|
|
150
|
+
sfields = []
|
|
151
|
+
gfields = []
|
|
152
|
+
dfields = []
|
|
153
|
+
pgname = 'viewallusage'
|
|
154
|
+
rsname = "size_request"
|
|
155
|
+
|
|
156
|
+
#
|
|
157
|
+
# main function to run this program
|
|
158
|
+
#
|
|
159
|
+
def main():
|
|
160
|
+
|
|
161
|
+
PgDBI.view_dbinfo()
|
|
162
|
+
argv = sys.argv[1:]
|
|
163
|
+
inputs = []
|
|
164
|
+
option = 'C' # default option
|
|
165
|
+
|
|
166
|
+
for arg in argv:
|
|
167
|
+
if re.match(r'^-.*$', arg):
|
|
168
|
+
curopt = arg[1:2]
|
|
169
|
+
if curopt and VUSG['OPTS'].find(curopt) > -1:
|
|
170
|
+
if VUSG['NOPT'].find(option) > -1:
|
|
171
|
+
params[option] = 1
|
|
172
|
+
elif inputs:
|
|
173
|
+
params[option]= inputs # record input array
|
|
174
|
+
inputs = [] # empty input array
|
|
175
|
+
option = curopt # start a new option
|
|
176
|
+
else:
|
|
177
|
+
PgLOG.pglog(arg + ": Unknown Option", PgLOG.LGWNEX)
|
|
178
|
+
else:
|
|
179
|
+
val = arg
|
|
180
|
+
if val != '!':
|
|
181
|
+
if option == 's':
|
|
182
|
+
val = int(val)*1000000 # convert MBytes to Bytes
|
|
183
|
+
elif option in SNS:
|
|
184
|
+
sfld = SNS[option]
|
|
185
|
+
if VUSG['SFLD'].find(sfld) > -1:
|
|
186
|
+
if VUSG['UFLD'].find(sfld) > -1:
|
|
187
|
+
val = arg.upper() # in case not in upper case
|
|
188
|
+
elif VUSG['LFLD'].find(sfld) > -1:
|
|
189
|
+
val = arg.lower() # in case not in lower case
|
|
190
|
+
if option == 'c':
|
|
191
|
+
val = PgView.get_country_name(val)
|
|
192
|
+
elif option == 't' or option == 'T':
|
|
193
|
+
val = PgUtil.format_dataset_id(val) # add 'ds' if only numbers
|
|
194
|
+
val = "'{}'".format(val)
|
|
195
|
+
inputs.append(val)
|
|
196
|
+
|
|
197
|
+
# record the last option
|
|
198
|
+
if VUSG['NOPT'].find(option) > -1:
|
|
199
|
+
params[option] = 1
|
|
200
|
+
elif inputs:
|
|
201
|
+
params[option] = inputs # record input array
|
|
202
|
+
|
|
203
|
+
if not params:
|
|
204
|
+
PgLOG.show_usage(pgname)
|
|
205
|
+
else:
|
|
206
|
+
check_enough_options()
|
|
207
|
+
|
|
208
|
+
if 'o' not in params:
|
|
209
|
+
if 'e' not in params:
|
|
210
|
+
params['o'] = ['!', "'DSS'"] # default to exclude 'DSS' for organization
|
|
211
|
+
elif params['o'][0] == "'ALL'":
|
|
212
|
+
del params['o']
|
|
213
|
+
|
|
214
|
+
usgtable = 'dspurge'
|
|
215
|
+
build_query_strings(usgtable) # build tablenames, fieldnames, and conditions
|
|
216
|
+
records = PgDBI.pgmget(tablenames, fieldnames, condition, PgLOG.UCLWEX)
|
|
217
|
+
if not records: PgLOG.pglog("No Usage Found For Given Conditions", PgLOG.LGWNEX)
|
|
218
|
+
totals = None if 'w' in params else {}
|
|
219
|
+
if dfields or totals != None:
|
|
220
|
+
records = PgView.compact_hash_groups(records, gfields, sfields, dfields, totals)
|
|
221
|
+
if 'z' in params: records = expand_records(records)
|
|
222
|
+
ostr = params['O'][0] if 'O' in params else params['C'][0]
|
|
223
|
+
records = PgView.order_records(records, ostr.replace('X', ''))
|
|
224
|
+
PgView.simple_output(params, FLDS, records, totals)
|
|
225
|
+
|
|
226
|
+
PgLOG.pgexit(0)
|
|
227
|
+
|
|
228
|
+
#
|
|
229
|
+
# check if enough information entered on command line for generate view/report, exit if not
|
|
230
|
+
#
|
|
231
|
+
def check_enough_options():
|
|
232
|
+
|
|
233
|
+
global rsname
|
|
234
|
+
cols = params['C'][0] if 'C' in params else 'X'
|
|
235
|
+
if cols == 'X': PgLOG.pglog("{}: miss field names '{}'".format(pgname, VUSG['SNMS']), PgLOG.LGWNEX)
|
|
236
|
+
|
|
237
|
+
if cols.find('Q') > -1 and cols.find('Y') < 0: # add Y if Q included
|
|
238
|
+
cols = re.sub('Q', 'YQ', cols)
|
|
239
|
+
params['C'][0] = cols
|
|
240
|
+
|
|
241
|
+
if cols.find('J') > -1 and (cols.find('B') > -1 or cols.find('F') > -1):
|
|
242
|
+
PgLOG.pglog(pgname + ": CANNOT view unique file counts with total data sizes", PgLOG.LGWNEX)
|
|
243
|
+
|
|
244
|
+
for sn in cols:
|
|
245
|
+
if sn == 'X': continue # do not process INDEX field
|
|
246
|
+
if VUSG['SNMS'].find(sn) < 0:
|
|
247
|
+
PgLOG.pglog("{}: Field {} must be in field names '{}X'".format(pgname, sn, VUSG['SNMS']), PgLOG.LGWNEX)
|
|
248
|
+
if 'z' not in params or sn in EXPAND: continue
|
|
249
|
+
fld = FLDS[sn]
|
|
250
|
+
if fld[6] != 'G': continue
|
|
251
|
+
PgLOG.pglog("{}: cannot show zero usage for unexpandable field {} - {}".formt(pgname, sn, fld[0]), PgLOG.LGWNEX)
|
|
252
|
+
|
|
253
|
+
if 'E' in params or 'I' in params:
|
|
254
|
+
if 'z' in params:
|
|
255
|
+
PgLOG.pglog(pgname + ": option -z and -E/-I can not be present at the same time", PgLOG.LGWNEX)
|
|
256
|
+
elif 't' not in params or len(params['t']) > 1:
|
|
257
|
+
PgLOG.pglog(pgname + ": specify one dataset for viewing usage of notified users", PgLOG.LGWNEX)
|
|
258
|
+
elif 'E' in params and 'I' in params:
|
|
259
|
+
PgLOG.pglog(pgname + ": option -E and -I can not be present at the same time", PgLOG.LGWNEX)
|
|
260
|
+
|
|
261
|
+
if cols.find('F') > -1:
|
|
262
|
+
ms = re.search(r'([USJ])', cols)
|
|
263
|
+
if ms:
|
|
264
|
+
PgLOG.pglog("{}: Canot get column F (input sizes) with column {}".format(pgname, ms.group(1)), PgLOG.LGWNEX)
|
|
265
|
+
else:
|
|
266
|
+
ms = re.search(r'([fFs])', params)
|
|
267
|
+
if ms: PgLOG.pglog("{}: Canot get column F (input sizes) for option -{}".format(pgname, ms.group(1)), PgLOG.LGWNEX)
|
|
268
|
+
|
|
269
|
+
if cols.find('B') > -1 and (re.search(r'[UJ]', cols) or 'f' in params or 'F' in params):
|
|
270
|
+
rsname = "size"
|
|
271
|
+
FLDS['B'][3] = "wfpurge"
|
|
272
|
+
if 'U' not in params: params['U'] = ['M']
|
|
273
|
+
|
|
274
|
+
for opt in params:
|
|
275
|
+
if VUSG['CNDS'].find(opt) > -1: return
|
|
276
|
+
PgLOG.pglog("{}: miss condition options '{}'".format(pgname, VUSG['CNDS']), PgLOG.LGWNEX)
|
|
277
|
+
|
|
278
|
+
#
|
|
279
|
+
# process parameter options to build all query strings
|
|
280
|
+
# global variables are used directly and nothing passes in and returns back
|
|
281
|
+
#
|
|
282
|
+
def build_query_strings(usgtable):
|
|
283
|
+
|
|
284
|
+
# initialize query strings
|
|
285
|
+
global condition, fieldnames, tablenames
|
|
286
|
+
joins = having = groupnames = ''
|
|
287
|
+
tablenames = usgtable
|
|
288
|
+
cols = params['C'][0]
|
|
289
|
+
|
|
290
|
+
if 'U' in params: # reset units for file and read sizes
|
|
291
|
+
if cols.find('B') > -1: FLDS['B'] = PgView.set_data_unit(FLDS['B'], params['U'][0], "sum({})".format(rsname))
|
|
292
|
+
if cols.find('S') > -1: FLDS['F'] = PgView.set_data_unit(FLDS['F'], params['U'][0], "size_input")
|
|
293
|
+
|
|
294
|
+
if 'e' in params and 'h' in params: params['e'] = PgView.include_historic_emails(params['e'], 3)
|
|
295
|
+
|
|
296
|
+
for opt in params:
|
|
297
|
+
if VUSG['NOPT'].find(opt) > -1: continue
|
|
298
|
+
if opt == 'C': # build field, table and group names
|
|
299
|
+
for sn in cols:
|
|
300
|
+
if sn == 'X': continue # do not process INDEX field
|
|
301
|
+
fld = FLDS[sn]
|
|
302
|
+
if fieldnames: fieldnames += ', '
|
|
303
|
+
fieldnames += "{} {}".format(fld[1], sn) # add to field name string
|
|
304
|
+
(tablenames, joins) = PgView.join_query_tables(fld[3], tablenames, joins, usgtable)
|
|
305
|
+
if fld[6] == 'S':
|
|
306
|
+
sfields.append(sn)
|
|
307
|
+
else:
|
|
308
|
+
if groupnames: groupnames += ', '
|
|
309
|
+
groupnames += sn # add to group name string
|
|
310
|
+
if fld[6] == 'D':
|
|
311
|
+
dfields.append(sn)
|
|
312
|
+
else:
|
|
313
|
+
gfields.append(sn)
|
|
314
|
+
elif opt == 'O':
|
|
315
|
+
continue # order records later
|
|
316
|
+
elif VUSG['CNDS'].find(opt) > -1:
|
|
317
|
+
sn = SNS[opt]
|
|
318
|
+
fld = FLDS[sn]
|
|
319
|
+
# build having and where conditon strings
|
|
320
|
+
cnd = PgView.get_view_condition(opt, sn, fld, params, VUSG)
|
|
321
|
+
if cnd:
|
|
322
|
+
if VUSG['HCND'].find(opt) > -1:
|
|
323
|
+
if having: having += ' AND '
|
|
324
|
+
having += cnd
|
|
325
|
+
else:
|
|
326
|
+
if condition: condition += ' AND '
|
|
327
|
+
condition += cnd
|
|
328
|
+
(tablenames, joins) = PgView.join_query_tables(fld[3], tablenames, joins, usgtable)
|
|
329
|
+
|
|
330
|
+
# append joins, group by, order by, and having strings to condition string
|
|
331
|
+
if 'E' in params or 'I' in params:
|
|
332
|
+
(tablenames, joins) = PgView.join_query_tables("emreceive", tablenames, joins, usgtable)
|
|
333
|
+
if joins:
|
|
334
|
+
if condition:
|
|
335
|
+
condition = "{} AND {}".format(joins, condition)
|
|
336
|
+
else:
|
|
337
|
+
condition = joins
|
|
338
|
+
if 'n' in params: condition += " AND location > ' '"
|
|
339
|
+
if 'E' in params or 'I' in params:
|
|
340
|
+
condition += PgView.notice_condition(params['E'], params['I'], params['t'][0])
|
|
341
|
+
if groupnames and sfields: condition += " GROUP BY " + groupnames
|
|
342
|
+
if having: condition += " HAVING " + having
|
|
343
|
+
|
|
344
|
+
def expand_records(records):
|
|
345
|
+
|
|
346
|
+
recs = PgView.expand_query("TIME", records, params, EXPAND)
|
|
347
|
+
|
|
348
|
+
trecs = PgView.expand_query("USER", records, params, EXPAND, VUSG, SNS, FLDS)
|
|
349
|
+
if trecs: PgUtil.crosshash(recs, trecs)
|
|
350
|
+
|
|
351
|
+
trecs = PgView.expand_query("FILE", records, params, EXPAND, VUSG, SNS, FLDS)
|
|
352
|
+
if trecs: recs = PgUtil.crosshash(recs, trecs)
|
|
353
|
+
|
|
354
|
+
trecs = PgView.expand_query("METHOD", records, params, EXPAND, VUSG, SNS, FLDS)
|
|
355
|
+
if trecs: recs = PgUtil.crosshash(recs, trecs)
|
|
356
|
+
|
|
357
|
+
return PgUtil.joinhash(records, recs, 0, 1)
|
|
358
|
+
|
|
359
|
+
#
|
|
360
|
+
# call main() to start program
|
|
361
|
+
#
|
|
362
|
+
if __name__ == "__main__": main()
|
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
|
|
2
|
+
View usage information of online data files staged on DSS Web Server via utility
|
|
3
|
+
program 'dsrqst' from file requesting information stored in MySQL database 'RDADB'.
|
|
4
|
+
|
|
5
|
+
Usage: viewrqstusage [-C] ColumnNames [-O OrderColumnNames] [-a] \
|
|
6
|
+
[-A RowLimit] [-c CountryCodes] [-d DataRequestDates] \
|
|
7
|
+
[-D StartRequestDate [EndRequestDate]] [-e EmailList] -h \
|
|
8
|
+
[-E StartNoticeDate [EndNoticeDate]] \
|
|
9
|
+
[-f FileNameList] [-F MinFileName [MaxFileName]] \
|
|
10
|
+
[-g OrganizationNames] [-i FirstNamelist] \
|
|
11
|
+
[-I EmailIDList] [-l LastNameList] \
|
|
12
|
+
[-m MonthList] [-M RequestTypeList] \
|
|
13
|
+
[-N MinNumberRead [MaxNumberRead]] \
|
|
14
|
+
[-o OrganizationTypes] [-q QuaterList] [-s FromFlags] \
|
|
15
|
+
[-S SpecialistLoginNames] [-t DatasetList] \
|
|
16
|
+
[-T MinDataset [MaxDataset]] [-v RequestIndices] [-y YearList] \
|
|
17
|
+
[-H Title] [-L Delimiter] [-U SizeUnit] [n] [-w] [-z] \
|
|
18
|
+
[> OutputFileName] [| lp -d PrinterName]
|
|
19
|
+
|
|
20
|
+
Specify [-C] ColumnNames, refer to Option -C section for detail
|
|
21
|
+
description, and choose at least one of the condition options, -a, -c,
|
|
22
|
+
-d, -D, -e, -E, -f, -F, -g, -i, -i, -l, -m, -M, -N, -o, -q, -s, -S, -t,
|
|
23
|
+
-T, -v and -y, to run this application.
|
|
24
|
+
|
|
25
|
+
For all condition options, except option -a, an '!' sign can be added
|
|
26
|
+
between an option flag and its option values to get an excluding
|
|
27
|
+
condition. For example, choose '-o ! OrganizationList' to gather WEB
|
|
28
|
+
file usage by users from organizations other than the ones given in
|
|
29
|
+
OrganizationList. Refer to the example given at the end of this help
|
|
30
|
+
document for how to select excluding condition.
|
|
31
|
+
|
|
32
|
+
String condition options, -c, -e, -f, -g, -i, -l, -o, -S and -t, allow
|
|
33
|
+
wildcard inputs. '%' matches any number of characters and '_' matches
|
|
34
|
+
any one character. Refer to the example given at the end of this help
|
|
35
|
+
document for how to use wildcard for string condition options.
|
|
36
|
+
|
|
37
|
+
Output of this application is defaulted to page format with a page
|
|
38
|
+
header on each page. A page header includes main title, sub titles and
|
|
39
|
+
column titles according to which column names and options are selected,
|
|
40
|
+
as well as page number and report date. If the output is used directly
|
|
41
|
+
for input of other applications, add option -w to remove page header
|
|
42
|
+
and show only the column titles and the WEB file usage information.
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
Column Options:
|
|
46
|
+
- Option -C, the ColumnNames must be present to run this application.
|
|
47
|
+
The flag -C can be omitted if it is the first parameter option on
|
|
48
|
+
the command line. The ColumnNames is a string that includes column
|
|
49
|
+
names listed below:
|
|
50
|
+
|
|
51
|
+
COLUMN - COLUMN - COLUMN
|
|
52
|
+
NAME - TITLE - DESCRIPTION
|
|
53
|
+
GroupColumns:
|
|
54
|
+
D*- DATE - date requested, format as YYYY-MM-DD, for example 2004-04-25
|
|
55
|
+
E*- EMAIL - user email address
|
|
56
|
+
G*- ORGNAME - organization name
|
|
57
|
+
I*- FIRSTNAME - user first name
|
|
58
|
+
K - TIME - time requested
|
|
59
|
+
L*- LASTNAME - user last name
|
|
60
|
+
M*- MONTH - format as YYYY-MM, for example 2004-04
|
|
61
|
+
N*- COUNTRY - country codes users from
|
|
62
|
+
O*- ORGTYPE - organization types (DSS, NCAR, UNIV and OTHER)
|
|
63
|
+
P*- PROCESSBY - login names of specialists who own the requests
|
|
64
|
+
Q*- QUARTER - quarter of year, 1, 2, 3, or 4
|
|
65
|
+
R - RQSTNOTE - detailed request note for subset
|
|
66
|
+
S - FROM - from-flags the requests are initiated
|
|
67
|
+
T*- DATASET - format as dsnnn.n, for example d540001
|
|
68
|
+
U*- FILENAME - WEB file name accessed
|
|
69
|
+
V*- RINDEX - request index
|
|
70
|
+
W*- METHOD - download method of the web data file
|
|
71
|
+
Y*- YEAR - format as YYYY, for example 2004
|
|
72
|
+
|
|
73
|
+
* - field names can processed with zero usages
|
|
74
|
+
SummaryColumns:
|
|
75
|
+
A - DSCOUNT - number of datasets in given GroupColumns
|
|
76
|
+
B - MBYTERQST - requested data sizes, in MB, by given GroupColumns
|
|
77
|
+
F - MBINPUT - input data sizes, in MB, by given GroupColumns
|
|
78
|
+
C - #UNIQUSER - number of users in given GroupColumns
|
|
79
|
+
J - #UNIQFILE - number of unique files requested by given GroupColumns
|
|
80
|
+
H - #FILERQST - number of files requested by given GroupColumns
|
|
81
|
+
Z - #REQUEST - number of requests by given GroupColumns
|
|
82
|
+
IndexColumn:
|
|
83
|
+
X - INDEX - index of line, it should be the first column
|
|
84
|
+
|
|
85
|
+
The column names are used to build up string of ColumnNames, while
|
|
86
|
+
their associated column titles are shown in view/report output of
|
|
87
|
+
this application. The display order of the column titles is
|
|
88
|
+
determined by the order of the column names in the ColumnNames
|
|
89
|
+
string. At least one of the group and summary columns must be
|
|
90
|
+
selected, in the ColumnNames string, to generate WEB file usage
|
|
91
|
+
view/report;
|
|
92
|
+
|
|
93
|
+
For example, choose '-C EMB' to display column titles of EMAIL,
|
|
94
|
+
MONTH and MBYTEREAD, in the first, second and third columns
|
|
95
|
+
respectively, for numbers of MBytes of data read by each user
|
|
96
|
+
in each month;
|
|
97
|
+
|
|
98
|
+
- Option -O, sort WEB file usage information in ascending or descending
|
|
99
|
+
order based on the column names specified in OrderColumnNames
|
|
100
|
+
string. These order column names must be in the selected
|
|
101
|
+
[-C] ColumnNames string. If an order column name is in upper case,
|
|
102
|
+
its associated column is sorted in ascending order, and a lower
|
|
103
|
+
case means sorting in descending order;
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
Condition Options:
|
|
107
|
+
- Option -a, for all recorded WEB file usage since 2005-01-01;
|
|
108
|
+
|
|
109
|
+
- Option -A, gives a row limit for querying;
|
|
110
|
+
|
|
111
|
+
- Option -c, for files read by users from given country codes;
|
|
112
|
+
|
|
113
|
+
- Option -d, for files read on given dates, in format YYYY-MM-DD;
|
|
114
|
+
|
|
115
|
+
- Option -D, for files read between two given dates, each date
|
|
116
|
+
is in format YYYY-MM-DD. Omit EndDate for no upper limit;
|
|
117
|
+
|
|
118
|
+
- Option -e, for data files requested by users with given email addresses.
|
|
119
|
+
To view the result properly, for email address zji@ucar.edu, the
|
|
120
|
+
character '@' need to be escaped as 'zji\@ucar.edu'
|
|
121
|
+
|
|
122
|
+
- Option -E, for data requested by users who have been notified
|
|
123
|
+
data update of a specified dataset between two given dates,
|
|
124
|
+
each date is in format YYYY-MM-DD. Omit EndNoticeDate for
|
|
125
|
+
no upper limit;
|
|
126
|
+
|
|
127
|
+
- Option -f, for data files of given file names;
|
|
128
|
+
|
|
129
|
+
- Option -F, for data files with names between MinFileName and
|
|
130
|
+
MaxFileName. Omit MaxFileName for no upper limit;
|
|
131
|
+
|
|
132
|
+
- Option -g, for WEB files read by users from given organization names;
|
|
133
|
+
|
|
134
|
+
- Option -h, works with Option -e to include historical user emails
|
|
135
|
+
registered before;
|
|
136
|
+
|
|
137
|
+
- Option -i, for WEB files read by users with given first names;
|
|
138
|
+
|
|
139
|
+
- Option -I, for data read by users who have been notified
|
|
140
|
+
data update of a specified dataset for given the email IDs;
|
|
141
|
+
|
|
142
|
+
- Option -l, for WEB files read by users with given last names;
|
|
143
|
+
|
|
144
|
+
- Option -m, for files read in given months, in format YYYY-MM;
|
|
145
|
+
|
|
146
|
+
- Option -M, for files read via given download methods;
|
|
147
|
+
|
|
148
|
+
- Option -N, for files for numbers of read by each group between
|
|
149
|
+
MinNumberRead and MaxNumberRead. Omit MaxNumberRead for no
|
|
150
|
+
upper limit;
|
|
151
|
+
|
|
152
|
+
- Option -o, for WEB files read by users from given organization types.
|
|
153
|
+
It defaults to -o ! DSS to exclude usage from DSS specialists;
|
|
154
|
+
Set it to ALL to include all orgnization types;
|
|
155
|
+
|
|
156
|
+
- Option -q, for data file requested in given quarters;
|
|
157
|
+
|
|
158
|
+
- Option -s, for from-flags the requests are initiated;
|
|
159
|
+
|
|
160
|
+
- Option -S, for login names of specialists who own the requests;
|
|
161
|
+
|
|
162
|
+
- Option -t, for data files associating to given dataset names;
|
|
163
|
+
|
|
164
|
+
- Option -T, for data files associating to datasets between
|
|
165
|
+
MinDataset and MaxDataset. Omit MaxDataset for no upper limit.
|
|
166
|
+
For example, -T d540000 d550009, for datasets numbers d540000-d550009;
|
|
167
|
+
|
|
168
|
+
- Option -v, for data files requested in given request indices;
|
|
169
|
+
|
|
170
|
+
- Option -y, for data files read in given years in format YYYY;
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
Miscellaneous Options:
|
|
174
|
+
- Option -n, view usage for requests from NCAR internal users who
|
|
175
|
+
submitted requests on command line;
|
|
176
|
+
|
|
177
|
+
- Option -w, view data usage info in simple format without totals;
|
|
178
|
+
|
|
179
|
+
- Option -z, include zero usages for specified fields;
|
|
180
|
+
|
|
181
|
+
- Option -L, use given delimiter for output, instead of defaulted spaces;
|
|
182
|
+
|
|
183
|
+
- Option -U, show file or data sizes in given unit SizeUnit [BKMG].
|
|
184
|
+
B - Byte, K - KiloBytes, M - MegaByte, and G - GigaByte;
|
|
185
|
+
|
|
186
|
+
- Option -H, use given report title to replace the default one;
|
|
187
|
+
|
|
188
|
+
- Option > OutputFilename, redirect output into an output file,
|
|
189
|
+
for example, rqstusage.out, instead of viewing on screen directly;
|
|
190
|
+
|
|
191
|
+
- Option | lp -d PrinterName, redirect output to printer of PrinterName.
|
|
192
|
+
Replace PrinterName with lj100 to print through DSS LaserJet printer.
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
For example:
|
|
196
|
+
To view annual data file usage requested in year 2005 with columns, INDEX(X),
|
|
197
|
+
EMAIL(E), ORGTYPE(O), #FILERQST(H), and MBYTESRQST(B); ordered by ORGTYPE as
|
|
198
|
+
ascending and MBYTESRQST(B) as descending; the command line should be:
|
|
199
|
+
|
|
200
|
+
viewrqstusage XEORB -y 2005 -O Ob
|
|
201
|
+
|
|
202
|
+
For usage of 'Y' series files only, out of the file usage gathered
|
|
203
|
+
above, the command line should be:
|
|
204
|
+
|
|
205
|
+
viewrqstusage XEOHB -y 2005 -f /DSS/Y% -O Ob
|
|
206
|
+
|
|
207
|
+
For usage by users not in Organization 'DDS', out of the file usage
|
|
208
|
+
gathered above, the command line should be:
|
|
209
|
+
|
|
210
|
+
viewrqstusage XEOHB -y 2005 -f /DSS/Y% -o ! DSS -O Ob
|
|
211
|
+
|
|
212
|
+
To redirect the previous output to a file named rqstusage.out:
|
|
213
|
+
|
|
214
|
+
viewrqstusage XEOHB -y 2005 -f /DSS/Y% -o ! DSS -O Ob > rqstusage.out
|
|
215
|
+
|
|
216
|
+
Then you can view the file or print it as a report.
|
|
217
|
+
|