rda-python-metrics 1.0.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of rda-python-metrics might be problematic. Click here for more details.
- rda_python_metrics/PgIPInfo.py +188 -0
- rda_python_metrics/PgView.py +782 -0
- rda_python_metrics/__init__.py +1 -0
- rda_python_metrics/fillawsusage.py +282 -0
- rda_python_metrics/fillawsusage.usg +17 -0
- rda_python_metrics/fillcodusage.py +247 -0
- rda_python_metrics/fillcodusage.usg +21 -0
- rda_python_metrics/fillcountry.py +79 -0
- rda_python_metrics/fillendtime.py +93 -0
- rda_python_metrics/fillglobususage.py +287 -0
- rda_python_metrics/fillglobususage.usg +17 -0
- rda_python_metrics/fillipinfo.py +185 -0
- rda_python_metrics/fillipinfo.usg +18 -0
- rda_python_metrics/filloneorder.py +155 -0
- rda_python_metrics/filloneorder.usg +41 -0
- rda_python_metrics/fillrdadb.py +151 -0
- rda_python_metrics/fillrdadb.usg +32 -0
- rda_python_metrics/filltdsusage.py +289 -0
- rda_python_metrics/filltdsusage.usg +17 -0
- rda_python_metrics/filluser.py +216 -0
- rda_python_metrics/filluser.usg +16 -0
- rda_python_metrics/logarch.py +359 -0
- rda_python_metrics/logarch.usg +27 -0
- rda_python_metrics/pgperson.py +72 -0
- rda_python_metrics/pgusername.py +50 -0
- rda_python_metrics/viewallusage.py +350 -0
- rda_python_metrics/viewallusage.usg +198 -0
- rda_python_metrics/viewcheckusage.py +289 -0
- rda_python_metrics/viewcheckusage.usg +185 -0
- rda_python_metrics/viewcodusage.py +314 -0
- rda_python_metrics/viewcodusage.usg +184 -0
- rda_python_metrics/viewordusage.py +340 -0
- rda_python_metrics/viewordusage.usg +224 -0
- rda_python_metrics/viewrqstusage.py +362 -0
- rda_python_metrics/viewrqstusage.usg +217 -0
- rda_python_metrics/viewtdsusage.py +323 -0
- rda_python_metrics/viewtdsusage.usg +191 -0
- rda_python_metrics/viewwebfile.py +294 -0
- rda_python_metrics/viewwebfile.usg +212 -0
- rda_python_metrics/viewwebusage.py +371 -0
- rda_python_metrics/viewwebusage.usg +211 -0
- rda_python_metrics-1.0.4.dist-info/METADATA +18 -0
- rda_python_metrics-1.0.4.dist-info/RECORD +47 -0
- rda_python_metrics-1.0.4.dist-info/WHEEL +5 -0
- rda_python_metrics-1.0.4.dist-info/entry_points.txt +22 -0
- rda_python_metrics-1.0.4.dist-info/licenses/LICENSE +21 -0
- rda_python_metrics-1.0.4.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,340 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
#
|
|
3
|
+
###############################################################################
|
|
4
|
+
#
|
|
5
|
+
# Title : viewordusage
|
|
6
|
+
# Author : Zaihua Ji, zji@ucar.edu
|
|
7
|
+
# Date : 03/15/2022
|
|
8
|
+
# Purpose : python program to view usage information for order data
|
|
9
|
+
#
|
|
10
|
+
# Work File : $DSSHOME/bin/viewordusage*
|
|
11
|
+
#
|
|
12
|
+
###############################################################################
|
|
13
|
+
#
|
|
14
|
+
import os
|
|
15
|
+
import re
|
|
16
|
+
import sys
|
|
17
|
+
import pgsyspath
|
|
18
|
+
import PgLOG
|
|
19
|
+
import PgUtil
|
|
20
|
+
import PgView
|
|
21
|
+
import PgDBI
|
|
22
|
+
|
|
23
|
+
VUSG = {
|
|
24
|
+
'SNMS' : "ABCDEFGHIJKLMNOPQRSTVWYZ", # all available short field names in FLDS
|
|
25
|
+
'OPTS' : 'AabcCdDeEghHijlLmMnNoOpqsStTUvwyz', # all available options, used for %params
|
|
26
|
+
'NOPT' : 'abhwz', # stand alone option without inputs
|
|
27
|
+
'ACND' : 'cdegijlmMnopqStvy', # available array condition options
|
|
28
|
+
'RCND' : 'DsNT', # available range condition options
|
|
29
|
+
'CNDS' : 'acdDegijlmMnNopqsStTvy', # condition options, ACND, RCND and 'a'
|
|
30
|
+
'HCND' : 'N', # condition options for having clause
|
|
31
|
+
'ECND' : 'my', # condition options need evaluating
|
|
32
|
+
'SFLD' : 'DEGIJLNOPTVW', # string fields, to be quoted in condition
|
|
33
|
+
'UFLD' : 'ILNOW', # string fields must be in upper case
|
|
34
|
+
'LFLD' : 'ETP' # string fields must be in lower case
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
# keys FLDS - short field names
|
|
38
|
+
# column 0 - column title showing in usage view
|
|
39
|
+
# column 1 - field name in format as shown in select clauses
|
|
40
|
+
# column 2 - field name shown in where condition query string
|
|
41
|
+
# column 3 - table name that the field belongs to
|
|
42
|
+
# column 4 - output field length, the longer one of data size and comlun title, determine
|
|
43
|
+
# dynamically if it is 0. Negative values indicate right justification
|
|
44
|
+
# column 5 - precision for floating point value if positive and show total value if not zero
|
|
45
|
+
# column 6 - field flag to indicate it is a group, distinct or sum field
|
|
46
|
+
FLDS = {
|
|
47
|
+
# SHRTNM COLUMNNANE FIELDNAME CNDNAME TBLNAM Size Prc Grp/Sum
|
|
48
|
+
'D' : ['DATE', "date_request", 'date_request','ousage', 10, 0, 'G'],
|
|
49
|
+
'E' : ['EMAIL', "wuser.email", 'wuser.email', 'wuser', 0, 0, 'G'],
|
|
50
|
+
'G' : ['ORGNAME', "org_name", 'org_name', 'wuser', 0, 0, 'G'],
|
|
51
|
+
'I' : ['FIRSTNAME', "fstname", 'fstname', 'wuser', 0, 0, 'G'],
|
|
52
|
+
'J' : ['PROJECT', "project", 'project', 'ousage', 7, 0, 'G'],
|
|
53
|
+
'K' : ['PAYMENT', "pay_method", 'pay_method', 'ousage', 0, 0, 'G'],
|
|
54
|
+
'L' : ['LASTNAME', "lstname", 'lstname', 'wuser', 0, 0, 'G'],
|
|
55
|
+
'M' : ['MONTH', PgDBI.fmtym("date_request"), 'date_request','ousage', 7, 0, 'G'],
|
|
56
|
+
'N' : ['COUNTRY', "country", 'country', 'wuser', 0, 0, 'G'],
|
|
57
|
+
'O' : ['ORGTYPE', "org_type", 'org_type', 'wuser', 5, 0, 'G'],
|
|
58
|
+
'P' : ['PROCESSBY', "dss_uname", 'dss_uname', 'ousage', 9, 0, 'G'],
|
|
59
|
+
'Q' : ['QUARTER', "quarter", 'quarter', 'ousage', 7, 0, 'G'],
|
|
60
|
+
'R' : ['DSTITLE', "search.datasets.title", 'search.datasets.title', 'search.datasets', 0, 0, 'G'],
|
|
61
|
+
'S' : ['OUTSIZE', "size_request", 'size_request','ousage', -9, -1, 'G'],
|
|
62
|
+
'T' : ['DATASET', "ousage.dsid", 'ousage.dsid', 'ousage', 7, 0, 'G'],
|
|
63
|
+
'V' : ['ORDERNO', "order_number", 'order_number','ousage', 0, 0, 'G'],
|
|
64
|
+
'W' : ['METHOD', "method", 'method', 'ousage', 6, 0, 'G'],
|
|
65
|
+
'Y' : ['YEAR', PgDBI.fmtyr("date_request"), 'date_request','ousage', 4, 0, 'G'],
|
|
66
|
+
'A' : ['DSCOUNT', "ousage.dsid", 'A', 'ousage', -7, -1, 'D'],
|
|
67
|
+
'B' : ['MBOUTPUT', "round(sum(size_request)/1000000, 4)", 'B', 'ousage', -14, 3, 'S'],
|
|
68
|
+
'C' : ['#UNIQUSER', "wuid_request", 'C', 'ousage', -9, -1, 'D'],
|
|
69
|
+
'F' : ['MBINPUT', "round(sum(size_input)/1000000, 4)", 'F', 'ousage', -14, 3, 'S'],
|
|
70
|
+
'H' : ['#ORDER', "count(order_number)", 'H', 'ousage', -8, -1, 'S'],
|
|
71
|
+
'Z' : ['COST()', "sum(amount)", 'Z', 'ousage', -12, -1, 'S'],
|
|
72
|
+
'X' : ['INDEX', "", 'X', '', -6, 0, ' ']
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
# keys %EXPAND - short field names allow zero usage
|
|
76
|
+
# column 0 - expand ID for group of fields
|
|
77
|
+
# column 1 - field name shown in where condition query string
|
|
78
|
+
# column 2 - field name in format as shown in select clauses
|
|
79
|
+
# column 3 - table name that the field belongs to
|
|
80
|
+
EXPAND = {
|
|
81
|
+
# SHRTNM EXPID CNDSTR FIELDNAME TBLNAM
|
|
82
|
+
'D' : ["TIME", "dDmy"],
|
|
83
|
+
'M' : ["TIME", "dDmy"],
|
|
84
|
+
'Q' : ["TIME", "dDmy"],
|
|
85
|
+
'Y' : ["TIME", "dDmy"],
|
|
86
|
+
|
|
87
|
+
'E' : ["USER", "eilgco", "email", "wuser"],
|
|
88
|
+
'I' : ["USER", "eilgco", "fstname", "wuser"],
|
|
89
|
+
'L' : ["USER", "eilgco", "lstname", "wuser"],
|
|
90
|
+
'G' : ["USER", "eilgco", "org_name", "wuser"],
|
|
91
|
+
'O' : ["USER", "eilgco", "org_type", "wuser"],
|
|
92
|
+
'N' : ["USER", "eilgco", "country", "wuser"],
|
|
93
|
+
|
|
94
|
+
'T' : ["DSID", "fFsStT", "dataset.dsid", "dataset"],
|
|
95
|
+
'R' : ["DSID", "fFsStT", "search.datasets.title", "search.datasets"],
|
|
96
|
+
'P' : ["DSID", "fFsStT", "specialist", "dsowner"],
|
|
97
|
+
|
|
98
|
+
'U' : ["OWNER", "u", "dssname", "ousage"],
|
|
99
|
+
|
|
100
|
+
'W' : ["METHOD", "M", "method", "ousage"],
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
# valid options for %params, a hash array of command line parameters
|
|
104
|
+
# a -- 1 to view all usage info available
|
|
105
|
+
# A -- number or records to return
|
|
106
|
+
# c -- array of specified country codes
|
|
107
|
+
# C -- a string of short field names for viewing usages
|
|
108
|
+
# d -- array of specified dates
|
|
109
|
+
# D -- dates range, array of 1 or 2 dates in format of YYYY-MM-DD
|
|
110
|
+
# e -- array of specified email addresses
|
|
111
|
+
# E -- use given date or date range for email notice of data update
|
|
112
|
+
# g -- array of specified orginization names
|
|
113
|
+
# h -- for give emails, include their histical emails registered before
|
|
114
|
+
# H -- a string of report title to replace the default one
|
|
115
|
+
# i -- array of specified first names
|
|
116
|
+
# j -- array of specified projects
|
|
117
|
+
# l -- array of specified last names
|
|
118
|
+
# L -- column delimiter for output
|
|
119
|
+
# m -- array of specified months
|
|
120
|
+
# M -- array of specified download methods
|
|
121
|
+
# n -- array of specified order numbers
|
|
122
|
+
# N -- number request range, arrage of 1 or 2 integers
|
|
123
|
+
# o -- array of specified orginization types
|
|
124
|
+
# O -- a string of short field names for sorting on
|
|
125
|
+
# p -- array of specified payment methods
|
|
126
|
+
# s -- output data size range, arrage of 1 or 2 sizes in unit of MByte
|
|
127
|
+
# S -- array of login names of specialists who processed the orders
|
|
128
|
+
# t -- array of specified dataset names
|
|
129
|
+
# T -- dataset range, array of 1 or 2 dataset names
|
|
130
|
+
# U -- use given unit for file or data sizes
|
|
131
|
+
# v -- aray of specified roder numbers
|
|
132
|
+
# w -- generate view without totals
|
|
133
|
+
# y -- array of specified years
|
|
134
|
+
# z -- generate view including entries without usage
|
|
135
|
+
params = {}
|
|
136
|
+
|
|
137
|
+
# relationship between parameter options and short field names, A option is not
|
|
138
|
+
# related to a field name if it is not in keys %SNS
|
|
139
|
+
SNS = {
|
|
140
|
+
'c' : 'N', 'd' : 'D', 'D' : 'D', 'e' : 'E', 'g' : 'G', 'i' : 'I', 'j' : 'J',
|
|
141
|
+
'l' : 'L', 'm' : 'M', 'M' : 'W', 'n' : 'V', 'N' : 'H', 'o' : 'O', 'p' : 'K',
|
|
142
|
+
'q' : 'Q', 's' : 'S', 'S' : 'P', 't' : 'T', 'T' : 'T', 'y' : 'Y'
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
tablenames = fieldnames = condition = ''
|
|
146
|
+
sfields = []
|
|
147
|
+
gfields = []
|
|
148
|
+
dfields = []
|
|
149
|
+
pgname = 'viewordusage'
|
|
150
|
+
|
|
151
|
+
#
|
|
152
|
+
# main function to run this program
|
|
153
|
+
#
|
|
154
|
+
def main():
|
|
155
|
+
|
|
156
|
+
PgDBI.view_dbinfo()
|
|
157
|
+
argv = sys.argv[1:]
|
|
158
|
+
inputs = []
|
|
159
|
+
option = 'C' # default option
|
|
160
|
+
|
|
161
|
+
for arg in argv:
|
|
162
|
+
if re.match(r'^-.*$', arg):
|
|
163
|
+
curopt = arg[1:2]
|
|
164
|
+
if curopt and VUSG['OPTS'].find(curopt) > -1:
|
|
165
|
+
if VUSG['NOPT'].find(option) > -1:
|
|
166
|
+
params[option] = 1
|
|
167
|
+
elif inputs:
|
|
168
|
+
params[option]= inputs # record input array
|
|
169
|
+
inputs = [] # empty input array
|
|
170
|
+
option = curopt # start a new option
|
|
171
|
+
else:
|
|
172
|
+
PgLOG.pglog(arg + ": Unknown Option", PgLOG.LGWNEX)
|
|
173
|
+
else:
|
|
174
|
+
val = arg
|
|
175
|
+
if val != '!':
|
|
176
|
+
if option == 's':
|
|
177
|
+
val = int(val)*1000000 # convert MBytes to Bytes
|
|
178
|
+
elif option in SNS:
|
|
179
|
+
sfld = SNS[option]
|
|
180
|
+
if VUSG['SFLD'].find(sfld) > -1:
|
|
181
|
+
if VUSG['UFLD'].find(sfld) > -1:
|
|
182
|
+
val = arg.upper() # in case not in upper case
|
|
183
|
+
elif VUSG['LFLD'].find(sfld) > -1:
|
|
184
|
+
val = arg.lower() # in case not in lower case
|
|
185
|
+
if option == 'c':
|
|
186
|
+
val = PgView.get_country_name(val)
|
|
187
|
+
elif option == 't' or option == 'T':
|
|
188
|
+
val = PgUtil.format_dataset_id(val) # add 'ds' if only numbers
|
|
189
|
+
val = "'{}'".format(val)
|
|
190
|
+
inputs.append(val)
|
|
191
|
+
|
|
192
|
+
# record the last option
|
|
193
|
+
if VUSG['NOPT'].find(option) > -1:
|
|
194
|
+
params[option] = 1
|
|
195
|
+
elif inputs:
|
|
196
|
+
params[option] = inputs # record input array
|
|
197
|
+
|
|
198
|
+
if not params:
|
|
199
|
+
PgLOG.show_usage(pgname)
|
|
200
|
+
else:
|
|
201
|
+
check_enough_options()
|
|
202
|
+
|
|
203
|
+
if 'o' not in params:
|
|
204
|
+
if 'e' not in params:
|
|
205
|
+
params['o'] = ['!', "'DSS'"] # default to exclude 'DSS' for organization
|
|
206
|
+
elif params['o'][0] == "'ALL'":
|
|
207
|
+
del params['o']
|
|
208
|
+
|
|
209
|
+
usgtable = "ousage"
|
|
210
|
+
build_query_strings(usgtable) # build tablenames, fieldnames, and conditions
|
|
211
|
+
records = PgDBI.pgmget(tablenames, fieldnames, condition, PgLOG.UCLWEX)
|
|
212
|
+
if not records: PgLOG.pglog("No Usage Found For Given Conditions", PgLOG.LGWNEX)
|
|
213
|
+
totals = None if 'w' in params else {}
|
|
214
|
+
if dfields or totals != None:
|
|
215
|
+
records = PgView.compact_hash_groups(records, gfields, sfields, dfields, totals)
|
|
216
|
+
if 'z' in params: records = expand_records(records)
|
|
217
|
+
ostr = params['O'][0] if 'O' in params else params['C'][0]
|
|
218
|
+
records = PgView.order_records(records, ostr.replace('X', ''))
|
|
219
|
+
PgView.simple_output(params, FLDS, records, totals)
|
|
220
|
+
|
|
221
|
+
PgLOG.pgexit(0)
|
|
222
|
+
|
|
223
|
+
#
|
|
224
|
+
# cehck if enough information entered on command line for generate view/report, exit if not
|
|
225
|
+
#
|
|
226
|
+
def check_enough_options():
|
|
227
|
+
|
|
228
|
+
cols = params['C'][0] if 'C' in params else 'X'
|
|
229
|
+
if cols == 'X': PgLOG.pglog("{}: miss field names '{}'".format(pgname, VUSG['SNMS']), PgLOG.LGWNEX)
|
|
230
|
+
|
|
231
|
+
if cols.find('Q') > -1 and cols.find('Y') < 0: # add Y if Q included
|
|
232
|
+
cols = re.sub('Q', 'YQ', cols)
|
|
233
|
+
params['C'][0] = cols
|
|
234
|
+
|
|
235
|
+
for sn in cols:
|
|
236
|
+
if sn == 'X': continue # do not process INDEX field
|
|
237
|
+
if VUSG['SNMS'].find(sn) < 0:
|
|
238
|
+
PgLOG.pglog("{}: Field {} must be in field names '{}X'".format(pgname, sn, VUSG['SNMS']), PgLOG.LGWNEX)
|
|
239
|
+
if 'z' not in params or sn in EXPAND: continue
|
|
240
|
+
fld = FLDS[sn]
|
|
241
|
+
if fld[6] != 'G': continue
|
|
242
|
+
PgLOG.pglog("{}: cannot show zero usage for unexpandable field {} - {}".formt(pgname, sn, fld[0]), PgLOG.LGWNEX)
|
|
243
|
+
|
|
244
|
+
if 'E' in params:
|
|
245
|
+
if 'z' in params:
|
|
246
|
+
PgLOG.pglog(pgname + ": option -z and -E can not be present at the same time", PgLOG.LGWNEX)
|
|
247
|
+
elif 't' not in params or len(params['t']) > 1:
|
|
248
|
+
PgLOG.pglog(pgname + ": specify one dataset for viewing usage of notified users", PgLOG.LGWNEX)
|
|
249
|
+
|
|
250
|
+
for opt in params:
|
|
251
|
+
if VUSG['CNDS'].find(opt) > -1: return
|
|
252
|
+
PgLOG.pglog("{}: miss condition options '{}'".format(pgname, VUSG['CNDS']), PgLOG.LGWNEX)
|
|
253
|
+
|
|
254
|
+
#
|
|
255
|
+
# process parameter options to build all query strings
|
|
256
|
+
# global variables are used directly and nothing passes in and returns back
|
|
257
|
+
#
|
|
258
|
+
def build_query_strings(usgtable):
|
|
259
|
+
|
|
260
|
+
# initialize query strings
|
|
261
|
+
global condition, fieldnames, tablenames
|
|
262
|
+
joins = having = groupnames = ''
|
|
263
|
+
tablenames = usgtable
|
|
264
|
+
cols = params['C'][0]
|
|
265
|
+
|
|
266
|
+
if 'U' in params: # reset units for file and read sizes
|
|
267
|
+
if cols.find('B') > -1: FLDS['B'] = PgView.set_data_unit(FLDS['B'], params['U'][0], "sum(size_request)")
|
|
268
|
+
if cols.find('F') > -1: FLDS['F'] = PgView.set_data_unit(FLDS['F'], params['U'][0], "sum(size_input)")
|
|
269
|
+
|
|
270
|
+
if 'e' in params and 'h' in params: params['e'] = PgView.include_historic_emails(params['e'], 3)
|
|
271
|
+
|
|
272
|
+
for opt in params:
|
|
273
|
+
if opt == 'C': # build field, table and group names
|
|
274
|
+
for sn in cols:
|
|
275
|
+
if sn == 'X': continue # do not process INDEX field
|
|
276
|
+
fld = FLDS[sn]
|
|
277
|
+
if fieldnames: fieldnames += ', '
|
|
278
|
+
fieldnames += "{} {}".format(fld[1], sn) # add to field name string
|
|
279
|
+
(tablenames, joins) = PgView.join_query_tables(fld[3], tablenames, joins, usgtable)
|
|
280
|
+
if fld[6] == 'S':
|
|
281
|
+
sfields.append(sn)
|
|
282
|
+
else:
|
|
283
|
+
if groupnames: groupnames += ', '
|
|
284
|
+
groupnames += sn # add to group name string
|
|
285
|
+
if fld[6] == 'D':
|
|
286
|
+
dfields.append(sn)
|
|
287
|
+
else:
|
|
288
|
+
gfields.append(sn)
|
|
289
|
+
elif opt == 'O':
|
|
290
|
+
continue # order records later
|
|
291
|
+
elif VUSG['CNDS'].find(opt) > -1:
|
|
292
|
+
if VUSG['NOPT'].find(opt) > -1: continue
|
|
293
|
+
sn = SNS[opt]
|
|
294
|
+
fld = FLDS[sn]
|
|
295
|
+
# build having and where conditon strings
|
|
296
|
+
cnd = PgView.get_view_condition(opt, sn, fld, params, VUSG)
|
|
297
|
+
if cnd:
|
|
298
|
+
if VUSG['HCND'].find(opt) > -1:
|
|
299
|
+
if having: having += ' AND '
|
|
300
|
+
having += cnd
|
|
301
|
+
else:
|
|
302
|
+
if condition: condition += ' AND '
|
|
303
|
+
condition += cnd
|
|
304
|
+
(tablenames, joins) = PgView.join_query_tables(fld[3], tablenames, joins, usgtable)
|
|
305
|
+
|
|
306
|
+
# append joins, group by, order by, and having strings to condition string
|
|
307
|
+
if 'E' in params:
|
|
308
|
+
(tablenames, joins) = PgView.join_query_tables("emreceive", tablenames, joins, usgtable)
|
|
309
|
+
if joins:
|
|
310
|
+
if condition:
|
|
311
|
+
condition = "{} AND {}".format(joins, condition)
|
|
312
|
+
else:
|
|
313
|
+
condition = joins
|
|
314
|
+
if 'E' in params:
|
|
315
|
+
condition += PgView.notice_condition(params['E'], None, params['t'][0])
|
|
316
|
+
if groupnames and sfields: condition += " GROUP BY " + groupnames
|
|
317
|
+
if having: condition += " HAVING " + having
|
|
318
|
+
|
|
319
|
+
def expand_records(records):
|
|
320
|
+
|
|
321
|
+
recs = PgView.expand_query("TIME", records, params, EXPAND)
|
|
322
|
+
|
|
323
|
+
trecs = PgView.expand_query("USER", records, params, EXPAND, VUSG, SNS, FLDS)
|
|
324
|
+
if trecs: PgUtil.crosshash(recs, trecs)
|
|
325
|
+
|
|
326
|
+
trecs = PgView.expand_query("DSID", records, params, EXPAND, VUSG, SNS, FLDS)
|
|
327
|
+
if trecs: recs = PgUtil.crosshash(recs, trecs)
|
|
328
|
+
|
|
329
|
+
trecs = PgView.expand_query("OWNER", records, params, EXPAND, VUSG, SNS, FLDS)
|
|
330
|
+
if trecs: recs = PgUtil.crosshash(recs, trecs)
|
|
331
|
+
|
|
332
|
+
trecs = PgView.expand_query("METHOD", records, params, EXPAND, VUSG, SNS, FLDS)
|
|
333
|
+
if trecs: recs = PgUtil.crosshash(recs, trecs)
|
|
334
|
+
|
|
335
|
+
return PgUtil.joinhash(records, recs, 0, 1)
|
|
336
|
+
|
|
337
|
+
#
|
|
338
|
+
# call main() to start program
|
|
339
|
+
#
|
|
340
|
+
if __name__ == "__main__": main()
|
|
@@ -0,0 +1,224 @@
|
|
|
1
|
+
|
|
2
|
+
View usage information of Order data from order information
|
|
3
|
+
stored in MySQL database 'RDADB'.
|
|
4
|
+
|
|
5
|
+
Usage: viewordusage [-C] ColumnNames [-O OrderColumnNames] [-a] \
|
|
6
|
+
[-A RowLimit] [-c CountryCodes] [-d DataRequestDates] \
|
|
7
|
+
[-D StartRquestDate [EndRequestDate]] \
|
|
8
|
+
[-e EMailList] -h [-E StartNoticeDate [EndNoticeDate]] \
|
|
9
|
+
[-g OrganizationNames] \
|
|
10
|
+
[-i FirstNamelist] [-j Projects] [-l LastNameList] \
|
|
11
|
+
[-m MonthList] [-M MethodList] [-n OrderNumberList] \
|
|
12
|
+
[-o OrganizationTypes] [-p PaymentMethods] \
|
|
13
|
+
[-s MinInputSize [MaxInputSize]] \
|
|
14
|
+
[-S SpecialistLoginNames] [-t DatasetList] \
|
|
15
|
+
[-T MinDataset [MaxDataset]] [-u ProcessSpecialists] \
|
|
16
|
+
[-v OrderNumbers] [-y YearList] \
|
|
17
|
+
[-H Title] [-L Delimiter] [-w] [-z] \
|
|
18
|
+
[> OutputFileName] [| lp -d PrinterName]
|
|
19
|
+
|
|
20
|
+
Specify [-C] ColumnNames, refer to Option -C section for detail
|
|
21
|
+
description, and choose at least one of the condition options, -a, -c,
|
|
22
|
+
-d, -D, -e, -g, -i, -j, -l, -m, -M, -N, -n, -o, -p, -s, -S, -t, -T, -u
|
|
23
|
+
-v and -y, to run this application.
|
|
24
|
+
|
|
25
|
+
For all condition options, except option -a, an '!' sign can be added
|
|
26
|
+
between an option flag and its option values to get an excluding
|
|
27
|
+
condition. For example, choose '-o ! OrganizationList' to gather order
|
|
28
|
+
data usage by users from organizations other than the ones given in
|
|
29
|
+
OrganizationList. Refer to the example given at the end of this help
|
|
30
|
+
document for how to select excluding condition.
|
|
31
|
+
|
|
32
|
+
String condition options, -c, -e, -g, -i, -j, -l, -n, -o, -p, -S, -t, -u
|
|
33
|
+
and -v, allow wildcard inputs. '%' matches any number of characters and '_'
|
|
34
|
+
matches any one character. Refer to the example given at the end of
|
|
35
|
+
this help document for how to use wildcard for string condition options.
|
|
36
|
+
|
|
37
|
+
Output of this application is defaulted to page format with a page
|
|
38
|
+
header on each page. A page header includes main title, sub titles and
|
|
39
|
+
column titles according to which column names and options are selected,
|
|
40
|
+
as well as page number and report date. If the output is used directly
|
|
41
|
+
for input of other applications, add option -w to remove page header
|
|
42
|
+
and show only the column titles and the order usage information.
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
Column Options:
|
|
46
|
+
- Option -C, the ColumnNames must be present to run this application.
|
|
47
|
+
The flag -C can be omitted if it is the first parameter option on
|
|
48
|
+
the command line. The ColumnNames is a string that includes column
|
|
49
|
+
names listed below:
|
|
50
|
+
|
|
51
|
+
COLUMN - COLUMN - COLUMN
|
|
52
|
+
NAME - TITLE - DESCRIPTION
|
|
53
|
+
GroupColumns:
|
|
54
|
+
D*- DATE - format as YYYY-MM-DD, for example 2004-04-25
|
|
55
|
+
E*- EMAIL - user email address
|
|
56
|
+
G*- ORGNAME - organization name
|
|
57
|
+
I*- FIRSTNAME - user first name
|
|
58
|
+
J - PROJECT - project id, i.e, icoads, era40, tigge
|
|
59
|
+
K - PAYMENT - payment methods (free, invce, etc.)
|
|
60
|
+
L*- LASTNAME - user last name
|
|
61
|
+
M*- MONTH - format as YYYY-MM, for example 2004-04
|
|
62
|
+
N*- ORDERNO - order number
|
|
63
|
+
O*- ORGTYPE - organization types (DSS, NCAR, UNIV and OTHER)
|
|
64
|
+
P - PROCESSBY - specialist login names who own the orders
|
|
65
|
+
Q*- COUNTRY - user country names
|
|
66
|
+
R*- DSTITLE - dataset titles
|
|
67
|
+
S*- OUTSIZE - requested data size in MB
|
|
68
|
+
T*- DATASET - format as dsnnn.n, for example d540001
|
|
69
|
+
V - ORDERNO - order numbers
|
|
70
|
+
W*- METHOD - method of order data (ftp, tape, etc.)
|
|
71
|
+
Y*- YEAR - format as YYYY, for example 2004
|
|
72
|
+
|
|
73
|
+
* - field names can processed with zero usages
|
|
74
|
+
SummaryColumns:
|
|
75
|
+
A - DSCOUNT - number of datasets in given GroupColumns
|
|
76
|
+
B - MBOUTPUT - datasizes, in MB, requested by given
|
|
77
|
+
GroupColumns
|
|
78
|
+
C - #UNIQUSER - number of users in in given GroupColumns
|
|
79
|
+
F - MBINPUT - datasizes, in MB, input for given GroupColumns
|
|
80
|
+
H - #ORDER - number of orders by given GroupColumns
|
|
81
|
+
Z - COST($) - data costs, in dollar, requested by given
|
|
82
|
+
GroupColumns
|
|
83
|
+
IndexColumn:
|
|
84
|
+
X - INDEX - index of line, it should be the first column
|
|
85
|
+
|
|
86
|
+
The column names are used to build up string of ColumnNames, while
|
|
87
|
+
their associated column titles are shown in view/report output of
|
|
88
|
+
this application. The display order of the column titles is
|
|
89
|
+
determined by the order of the column names in the ColumnNames
|
|
90
|
+
string. At least one of the group and summary columns must be
|
|
91
|
+
selected, in the ColumnNames string, to generate order usage
|
|
92
|
+
view/report;
|
|
93
|
+
|
|
94
|
+
For example, choose '-C EMB' to display column titles of EMAIL,
|
|
95
|
+
MONTH and MBYTEREAD, in the first, second and third columns
|
|
96
|
+
respectively, for numbers of MBytes of data read by each user
|
|
97
|
+
in each month;
|
|
98
|
+
|
|
99
|
+
- Option -O, sort order usage information in ascending or descending
|
|
100
|
+
order based on the column names specified in OrderColumnNames
|
|
101
|
+
string. These order column names must be in the selected
|
|
102
|
+
[-C] ColumnNames string. If an order column name is in upper case,
|
|
103
|
+
its associated column is sorted in ascending order, and a lower
|
|
104
|
+
case means sorting in descending order;
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
Condition Options:
|
|
108
|
+
- Option -a, for all order usage in table 'ousage';
|
|
109
|
+
|
|
110
|
+
- Option -A, gives a row limit for querying;
|
|
111
|
+
|
|
112
|
+
- Option -c, for order data requested by users from given countries;
|
|
113
|
+
|
|
114
|
+
- Option -d, for order data requested on given dates, in format YYYY-MM-DD;
|
|
115
|
+
|
|
116
|
+
- Option -D, for order data requested between two given dates, each date
|
|
117
|
+
is in format YYYY-MM-DD. Omit EndDate for no upper limit;
|
|
118
|
+
|
|
119
|
+
- Option -e, for order data requested by users wth given email addresses;
|
|
120
|
+
|
|
121
|
+
- Option -E, for data read by users who have been notified
|
|
122
|
+
data update of a specified dataset between two given dates,
|
|
123
|
+
each date is in format YYYY-MM-DD. Omit EndNoticeDate for
|
|
124
|
+
no upper limit;
|
|
125
|
+
|
|
126
|
+
- Option -g, for WEB files read by users from given organization names;
|
|
127
|
+
|
|
128
|
+
- Option -h, works with Option -e to include historical user emails
|
|
129
|
+
registered before;
|
|
130
|
+
|
|
131
|
+
- Option -i, for order data requested by users with given first names;
|
|
132
|
+
|
|
133
|
+
- Option -j, for order data requested for specified projects;
|
|
134
|
+
|
|
135
|
+
- Option -l, for order data requested by users with given last names;
|
|
136
|
+
|
|
137
|
+
- Option -m, for data requested in given months, in format YYYY-MM;
|
|
138
|
+
|
|
139
|
+
- Option -M, for data requested via given medium methods;
|
|
140
|
+
|
|
141
|
+
- Option -n, for order data of given order numbers;
|
|
142
|
+
|
|
143
|
+
- Option -N, for files for numbers of read by each group between
|
|
144
|
+
MinNumberRead and MaxNumberRead. Omit MaxNumberRead for no
|
|
145
|
+
upper limit;
|
|
146
|
+
|
|
147
|
+
- Option -o, for order data requested by users from given organization
|
|
148
|
+
types. It defaults to -o ! DSS to exclude usage from DSS specialists;
|
|
149
|
+
Set it to ALL to include all orgnization types;
|
|
150
|
+
|
|
151
|
+
- Option -p, for order data via given payment methods;
|
|
152
|
+
|
|
153
|
+
- Option -s, for output data with sizes, in unit of MByte, between
|
|
154
|
+
MinInputSize and MaxInputSize. Omit MaxInputSize for no upper limit;
|
|
155
|
+
|
|
156
|
+
- Option -S, for login names of specialists who own the orders;
|
|
157
|
+
|
|
158
|
+
- Option -t, for order data associating to given dataset names;
|
|
159
|
+
|
|
160
|
+
- Option -T, for order data associating to datasets between
|
|
161
|
+
MinDataset and MaxDataset. Omit MaxDataset for no upper limit.
|
|
162
|
+
For example, -T d540000 d550009, for datasets numbers d540000-d550009;
|
|
163
|
+
|
|
164
|
+
- Option -v, for order data processed by given order numbers;
|
|
165
|
+
|
|
166
|
+
- Option -y, for order data read in given years in format YYYY;
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
Miscellaneous Options:
|
|
170
|
+
- Option -w, view order data in simple format without totals;
|
|
171
|
+
|
|
172
|
+
- Option -z, include users and/or datasets without usage;
|
|
173
|
+
|
|
174
|
+
- Option -H, use given report title to replace the default one;
|
|
175
|
+
|
|
176
|
+
- Option -L, use given delimiter for output, instead of defaulted spaces;
|
|
177
|
+
|
|
178
|
+
- Option -U, show file or data sizes in given unit SizeUnit [BKMG].
|
|
179
|
+
B - Byte, K - KiloBytes, M - MegaByte, and G - GigaByte;
|
|
180
|
+
|
|
181
|
+
- Option > OutputFilename, redirect output into an output file,
|
|
182
|
+
for example, ordusage.out, instead of viewing on screen directly;
|
|
183
|
+
|
|
184
|
+
- Option | lp -d PrinterName, redirect output to printer of PrinterName.
|
|
185
|
+
Replace PrinterName with lj100 to print through DSS LaserJet printer.
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
For example:
|
|
189
|
+
To view annual order data usage in year 2005 with columns, INDEX(X),
|
|
190
|
+
EMAIL(E), ORGTYPE(O), #REQUEST(H), and MBREQUEST(B); ordered by ORGTYPE as
|
|
191
|
+
ascending and MBYTESREAD(B) as descending; the command line should be:
|
|
192
|
+
|
|
193
|
+
viewordusage XEOHB -y 2005 -O Ob
|
|
194
|
+
|
|
195
|
+
For usage by users not in Organization 'DDS', out of the file usage
|
|
196
|
+
- Option -H, use given report title to replace the default one;
|
|
197
|
+
|
|
198
|
+
- Option -L, use given delimiter for output, instead of defaulted spaces;
|
|
199
|
+
|
|
200
|
+
- Option > OutputFilename, redirect output into an output file,
|
|
201
|
+
for example, ordusage.out, instead of viewing on screen directly;
|
|
202
|
+
|
|
203
|
+
- Option | lp -d PrinterName, redirect output to printer of PrinterName.
|
|
204
|
+
Replace PrinterName with lj100 to print through DSS LaserJet printer.
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
For example:
|
|
208
|
+
To view annual order data usage in year 2005 with columns, INDEX(X),
|
|
209
|
+
EMAIL(E), ORGTYPE(O), #REQUEST(H), and MBREQUEST(B); ordered by ORGTYPE as
|
|
210
|
+
ascending and MBYTESREAD(B) as descending; the command line should be:
|
|
211
|
+
|
|
212
|
+
viewordusage XEOHB -y 2005 -O Ob
|
|
213
|
+
|
|
214
|
+
For usage by users not in Organization 'DDS', out of the file usage
|
|
215
|
+
gathered above, the command line should be:
|
|
216
|
+
|
|
217
|
+
viewordusage XEOHB -y 2005 -o ! DSS -O Ob
|
|
218
|
+
|
|
219
|
+
To redirect the previous output to a file named ordusage.out:
|
|
220
|
+
|
|
221
|
+
viewordusage XEOHB -y 2005 ! DSS -O Ob > ordusage.out
|
|
222
|
+
|
|
223
|
+
Then you can view the file or print it as a report.
|
|
224
|
+
|