rda-python-metrics 1.0.41__py3-none-any.whl → 1.0.42__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of rda-python-metrics might be problematic. Click here for more details.
- rda_python_metrics/fillgdexusage.py +51 -37
- rda_python_metrics/fillzenodousage.py +408 -0
- {rda_python_metrics-1.0.41.dist-info → rda_python_metrics-1.0.42.dist-info}/METADATA +1 -1
- {rda_python_metrics-1.0.41.dist-info → rda_python_metrics-1.0.42.dist-info}/RECORD +8 -7
- {rda_python_metrics-1.0.41.dist-info → rda_python_metrics-1.0.42.dist-info}/entry_points.txt +1 -0
- {rda_python_metrics-1.0.41.dist-info → rda_python_metrics-1.0.42.dist-info}/WHEEL +0 -0
- {rda_python_metrics-1.0.41.dist-info → rda_python_metrics-1.0.42.dist-info}/licenses/LICENSE +0 -0
- {rda_python_metrics-1.0.41.dist-info → rda_python_metrics-1.0.42.dist-info}/top_level.txt +0 -0
|
@@ -30,44 +30,58 @@ USAGE = {
|
|
|
30
30
|
}
|
|
31
31
|
|
|
32
32
|
DSIDS = {
|
|
33
|
+
"14_schuster" : "d583138",
|
|
34
|
+
"384_duvivier" : "d583139",
|
|
35
|
+
"model_data_for_17_march_2015_storm_event" : "d583140",
|
|
36
|
+
"326_qingyu" : "d583141",
|
|
37
|
+
"354_fasullo" : "d583142",
|
|
38
|
+
"157_gaubert" : "d583143",
|
|
39
|
+
"370_gaubert" : "d583144",
|
|
40
|
+
"459" : "d583145",
|
|
41
|
+
"108_kristenk" : "d583146",
|
|
42
|
+
"82_abaker" : "d583147",
|
|
43
|
+
"215_grabow" : "d583148",
|
|
44
|
+
"222_duvivier" : "d583149",
|
|
45
|
+
"366_rberrios" : "d583150",
|
|
46
|
+
"210_dxyang" : "d583100",
|
|
47
|
+
"238_junkyung" : "d583101",
|
|
48
|
+
"361_islas" : "d583102",
|
|
49
|
+
"448" : "d583103",
|
|
50
|
+
"187_jaredlee" : "d583104",
|
|
51
|
+
"273_kdagon" : "d583105",
|
|
52
|
+
"149_liuh" : "d583106",
|
|
53
|
+
"279_schwantes" : "d583107",
|
|
54
|
+
"60_pinto" : "d583108",
|
|
55
|
+
"WACC_Dependence_on_Solar_Activity" : "d583109",
|
|
56
|
+
"275_domingom" : "d583110",
|
|
57
|
+
"445" : "d583111",
|
|
58
|
+
"TIE-GCM_2017_Solar_Storm" : "d583112",
|
|
59
|
+
"123_andrew" : "d583113",
|
|
60
|
+
"453" : "d583114",
|
|
61
|
+
"367_mclong" : "d583115",
|
|
62
|
+
"258_jiangzhu" : "d583116",
|
|
63
|
+
"136_hannay" : "d583117",
|
|
64
|
+
"173B_dcherian" : "d583118",
|
|
65
|
+
"407_islas" : "d583119",
|
|
66
|
+
"189_islas" : "d583120",
|
|
67
|
+
"289_shields" : "d583121",
|
|
68
|
+
"415_mcinerney" : "d583122",
|
|
69
|
+
"151_dll" : "d583123",
|
|
70
|
+
"460" : "d583124",
|
|
71
|
+
"117_phamkh" : "d583125",
|
|
72
|
+
"109_erichend" : "d583126",
|
|
73
|
+
"412" : "d583127",
|
|
74
|
+
"390_knocasio" : "d583128",
|
|
75
|
+
"382_kumar" : "d583129",
|
|
76
|
+
"470" : "d583130",
|
|
77
|
+
"371_abaker" : "d583131",
|
|
78
|
+
"CESM-cocco_CO2_experiments" : "d583132",
|
|
79
|
+
"fuel_moisture_content" : "d583133",
|
|
80
|
+
"81_rberrios" : "d583134",
|
|
81
|
+
"286_knocasio" : "d583135",
|
|
82
|
+
"383_stebbins" : "d583136",
|
|
83
|
+
"256_tilmes" : "d583137",
|
|
33
84
|
'371_abaker' : 'd583131',
|
|
34
|
-
'CESM-cocco_CO2_experiments' : 'd583132',
|
|
35
|
-
'256_tilmes' : 'd583137',
|
|
36
|
-
'fuel_moisture_content' : 'd583133',
|
|
37
|
-
'81_rberrios' : 'd583134',
|
|
38
|
-
'210_dxyang' : 'd583100',
|
|
39
|
-
'286_knocasio' : 'd583135',
|
|
40
|
-
'238_junkyung' : 'd583101',
|
|
41
|
-
'361_islas' : 'd583102',
|
|
42
|
-
'448' : 'd583103',
|
|
43
|
-
'383_stebbins' : 'd583136',
|
|
44
|
-
'187_jaredlee' : 'd583104',
|
|
45
|
-
'273_kdagon' : 'd583105',
|
|
46
|
-
'149_liuh' : 'd583106',
|
|
47
|
-
'279_schwantes' : 'd583107',
|
|
48
|
-
'60_pinto' : 'd583108',
|
|
49
|
-
'WACC_Dependence_on_Solar_Activity' : 'd583109',
|
|
50
|
-
'275_domingom' : 'd583110',
|
|
51
|
-
'445' : 'd583111',
|
|
52
|
-
'TIE-GCM_2017_Solar_Storm' : 'd583112',
|
|
53
|
-
'123_andrew' : 'd583113',
|
|
54
|
-
'453' : 'd583114',
|
|
55
|
-
'367_mclong' : 'd583115',
|
|
56
|
-
'258_jiangzhu' : 'd583116',
|
|
57
|
-
'136_hannay' : 'd583117',
|
|
58
|
-
'173B_dcherian' : 'd583118',
|
|
59
|
-
'407_islas' : 'd583119',
|
|
60
|
-
'189_islas' : 'd583120',
|
|
61
|
-
'289_shields' : 'd583121',
|
|
62
|
-
'415_mcinerney' : 'd583122',
|
|
63
|
-
'151_dll' : 'd583123',
|
|
64
|
-
'460' : 'd583124',
|
|
65
|
-
'117_phamkh' : 'd583125',
|
|
66
|
-
'109_erichend' : 'd583126',
|
|
67
|
-
'412' : 'd583127',
|
|
68
|
-
'390_knocasio' : 'd583128',
|
|
69
|
-
'382_kumar' : 'd583129',
|
|
70
|
-
'470' : 'd583130',
|
|
71
85
|
# icarus
|
|
72
86
|
'icarus.experiment.403' : 'd789010',
|
|
73
87
|
'icarus.experiment.404' : 'd789011',
|
|
@@ -0,0 +1,408 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
#
|
|
3
|
+
###############################################################################
|
|
4
|
+
#
|
|
5
|
+
# Title : fillgdexusage
|
|
6
|
+
# Author : Zaihua Ji, zji@ucar.edu
|
|
7
|
+
# Date : 2025-04-14
|
|
8
|
+
# Purpose : python program to retrieve info from ZENODO Postgres database for GDS
|
|
9
|
+
# file accesses and backup fill table tdsusage in PostgreSQL database dssdb.
|
|
10
|
+
#
|
|
11
|
+
# Github : https://github.com/NCAR/rda-python-metrics.git
|
|
12
|
+
#
|
|
13
|
+
###############################################################################
|
|
14
|
+
#
|
|
15
|
+
import sys
|
|
16
|
+
import re
|
|
17
|
+
import glob
|
|
18
|
+
from os import path as op
|
|
19
|
+
from time import time as tm
|
|
20
|
+
from rda_python_common import PgLOG
|
|
21
|
+
from rda_python_common import PgUtil
|
|
22
|
+
from rda_python_common import PgFile
|
|
23
|
+
from rda_python_common import PgDBI
|
|
24
|
+
from rda_python_common import PgSplit
|
|
25
|
+
from . import PgIPInfo
|
|
26
|
+
|
|
27
|
+
USAGE = {
|
|
28
|
+
'ZNDTBL' : "gdexzenodo",
|
|
29
|
+
'CDATE' : PgUtil.curdate(),
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
DSIDS = {
|
|
33
|
+
"498" : "15760289",
|
|
34
|
+
"386_ahijevych" : "15501153",
|
|
35
|
+
"66_oclifton" : "15501155",
|
|
36
|
+
"196_mayernik" : "15501157",
|
|
37
|
+
"83_heyms1" : "15501159",
|
|
38
|
+
"405_qwu" : "15501163",
|
|
39
|
+
"WACCMX_DART_Hindcast" : "15501165",
|
|
40
|
+
"234_buchholz" : "15501167",
|
|
41
|
+
"365_barthm" : "15501171",
|
|
42
|
+
"Polar_Cap_HIWIND_observation_and_TIEGCM_simulations" : "15501173",
|
|
43
|
+
"266_xuguang" : "15501175",
|
|
44
|
+
"277_lqian" : "15501177",
|
|
45
|
+
"Ocean_MHT_Values" : "15501181",
|
|
46
|
+
"92_xuguang" : "15501184",
|
|
47
|
+
"118_nickp" : "15501186",
|
|
48
|
+
"398_kunwu" : "15501188",
|
|
49
|
+
"107_siyuan" : "15501190",
|
|
50
|
+
"274_gaubert" : "15501192",
|
|
51
|
+
"301_ldong" : "15501194",
|
|
52
|
+
"GRL_Kurowski_al_2019" : "15501196",
|
|
53
|
+
"112_fjudt" : "15501201",
|
|
54
|
+
"14_schuster" : "15501203",
|
|
55
|
+
"102_abaker" : "15501205",
|
|
56
|
+
"402_qwu" : "15501207",
|
|
57
|
+
"324_lqian" : "15501209",
|
|
58
|
+
"Response_of_the_F2-region_Ionosphere_to_the_21_August_2017_Solar_Eclipse_at_Millstone_Hill" : "15501211",
|
|
59
|
+
"362_kunwu" : "15501213",
|
|
60
|
+
"LFM-TIEGCM-RCM_Simulation" : "15501216",
|
|
61
|
+
"140_xuguang" : "15501221",
|
|
62
|
+
"240_ldong" : "15501223",
|
|
63
|
+
"93_sallyz" : "15501229",
|
|
64
|
+
"452" : "15501231",
|
|
65
|
+
"318_ldong" : "15501233",
|
|
66
|
+
"161_ldong" : "15501237",
|
|
67
|
+
"288_voemel" : "15501239",
|
|
68
|
+
"160_xuguang" : "15501241",
|
|
69
|
+
"CO_Flux_Inversion_Attribution" : "15501243",
|
|
70
|
+
"404_buchholz" : "15501245",
|
|
71
|
+
"193_wily" : "15501247",
|
|
72
|
+
"471" : "15501249",
|
|
73
|
+
"323_grabow" : "15501251",
|
|
74
|
+
"2019_GrandChallengeIT_paper" : "15501253",
|
|
75
|
+
"203_patton" : "15501255",
|
|
76
|
+
"450" : "15501259",
|
|
77
|
+
"359_wwieder" : "15501261",
|
|
78
|
+
"205_xuguang" : "15501265",
|
|
79
|
+
"218_xuguang" : "15501267",
|
|
80
|
+
"236_xuguang" : "15501269",
|
|
81
|
+
"211_fjudt" : "15501273",
|
|
82
|
+
"2018_Schmidt_JGR_Volcanic_RF" : "15501277",
|
|
83
|
+
"438" : "15501279",
|
|
84
|
+
"267_fredc" : "15501282",
|
|
85
|
+
"321_patton" : "15501286",
|
|
86
|
+
"467" : "15501294",
|
|
87
|
+
"145_lqian" : "15501296",
|
|
88
|
+
"317_jaredlee" : "15501298",
|
|
89
|
+
"WACCMX_2005_Solar_Flare" : "15501305",
|
|
90
|
+
"369_buchholz" : "15501313",
|
|
91
|
+
"148_bjohns" : "15501319",
|
|
92
|
+
"158_asphilli" : "15501326",
|
|
93
|
+
"VAPOR_Sample_Data" : "15501333",
|
|
94
|
+
"188b_oleson" : "15501339",
|
|
95
|
+
"206_grabow" : "15501349",
|
|
96
|
+
"127_cweeks" : "15501358",
|
|
97
|
+
"2019_Ionosphere_Thermosphere_Qian" : "15501375",
|
|
98
|
+
"391_kim" : "15501388",
|
|
99
|
+
"200_fredc" : "15501403",
|
|
100
|
+
"147_miesch" : "15501412",
|
|
101
|
+
"141_maute" : "15501419",
|
|
102
|
+
"147b_jcsda" : "15501432",
|
|
103
|
+
"283_dll" : "15501444",
|
|
104
|
+
"78_clyne_wrf" : "15501451",
|
|
105
|
+
"345_wwieder" : "15501470",
|
|
106
|
+
"221_mclong" : "15501488",
|
|
107
|
+
"257_ottobli" : "15501516",
|
|
108
|
+
"287_qwu" : "15501551",
|
|
109
|
+
"400_qwu" : "15501569",
|
|
110
|
+
"184_jjang" : "15501603",
|
|
111
|
+
"197_islas" : "15501620",
|
|
112
|
+
"ftir" : "15501660",
|
|
113
|
+
"282_fasullo" : "15501697",
|
|
114
|
+
"139_oclifton" : "15501795",
|
|
115
|
+
"348_wwieder" : "15501824",
|
|
116
|
+
"451" : "15501860",
|
|
117
|
+
"188a_oleson" : "15501878",
|
|
118
|
+
"173A_dcherian" : "15501913",
|
|
119
|
+
"297_qingyu" : "15501946",
|
|
120
|
+
"523" : "15501974",
|
|
121
|
+
"472" : "15502021",
|
|
122
|
+
"138_maute" : "15502102",
|
|
123
|
+
"387_jay" : "15502178",
|
|
124
|
+
"jedi-skylab" : "15502247",
|
|
125
|
+
"CLM5_Sensitivity_Analyses" : "15502308",
|
|
126
|
+
"531" : "15502409",
|
|
127
|
+
"327_qwu" : "15502530",
|
|
128
|
+
"94_nickp" : "15502666",
|
|
129
|
+
"camels" : "15529996",
|
|
130
|
+
"68_rschwant" : "15502739",
|
|
131
|
+
"WACCM_VolcAerProp" : "15503247",
|
|
132
|
+
"213_fasullo" : "15503556",
|
|
133
|
+
"263_nystrom" : "15504044",
|
|
134
|
+
"239_fasullo" : "15504166",
|
|
135
|
+
"292_qingyu" : "15504276",
|
|
136
|
+
"294_maute" : "15504555",
|
|
137
|
+
"209_yeager" : "15504665",
|
|
138
|
+
"440" : "15504780",
|
|
139
|
+
"207_gunterl" : "15504857",
|
|
140
|
+
"172_cdswk" : "15505157",
|
|
141
|
+
"285_qingyu" : "15505262",
|
|
142
|
+
"422" : "15505355",
|
|
143
|
+
"120_maute" : "15505595",
|
|
144
|
+
"69_maute" : "15507025",
|
|
145
|
+
"63_maute" : "15507365",
|
|
146
|
+
"280_islas" : "15507449",
|
|
147
|
+
"JAMES_2019MS001833" : "15507514",
|
|
148
|
+
"165_nickp" : "15507754",
|
|
149
|
+
"510" : "15507947",
|
|
150
|
+
"96_morrison" : "15508039",
|
|
151
|
+
"164_bukovsky" : "15508226",
|
|
152
|
+
"378_caron" : "15508458",
|
|
153
|
+
"483" : "15508746",
|
|
154
|
+
"144_grabow" : "15508914",
|
|
155
|
+
"188c_oleson" : "15509127",
|
|
156
|
+
"204_ajahn" : "15510016",
|
|
157
|
+
"72_kdagon" : "15511424",
|
|
158
|
+
"302_yeager" : "15511789",
|
|
159
|
+
"86_morrison" : "15513137",
|
|
160
|
+
"259_jimenez" : "15514297"
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
ALLIDS = list(DSIDS.keys())
|
|
164
|
+
|
|
165
|
+
WFILES = {}
|
|
166
|
+
|
|
167
|
+
#
|
|
168
|
+
# main function to run this program
|
|
169
|
+
#
|
|
170
|
+
def main():
|
|
171
|
+
|
|
172
|
+
params = {} # array of input values
|
|
173
|
+
argv = sys.argv[1:]
|
|
174
|
+
opt = None
|
|
175
|
+
|
|
176
|
+
for arg in argv:
|
|
177
|
+
if arg == "-b":
|
|
178
|
+
PgLOG.PGLOG['BCKGRND'] = 1
|
|
179
|
+
elif re.match(r'^-[msNy]$', arg):
|
|
180
|
+
opt = arg[1]
|
|
181
|
+
params[opt] = []
|
|
182
|
+
elif re.match(r'^-', arg):
|
|
183
|
+
PgLOG.pglog(arg + ": Invalid Option", PgLOG.LGWNEX)
|
|
184
|
+
elif opt:
|
|
185
|
+
params[opt].append(arg)
|
|
186
|
+
else:
|
|
187
|
+
PgLOG.pglog(arg + ": Value passed in without leading option", PgLOG.LGWNEX)
|
|
188
|
+
|
|
189
|
+
if not opt:
|
|
190
|
+
PgLOG.show_usage('fillgdexusage')
|
|
191
|
+
elif 's' not in params:
|
|
192
|
+
PgLOG.pglog("-s: Missing dataset short name to gather ZENODO metrics", PgLOG.LGWNEX)
|
|
193
|
+
elif len(params) < 2:
|
|
194
|
+
PgLOG.pglog("-(m|N|y): Missing Month, NumberDays or Year to gather ZENODO metrics", PgLOG.LGWNEX)
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
PgLOG.cmdlog("fillgdexusage {}".format(' '.join(argv)))
|
|
198
|
+
dranges = get_date_ranges(params)
|
|
199
|
+
dsids = get_dataset_ids(params['s'])
|
|
200
|
+
if dranges and dsids: fill_gdex_usages(dsids, dranges)
|
|
201
|
+
PgLOG.pglog(None, PgLOG.LOGWRN|PgLOG.SNDEML) # send email out if any
|
|
202
|
+
|
|
203
|
+
sys.exit(0)
|
|
204
|
+
|
|
205
|
+
#
|
|
206
|
+
# connect to the gdex database esg-production
|
|
207
|
+
#
|
|
208
|
+
def gdex_dbname():
|
|
209
|
+
PgDBI.set_scname('gdex-production', 'metrics', 'gateway-reader', None, 'sagedbprodalma.ucar.edu')
|
|
210
|
+
|
|
211
|
+
#
|
|
212
|
+
# get datasets
|
|
213
|
+
#
|
|
214
|
+
def get_dataset_ids(dsnames):
|
|
215
|
+
|
|
216
|
+
gdex_dbname()
|
|
217
|
+
dsids = []
|
|
218
|
+
tbname = 'metadata.dataset'
|
|
219
|
+
for dsname in dsnames:
|
|
220
|
+
if re.match(r'^all$', dsname, re.I): return get_dataset_ids(ALLIDS)
|
|
221
|
+
if dsname not in DSIDS:
|
|
222
|
+
PgLOG.pglog(dsname + ": Unknown ZENODO dataset short name", PgLOG.LOGWRN)
|
|
223
|
+
continue
|
|
224
|
+
bt = tm()
|
|
225
|
+
pgrec = PgDBI.pgget(tbname, 'id', "short_name = '{}'".format(dsname))
|
|
226
|
+
if not (pgrec and pgrec['id']): continue
|
|
227
|
+
zndid = DSIDS[dsname]
|
|
228
|
+
strids = "{}-{}".format(dsname, zndid)
|
|
229
|
+
gdexid = pgrec['id']
|
|
230
|
+
gdexids = [gdexid]
|
|
231
|
+
ccnt = 1
|
|
232
|
+
ccnt += recursive_dataset_ids(gdexid, gdexids)
|
|
233
|
+
dsids.append([dsname, zndid, gdexids, strids])
|
|
234
|
+
rmsg = PgLOG.seconds_to_string_time(tm() - bt)
|
|
235
|
+
PgLOG.pglog("{}: Found {} GDEX dsid/subdsids in {} at {}".format(strids, ccnt, rmsg, PgLOG.current_datetime()), PgLOG.LOGWRN)
|
|
236
|
+
|
|
237
|
+
if not dsids: PgLOG.pglog("No Dataset Id identified to gather GDEX metrics", PgLOG.LOGWRN)
|
|
238
|
+
|
|
239
|
+
return dsids
|
|
240
|
+
|
|
241
|
+
#
|
|
242
|
+
# get gdexids recursivley
|
|
243
|
+
#
|
|
244
|
+
def recursive_dataset_ids(pgdexid, gdexids):
|
|
245
|
+
|
|
246
|
+
tbname = 'metadata.dataset'
|
|
247
|
+
pgrecs = PgDBI.pgmget(tbname, 'id', "parent_dataset_id = '{}'".format(pgdexid))
|
|
248
|
+
if not pgrecs: return 0
|
|
249
|
+
|
|
250
|
+
ccnt = 0
|
|
251
|
+
for gdexid in pgrecs['id']:
|
|
252
|
+
if gdexid in gdexids: continue
|
|
253
|
+
gdexids.append(gdexid)
|
|
254
|
+
ccnt += 1
|
|
255
|
+
ccnt += recursive_dataset_ids(gdexid, gdexids)
|
|
256
|
+
|
|
257
|
+
return ccnt
|
|
258
|
+
|
|
259
|
+
#
|
|
260
|
+
# get the date ranges for given condition
|
|
261
|
+
#
|
|
262
|
+
def get_date_ranges(inputs):
|
|
263
|
+
|
|
264
|
+
dranges = []
|
|
265
|
+
for opt in inputs:
|
|
266
|
+
for input in inputs[opt]:
|
|
267
|
+
# get date range
|
|
268
|
+
dates = []
|
|
269
|
+
if opt == 'N':
|
|
270
|
+
dates.append(PgUtil.adddate(USAGE['CDATE'], 0, 0, -int(input)))
|
|
271
|
+
dates.append(USAGE['CDATE'])
|
|
272
|
+
elif opt == 'm':
|
|
273
|
+
tms = input.split('-')
|
|
274
|
+
dates.append(PgUtil.fmtdate(int(tms[0]), int(tms[1]), 1))
|
|
275
|
+
dates.append(PgUtil.enddate(dates[0], 0, 'M'))
|
|
276
|
+
elif opt == 'y':
|
|
277
|
+
dates.append(input + "-01-01")
|
|
278
|
+
dates.append(input + "-12-31")
|
|
279
|
+
if dates: dranges.append(dates)
|
|
280
|
+
|
|
281
|
+
return dranges
|
|
282
|
+
|
|
283
|
+
#
|
|
284
|
+
# get file download records for given dsid
|
|
285
|
+
#
|
|
286
|
+
def get_dsid_records(gdexids, dates, strids):
|
|
287
|
+
|
|
288
|
+
gdex_dbname()
|
|
289
|
+
tbname = 'metrics.file_download'
|
|
290
|
+
fields = ('date_completed, remote_address, logical_file_size, logical_file_name, file_access_point_uri, user_agent_name, bytes_sent, '
|
|
291
|
+
'subset_file_size, range_request, dataset_file_size, dataset_file_name, dataset_file_file_access_point_uri')
|
|
292
|
+
dscnt = len(gdexids)
|
|
293
|
+
dscnd = "dataset_id "
|
|
294
|
+
if dscnt == 1:
|
|
295
|
+
dscnd += "= '{}'".format(gdexids[0])
|
|
296
|
+
else:
|
|
297
|
+
dscnd += "IN ('" + "','".join(gdexids) + "')"
|
|
298
|
+
dtcnd = "date_completed BETWEEN '{} 00:00:00' AND '{} 23:59:59'".format(dates[0], dates[1])
|
|
299
|
+
cond = "{} AND {} ORDER BY date_completed".format(dscnd, dtcnd)
|
|
300
|
+
PgLOG.pglog("{}: Query for {} GDEX dsid/subdsids between {} and {} at {}".format(strids, dscnt, dates[0], dates[1], PgLOG.current_datetime()), PgLOG.LOGWRN)
|
|
301
|
+
pgrecs = PgDBI.pgmget(tbname, fields, cond)
|
|
302
|
+
PgDBI.dssdb_dbname()
|
|
303
|
+
|
|
304
|
+
return pgrecs
|
|
305
|
+
|
|
306
|
+
#
|
|
307
|
+
# Fill ZND usages into table dssdb.tdsusage from gdex access records
|
|
308
|
+
#
|
|
309
|
+
def fill_gdex_usages(dsids, dranges):
|
|
310
|
+
|
|
311
|
+
allcnt = awcnt = azcnt = lcnt = 0
|
|
312
|
+
for dates in dranges:
|
|
313
|
+
for dsid in dsids:
|
|
314
|
+
lcnt += 1
|
|
315
|
+
dsname = dsid[0]
|
|
316
|
+
zndid = dsid[1]
|
|
317
|
+
gdexids = dsid[2]
|
|
318
|
+
strids = dsid[3]
|
|
319
|
+
bt = tm()
|
|
320
|
+
pgrecs = get_dsid_records(gdexids, dates, strids)
|
|
321
|
+
pgcnt = len(pgrecs['dataset_file_name']) if pgrecs else 0
|
|
322
|
+
if pgcnt == 0:
|
|
323
|
+
PgLOG.pglog("{}: No record found to gather GDEX usage between {} and {}".format(strids, dates[0], dates[1]), PgLOG.LOGWRN)
|
|
324
|
+
continue
|
|
325
|
+
rmsg = PgLOG.seconds_to_string_time(tm() - bt)
|
|
326
|
+
PgLOG.pglog("{}: Got {} records in {} for processing GDEX usage at {}".format(strids, pgcnt, rmsg, PgLOG.current_datetime()), PgLOG.LOGWRN)
|
|
327
|
+
zcnt = 0
|
|
328
|
+
pwkey = wrec = cdate = None
|
|
329
|
+
zrecs = {}
|
|
330
|
+
bt = tm()
|
|
331
|
+
for i in range(pgcnt):
|
|
332
|
+
if (i+1)%20000 == 0:
|
|
333
|
+
PgLOG.pglog("{}/{} GDEX/ZND records processed to add".format(i, zcnt), PgLOG.WARNLG)
|
|
334
|
+
|
|
335
|
+
pgrec = PgUtil.onerecord(pgrecs, i)
|
|
336
|
+
dsize = pgrec['bytes_sent']
|
|
337
|
+
if not dsize: continue
|
|
338
|
+
(year, quarter, date, time) = get_record_date_time(pgrec['date_completed'])
|
|
339
|
+
url = pgrec['dataset_file_file_access_point_uri']
|
|
340
|
+
if not url: url = pgrec['file_access_point_uri']
|
|
341
|
+
ip = pgrec['remote_address']
|
|
342
|
+
engine = pgrec['user_agent_name']
|
|
343
|
+
ms = re.search(r'^https*://tds.ucar.edu/thredds/(\w+)/', url)
|
|
344
|
+
if ms:
|
|
345
|
+
# tds usage
|
|
346
|
+
if pgrec['subset_file_size']:
|
|
347
|
+
etype = 'S'
|
|
348
|
+
elif pgrec['range_request']:
|
|
349
|
+
etype = 'R'
|
|
350
|
+
else:
|
|
351
|
+
etype = 'F'
|
|
352
|
+
method = 'TDS-' + etype
|
|
353
|
+
else:
|
|
354
|
+
# web usage
|
|
355
|
+
method = 'WEB'
|
|
356
|
+
|
|
357
|
+
if date != cdate:
|
|
358
|
+
if zrecs:
|
|
359
|
+
zcnt += add_zusage_records(zrecs, cdate)
|
|
360
|
+
zrecs = {}
|
|
361
|
+
cdate = date
|
|
362
|
+
zkey = "{}:{}:{}:{}".format(ip, zndid, method)
|
|
363
|
+
if zkey in zrecs:
|
|
364
|
+
zrecs[zkey]['size'] += dsize
|
|
365
|
+
zrecs[zkey]['fcount'] += 1
|
|
366
|
+
else:
|
|
367
|
+
iprec = PgIPInfo.get_missing_ipinfo(ip)
|
|
368
|
+
if not iprec: continue
|
|
369
|
+
zrecs[zkey] = {'ip' : ip, 'zdsid' : zndid, 'date' : cdate, 'time' : time, 'quarter' : quarter,
|
|
370
|
+
'size' : dsize, 'fcount' : 1, 'method' : method}
|
|
371
|
+
|
|
372
|
+
if zrecs: zcnt += add_zusage_records(zrecs, cdate)
|
|
373
|
+
azcnt += zcnt
|
|
374
|
+
allcnt += pgcnt
|
|
375
|
+
rmsg = PgLOG.seconds_to_string_time(tm() - bt)
|
|
376
|
+
PgLOG.pglog("{}: {} usage records added for {} ZENODO entries in {}".format(strids, azcnt, allcnt, rmsg), PgLOG.LOGWRN)
|
|
377
|
+
|
|
378
|
+
def get_record_date_time(ctime):
|
|
379
|
+
|
|
380
|
+
ms = re.search(r'^(\d+)-(\d+)-(\d+) (\d\d:\d\d:\d\d)', str(ctime))
|
|
381
|
+
if ms:
|
|
382
|
+
y = ms.group(1)
|
|
383
|
+
m = int(ms.group(2))
|
|
384
|
+
d = ms.group(3)
|
|
385
|
+
q = 1 + int((m-1)/3)
|
|
386
|
+
t = ms.group(4)
|
|
387
|
+
return (y, q, "{}-{:02}-{}".format(y, m, d), t)
|
|
388
|
+
else:
|
|
389
|
+
PgLOG.pglog(str(ctime) + ": Invalid time format", PgLOG.LGEREX)
|
|
390
|
+
|
|
391
|
+
def add_zusage_records(records, date):
|
|
392
|
+
|
|
393
|
+
cnt = 0
|
|
394
|
+
for key in records:
|
|
395
|
+
record = records[key]
|
|
396
|
+
cond = "date = '{}' AND time = '{}' AND ip = '{}'".format(date, record['time'], record['ip'])
|
|
397
|
+
if PgDBI.pgget(USAGE['ZNDTBL'], '', cond, PgLOG.LGEREX): continue
|
|
398
|
+
|
|
399
|
+
cnt += PgDBI.pgadd(USAGE['ZNDTBL'], record, PgLOG.LOGWRN)
|
|
400
|
+
|
|
401
|
+
PgLOG.pglog("{}: {} ZND usage records added at {}".format(date, cnt, PgLOG.current_datetime()), PgLOG.LOGWRN)
|
|
402
|
+
|
|
403
|
+
return cnt
|
|
404
|
+
|
|
405
|
+
#
|
|
406
|
+
# call main() to start program
|
|
407
|
+
#
|
|
408
|
+
if __name__ == "__main__": main()
|
|
@@ -9,7 +9,7 @@ rda_python_metrics/fillcodusage.py,sha256=_RUcgY1Cf17dSjidhQCMOwVaPO1VL26s3bYGT8
|
|
|
9
9
|
rda_python_metrics/fillcodusage.usg,sha256=3B5IkQ4uwylqY8uEfUdnZ_MXqhYudeylMp5ulhUGXH8,678
|
|
10
10
|
rda_python_metrics/fillcountry.py,sha256=7i5LNi3scRoyRCT6t7aeNTGKOpxzJ2mA9tnvUqje2AU,2314
|
|
11
11
|
rda_python_metrics/fillendtime.py,sha256=skZttlpoY19g0dGwqGQI8t_1YPPTPEXwg3EfNlfL90I,2533
|
|
12
|
-
rda_python_metrics/fillgdexusage.py,sha256=
|
|
12
|
+
rda_python_metrics/fillgdexusage.py,sha256=9U8k2WykGAmnyKvY6ZiYqzpw9MXpE1fpyJUjFDMb_Fc,38258
|
|
13
13
|
rda_python_metrics/fillgdexusage.usg,sha256=mVYtK0pIYmvma0skT-wXM-NOEpkN_i3E61UdWgQWLfs,648
|
|
14
14
|
rda_python_metrics/fillglobususage.py,sha256=zuxzoeV9BAMoVUu_VCYECPddYNrBWpV74kkYpyqIQhg,8443
|
|
15
15
|
rda_python_metrics/fillglobususage.usg,sha256=1GgmCP22IQZdADwL5Mmkz3v8Ws-G7U3teQ1AxRJfV_4,637
|
|
@@ -25,6 +25,7 @@ rda_python_metrics/filltdsusage.py,sha256=vOwVzAtWUHO4O-FCSJMg0GKxw-Xc5AzSbfqVFk
|
|
|
25
25
|
rda_python_metrics/filltdsusage.usg,sha256=yqTHRe8WpZWpu3gso_obgt6LO41-JC27rTIDfdZcISo,538
|
|
26
26
|
rda_python_metrics/filluser.py,sha256=CvaMRaUPaR9nxJAExkLTb3Ci4sD7RQMOpWhWJdbyMF0,8907
|
|
27
27
|
rda_python_metrics/filluser.usg,sha256=Xgqi0QwA9-4jpYj3L4Q4TISpVwRlsomt2G7T0oYAFak,520
|
|
28
|
+
rda_python_metrics/fillzenodousage.py,sha256=BWm1SLNNX7nhO6he9Ccl6lH9jUY9G83AmrusDMY6Dc8,13204
|
|
28
29
|
rda_python_metrics/logarch.py,sha256=8OFirXeO24dljXQPbNJqGDvP9F0qWK11VHSw38CdUuM,13896
|
|
29
30
|
rda_python_metrics/logarch.usg,sha256=6ksFxu_-lzhRpQE2YFAqgGVhwFP4PTweCbFBuGhhICU,960
|
|
30
31
|
rda_python_metrics/pgperson.py,sha256=4XTDnVBFyF9drzx2I_DGCxQ-8I1bpclqTuQvOx3wST4,2088
|
|
@@ -50,9 +51,9 @@ rda_python_metrics/viewwebfile.py,sha256=HSMNkQQawonu6W3blV7g9UbJuNy9VAOn9COqgmj
|
|
|
50
51
|
rda_python_metrics/viewwebfile.usg,sha256=lTNi8Yu8BUJuExEDJX-vsJyWUSUIQTS-DiiBEVFo33s,10054
|
|
51
52
|
rda_python_metrics/viewwebusage.py,sha256=ES2lI8NaCeCpTGi94HU-cDRBxHMiUBbplyYsZf2KqF0,16650
|
|
52
53
|
rda_python_metrics/viewwebusage.usg,sha256=OVDZ78p87E3HLW34ZhasNJ7Zmw8XXjmZPPWZfRhPLXo,9936
|
|
53
|
-
rda_python_metrics-1.0.
|
|
54
|
-
rda_python_metrics-1.0.
|
|
55
|
-
rda_python_metrics-1.0.
|
|
56
|
-
rda_python_metrics-1.0.
|
|
57
|
-
rda_python_metrics-1.0.
|
|
58
|
-
rda_python_metrics-1.0.
|
|
54
|
+
rda_python_metrics-1.0.42.dist-info/licenses/LICENSE,sha256=1dck4EAQwv8QweDWCXDx-4Or0S8YwiCstaso_H57Pno,1097
|
|
55
|
+
rda_python_metrics-1.0.42.dist-info/METADATA,sha256=X6jSYha4z0rwsLp1cFusK5TL2fAFzQbHNqEQt0ocJlw,761
|
|
56
|
+
rda_python_metrics-1.0.42.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
57
|
+
rda_python_metrics-1.0.42.dist-info/entry_points.txt,sha256=fv4wjyEdiLFYNaJzhmet0SxZcF_J6M42koft2Na1XMs,1403
|
|
58
|
+
rda_python_metrics-1.0.42.dist-info/top_level.txt,sha256=aoBgbR_o70TP0QmMW0U6inRHYtfKld47OBmnWnLnDOs,19
|
|
59
|
+
rda_python_metrics-1.0.42.dist-info/RECORD,,
|
{rda_python_metrics-1.0.41.dist-info → rda_python_metrics-1.0.42.dist-info}/entry_points.txt
RENAMED
|
@@ -12,6 +12,7 @@ fillosdfusage = rda_python_metrics.fillosdfusage:main
|
|
|
12
12
|
fillrdadb = rda_python_metrics.fillrdadb:main
|
|
13
13
|
filltdsusage = rda_python_metrics.filltdsusage:main
|
|
14
14
|
filluser = rda_python_metrics.filluser:main
|
|
15
|
+
fillzenodousage = rda_python_metrics.fillzenodousage:main
|
|
15
16
|
logarch.py = rda_python_metrics.logarch:main
|
|
16
17
|
pgperson = rda_python_metrics.pgperson:main
|
|
17
18
|
pgusername = rda_python_metrics.pgusername:main
|
|
File without changes
|
{rda_python_metrics-1.0.41.dist-info → rda_python_metrics-1.0.42.dist-info}/licenses/LICENSE
RENAMED
|
File without changes
|
|
File without changes
|