rda-python-metrics 1.0.28__py3-none-any.whl → 1.0.31__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of rda-python-metrics might be problematic. Click here for more details.
- rda_python_metrics/PgIPInfo.py +6 -6
- rda_python_metrics/fillcdgusage.py +39 -25
- rda_python_metrics/fillgdexusage.py +1022 -0
- rda_python_metrics/fillgdexusage.usg +18 -0
- rda_python_metrics/pgperson.py +2 -1
- rda_python_metrics/pgusername.py +2 -1
- {rda_python_metrics-1.0.28.dist-info → rda_python_metrics-1.0.31.dist-info}/METADATA +1 -1
- {rda_python_metrics-1.0.28.dist-info → rda_python_metrics-1.0.31.dist-info}/RECORD +12 -10
- {rda_python_metrics-1.0.28.dist-info → rda_python_metrics-1.0.31.dist-info}/WHEEL +0 -0
- {rda_python_metrics-1.0.28.dist-info → rda_python_metrics-1.0.31.dist-info}/entry_points.txt +0 -0
- {rda_python_metrics-1.0.28.dist-info → rda_python_metrics-1.0.31.dist-info}/licenses/LICENSE +0 -0
- {rda_python_metrics-1.0.28.dist-info → rda_python_metrics-1.0.31.dist-info}/top_level.txt +0 -0
rda_python_metrics/PgIPInfo.py
CHANGED
|
@@ -138,7 +138,10 @@ def get_ipinfo_record(ip):
|
|
|
138
138
|
except Exception as e:
|
|
139
139
|
PgLOG.pglog("ipinfo: {} - {}".format(ip, str(e)), PgLOG.LOGWRN)
|
|
140
140
|
return None
|
|
141
|
-
|
|
141
|
+
|
|
142
|
+
if 'bogon' in iprec and iprec['bogon']:
|
|
143
|
+
PgLOG.pglog(f"ipinfo: {ip} - bogon", PgLOG.LOGWRN)
|
|
144
|
+
return None
|
|
142
145
|
record = {'ip' : ip, 'stat_flag' : 'A', 'hostname' : ip, 'org_type' : '-'}
|
|
143
146
|
get_ip_hostname(ip, iprec, record)
|
|
144
147
|
record['lat'] = float(iprec['latitude']) if iprec['latitude'] else 0
|
|
@@ -234,12 +237,9 @@ def update_ipinfo_record(record, pgrec = None):
|
|
|
234
237
|
#
|
|
235
238
|
def set_ipinfo(ip, ipopt = True):
|
|
236
239
|
|
|
237
|
-
if ip in IPRECS:
|
|
238
|
-
pgrec = IPRECS[ip]
|
|
239
|
-
if pgrec or not ipopt: return pgrec
|
|
240
|
-
else:
|
|
241
|
-
pgrec = PgDBI.pgget('ipinfo', '*', "ip = '{}'".format(ip))
|
|
240
|
+
if ip in IPRECS: return IPRECS[ip]
|
|
242
241
|
|
|
242
|
+
pgrec = PgDBI.pgget('ipinfo', '*', "ip = '{}'".format(ip))
|
|
243
243
|
if not pgrec or ipopt and pgrec['stat_flag'] == 'M':
|
|
244
244
|
record = get_ipinfo_record(ip) if ipopt else None
|
|
245
245
|
if not record: record = get_geoip2_record(ip)
|
|
@@ -31,28 +31,28 @@ USAGE = {
|
|
|
31
31
|
}
|
|
32
32
|
|
|
33
33
|
DSIDS = {
|
|
34
|
-
'ucar.cgd.cesm2.cam6.prescribed_sst_amip' : ['d651010'],
|
|
35
|
-
'ucar.cgd.ccsm4.CLM_LAND_ONLY' : ['d651011'],
|
|
36
|
-
'ucar.cgd.artmip' : ['d651012', 'd651016', 'd651017', 'd651018'],
|
|
37
|
-
'tamip' : ['d651013'],
|
|
38
|
-
'ucar.cgd.ccsm4.CLIVAR_LE' : ['d651014'],
|
|
39
|
-
'ucar.cgd.cesm2.Gettelman_CESM2_ECS' : ['d651015'],
|
|
40
|
-
'ucar.cgd.ccsm4.geomip.ssp5' : ['d651024'],
|
|
41
|
-
'ucar.cgd.ccsm4.IOD-PACEMAKER' : ['d651021'],
|
|
42
|
-
'ucar.cgd.ccsm4.past2k_transient' : ['651023'],
|
|
43
|
-
'ucar.cgd.ccsm4.lowwarming' : ['d651025'],
|
|
44
|
-
'ucar.cgd.ccsm4.CESM_CAM5_BGC_ME' : ['d651000'],
|
|
45
|
-
'ucar.cgd.ccsm4.iTRACE' : ['d651022'],
|
|
46
|
-
'ucar.cgd.ccsm4.so2_geoeng' : ['d651026'],
|
|
47
|
-
'ucar.cgd.ccsm4.cesmLE' : ['d651027'],
|
|
48
|
-
'ucar.cgd.ccsm4.CESM1-CAM5-DP' : ['d651028'],
|
|
49
|
-
'ucar.cgd.ccsm4.amv_lens' : ['d651031'],
|
|
50
|
-
'ucar.cgd.ccsm4.ATL-PACEMAKER' : ['d651032'],
|
|
51
|
-
'ucar.cgd.ccsm4.pac-pacemaker' : ['d651033'],
|
|
52
|
-
'ucar.cgd.ccsm4.SD-WACCM-X_v2.1' : ['d651034'],
|
|
53
|
-
'ucar.cgd.ccsm4.amv_lens' : ['d651035'],
|
|
54
|
-
'ucar.cgd.cesm2.cism_ismip6' : ['d651036'],
|
|
55
|
-
'ucar.cgd.ccsm4.pliomip2' : ['d651037'],
|
|
34
|
+
# 'ucar.cgd.cesm2.cam6.prescribed_sst_amip' : ['d651010'],
|
|
35
|
+
# 'ucar.cgd.ccsm4.CLM_LAND_ONLY' : ['d651011'],
|
|
36
|
+
# 'ucar.cgd.artmip' : ['d651012', 'd651016', 'd651017', 'd651018'],
|
|
37
|
+
# 'tamip' : ['d651013'],
|
|
38
|
+
# 'ucar.cgd.ccsm4.CLIVAR_LE' : ['d651014'],
|
|
39
|
+
# 'ucar.cgd.cesm2.Gettelman_CESM2_ECS' : ['d651015'],
|
|
40
|
+
# 'ucar.cgd.ccsm4.geomip.ssp5' : ['d651024'],
|
|
41
|
+
# 'ucar.cgd.ccsm4.IOD-PACEMAKER' : ['d651021'],
|
|
42
|
+
# 'ucar.cgd.ccsm4.past2k_transient' : ['651023'],
|
|
43
|
+
# 'ucar.cgd.ccsm4.lowwarming' : ['d651025'],
|
|
44
|
+
# 'ucar.cgd.ccsm4.CESM_CAM5_BGC_ME' : ['d651000'],
|
|
45
|
+
# 'ucar.cgd.ccsm4.iTRACE' : ['d651022'],
|
|
46
|
+
# 'ucar.cgd.ccsm4.so2_geoeng' : ['d651026'],
|
|
47
|
+
# 'ucar.cgd.ccsm4.cesmLE' : ['d651027'],
|
|
48
|
+
# 'ucar.cgd.ccsm4.CESM1-CAM5-DP' : ['d651028'],
|
|
49
|
+
# 'ucar.cgd.ccsm4.amv_lens' : ['d651031'],
|
|
50
|
+
# 'ucar.cgd.ccsm4.ATL-PACEMAKER' : ['d651032'],
|
|
51
|
+
# 'ucar.cgd.ccsm4.pac-pacemaker' : ['d651033'],
|
|
52
|
+
# 'ucar.cgd.ccsm4.SD-WACCM-X_v2.1' : ['d651034'],
|
|
53
|
+
# 'ucar.cgd.ccsm4.amv_lens' : ['d651035'],
|
|
54
|
+
# 'ucar.cgd.cesm2.cism_ismip6' : ['d651036'],
|
|
55
|
+
# 'ucar.cgd.ccsm4.pliomip2' : ['d651037'],
|
|
56
56
|
# new added
|
|
57
57
|
'ucar.cgd.cesm2-waccm.s2s_hindcasts': ['d651040'],
|
|
58
58
|
'ucar.cgd.CESM1.3_SH_storm_tracks': ['d651044'],
|
|
@@ -61,18 +61,32 @@ DSIDS = {
|
|
|
61
61
|
'ucar.cgd.ccsm4.TC-CESM': ['d651047'],
|
|
62
62
|
'ucar.cgd.cesm2.ISSI_OSSE': ['d651048'],
|
|
63
63
|
'ucar.cgd.ccsm4.SOcean_Eddies_mclong': ['d651049'],
|
|
64
|
-
'trace': ['d651050'],
|
|
64
|
+
'ucar.cgd.ccsm.trace': ['d651050'],
|
|
65
65
|
'ucar.cgd.cesm2.waccm.solar': ['d651051'],
|
|
66
66
|
'ucar.cgd.ccsm4.CESM1-CCSM4_mid-Pliocene' : ['d651042'],
|
|
67
67
|
'ucar.cgd.ccsm4.PaleoIF' : ['d651052'],
|
|
68
68
|
'ucar.cgd.ccsm4.b.e11.B20LE_fixedO3' : ['d651053'],
|
|
69
69
|
'ucar.cgd.cesm2.single.forcing.large.ensemble' : ['d651055'],
|
|
70
|
-
'ucar.cgd.cesm2le.output': ['d651056'],
|
|
70
|
+
'ucar.cgd.cesm2le.output': ['d651056'], # 2025-01 done
|
|
71
|
+
'ucar.cgd.ccsm4.ARISE-SAI-1.5' : ['d651059'],
|
|
71
72
|
'ucar.cgd.cesm2.s2s_hindcasts': ['d651060'],
|
|
72
73
|
'ucar.cgd.cesm2.s2s_hindcasts.mjo': ['d651061'],
|
|
73
74
|
'ucar.cgd.cesm2.s2s_hindcasts.tc_tracks': ['d651062'],
|
|
74
75
|
'ucar.cgd.cesm2.s2s_hindcasts.cesm2.climo': ['d651063'],
|
|
75
|
-
|
|
76
|
+
#add 2025-08-06
|
|
77
|
+
'ucar.cgd.ccsm4.cesmLME' : ['d651058'],
|
|
78
|
+
'ucar.cgd.ccsm4.GLENS' : ['d651064'],
|
|
79
|
+
'ucar.cgd.ccsm4.CESM2-CISM2-LIGtransient' : ['d651066'],
|
|
80
|
+
'ucar.cgd.cesm2.pacific.pacemaker' : ['d651068'],
|
|
81
|
+
'ucar.cgd.cesm2.tuned.sea.ice.albedo' : ['d651070'],
|
|
82
|
+
'ucar.cgd.cesm2.cmip5.forcing' : ['d651075'],
|
|
83
|
+
'ucar.cgd.cesm2.ssp245.biomass.burning' : ['d651073'],
|
|
84
|
+
'ucar.cgd.cesm2.ssp585.biomass.burning' : ['d651067'],
|
|
85
|
+
'ucar.cgd.cesm1.cldmod': ['d651069'],
|
|
86
|
+
'ucar.cgd.cesm2.marine.biogeochemistry': ['d651071'],
|
|
87
|
+
'ucar.cgd.nw2.mom6': ['d651072'],
|
|
88
|
+
# 'ucar.cgd.cesm2.smyle': ['d651065'], # not done util the end of Aug.
|
|
89
|
+
|
|
76
90
|
}
|
|
77
91
|
|
|
78
92
|
ALLIDS = list(DSIDS.keys())
|
|
@@ -0,0 +1,1022 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
#
|
|
3
|
+
###############################################################################
|
|
4
|
+
#
|
|
5
|
+
# Title : fillgdexusage
|
|
6
|
+
# Author : Zaihua Ji, zji@ucar.edu
|
|
7
|
+
# Date : 2025-04-14
|
|
8
|
+
# Purpose : python program to retrieve info from GDEX Postgres database for GDS
|
|
9
|
+
# file accesses and backup fill table tdsusage in PostgreSQL database dssdb.
|
|
10
|
+
#
|
|
11
|
+
# Github : https://github.com/NCAR/rda-python-metrics.git
|
|
12
|
+
#
|
|
13
|
+
###############################################################################
|
|
14
|
+
#
|
|
15
|
+
import sys
|
|
16
|
+
import re
|
|
17
|
+
import glob
|
|
18
|
+
from os import path as op
|
|
19
|
+
from time import time as tm
|
|
20
|
+
from rda_python_common import PgLOG
|
|
21
|
+
from rda_python_common import PgUtil
|
|
22
|
+
from rda_python_common import PgFile
|
|
23
|
+
from rda_python_common import PgDBI
|
|
24
|
+
from rda_python_common import PgSplit
|
|
25
|
+
from . import PgIPInfo
|
|
26
|
+
|
|
27
|
+
USAGE = {
|
|
28
|
+
'WEBTBL' : "wusage",
|
|
29
|
+
'CDATE' : PgUtil.curdate(),
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
DSIDS = {
|
|
33
|
+
'371_abaker' : 'd583131',
|
|
34
|
+
'CESM-cocco_CO2_experiments' : 'd583132',
|
|
35
|
+
'256_tilmes' : 'd583137',
|
|
36
|
+
'fuel_moisture_content' : 'd583133',
|
|
37
|
+
'81_rberrios' : 'd583134',
|
|
38
|
+
'210_dxyang' : 'd583100',
|
|
39
|
+
'286_knocasio' : 'd583135',
|
|
40
|
+
'238_junkyung' : 'd583101',
|
|
41
|
+
'361_islas' : 'd583102',
|
|
42
|
+
'448' : 'd583103',
|
|
43
|
+
'383_stebbins' : 'd583136',
|
|
44
|
+
'187_jaredlee' : 'd583104',
|
|
45
|
+
'273_kdagon' : 'd583105',
|
|
46
|
+
'149_liuh' : 'd583106',
|
|
47
|
+
'279_schwantes' : 'd583107',
|
|
48
|
+
'60_pinto' : 'd583108',
|
|
49
|
+
'WACC_Dependence_on_Solar_Activity' : 'd583109',
|
|
50
|
+
'275_domingom' : 'd583110',
|
|
51
|
+
'445' : 'd583111',
|
|
52
|
+
'TIE-GCM_2017_Solar_Storm' : 'd583112',
|
|
53
|
+
'123_andrew' : 'd583113',
|
|
54
|
+
'453' : 'd583114',
|
|
55
|
+
'367_mclong' : 'd583115',
|
|
56
|
+
'258_jiangzhu' : 'd583116',
|
|
57
|
+
'136_hannay' : 'd583117',
|
|
58
|
+
'173B_dcherian' : 'd583118',
|
|
59
|
+
'407_islas' : 'd583119',
|
|
60
|
+
'189_islas' : 'd583120',
|
|
61
|
+
'289_shields' : 'd583121',
|
|
62
|
+
'415_mcinerney' : 'd583122',
|
|
63
|
+
'151_dll' : 'd583123',
|
|
64
|
+
'460' : 'd583124',
|
|
65
|
+
'117_phamkh' : 'd583125',
|
|
66
|
+
'109_erichend' : 'd583126',
|
|
67
|
+
'412' : 'd583127',
|
|
68
|
+
'390_knocasio' : 'd583128',
|
|
69
|
+
'382_kumar' : 'd583129',
|
|
70
|
+
'470' : 'd583130',
|
|
71
|
+
# icarus
|
|
72
|
+
'icarus.experiment.403' : 'd789010',
|
|
73
|
+
'icarus.experiment.404' : 'd789011',
|
|
74
|
+
'icarus.experiment.390' : 'd789014',
|
|
75
|
+
'icarus.experiment.651' : 'd789012',
|
|
76
|
+
'icarus.experiment.397' : 'd789013',
|
|
77
|
+
'icarus.experiment.401' : 'd789015',
|
|
78
|
+
'icarus.experiment.398' : 'd789016',
|
|
79
|
+
'icarus.experiment.95' : 'd789017',
|
|
80
|
+
'icarus.experiment.400' : 'd789018',
|
|
81
|
+
'icarus.experiment.407' : 'd789019',
|
|
82
|
+
'icarus.experiment.396' : 'd789020',
|
|
83
|
+
'icarus.experiment.252' : 'd789021',
|
|
84
|
+
'icarus.experiment.97' : 'd789022',
|
|
85
|
+
'icarus.experiment.408' : 'd789023',
|
|
86
|
+
'icarus.experiment.406' : 'd789024',
|
|
87
|
+
'icarus.experiment.405' : 'd789025',
|
|
88
|
+
'icarus.experiment.250' : 'd789026',
|
|
89
|
+
'icarus.experiment.247' : 'd789027',
|
|
90
|
+
'icarus.experiment.249' : 'd789028',
|
|
91
|
+
'icarus.experiment.265' : 'd789029',
|
|
92
|
+
'icarus.experiment.410' : 'd789030',
|
|
93
|
+
'icarus.experiment.268' : 'd789031',
|
|
94
|
+
'icarus.experiment.263' : 'd789032',
|
|
95
|
+
'icarus.experiment.260' : 'd789033',
|
|
96
|
+
'icarus.experiment.94' : 'd789034',
|
|
97
|
+
'icarus.experiment.264' : 'd789035',
|
|
98
|
+
'icarus.experiment.267' : 'd789036',
|
|
99
|
+
'icarus.experiment.399' : 'd789037',
|
|
100
|
+
'icarus.experiment.248' : 'd789038',
|
|
101
|
+
'icarus.experiment.237' : 'd789039',
|
|
102
|
+
'icarus.experiment.652' : 'd789040',
|
|
103
|
+
'icarus.experiment.409' : 'd789041',
|
|
104
|
+
'icarus.experiment.392' : 'd789042',
|
|
105
|
+
'icarus.experiment.153' : 'd789043',
|
|
106
|
+
'icarus.experiment.266' : 'd789044',
|
|
107
|
+
'icarus.experiment.262' : 'd789045',
|
|
108
|
+
'icarus.experiment.261' : 'd789046',
|
|
109
|
+
'icarus.experiment.243' : 'd789047',
|
|
110
|
+
'icarus.experiment.269' : 'd789048',
|
|
111
|
+
'icarus.experiment.391' : 'd789049',
|
|
112
|
+
'icarus.experiment.244' : 'd789050',
|
|
113
|
+
'icarus.experiment.158' : 'd789051',
|
|
114
|
+
'icarus.experiment.160' : 'd789052',
|
|
115
|
+
'icarus.experiment.156' : 'd789053',
|
|
116
|
+
'icarus.experiment.246' : 'd789054',
|
|
117
|
+
'icarus.experiment.245' : 'd789055',
|
|
118
|
+
'icarus.experiment.393' : 'd789056',
|
|
119
|
+
'icarus.experiment.154' : 'd789057',
|
|
120
|
+
'icarus.experiment.157' : 'd789058',
|
|
121
|
+
'icarus.experiment.394' : 'd789059',
|
|
122
|
+
'icarus.experiment.619' : 'd789060',
|
|
123
|
+
'icarus.experiment.159' : 'd789061',
|
|
124
|
+
'icarus.experiment.648' : 'd789062',
|
|
125
|
+
'icarus.experiment.155' : 'd789063',
|
|
126
|
+
'icarus.experiment.647' : 'd789064',
|
|
127
|
+
'icarus.experiment.449' : 'd789065',
|
|
128
|
+
'icarus.experiment.650' : 'd789066',
|
|
129
|
+
'icarus.experiment.644' : 'd789067',
|
|
130
|
+
'icarus.experiment.645' : 'd789068',
|
|
131
|
+
'icarus.experiment.527' : 'd789069',
|
|
132
|
+
'icarus.experiment.100' : 'd789070',
|
|
133
|
+
'icarus.experiment.112' : 'd789071',
|
|
134
|
+
'icarus.experiment.658' : 'd789072',
|
|
135
|
+
'icarus.experiment.222' : 'd789073',
|
|
136
|
+
'icarus.experiment.649' : 'd789074',
|
|
137
|
+
'icarus.experiment.251' : 'd789076',
|
|
138
|
+
'icarus.experiment.643' : 'd789077',
|
|
139
|
+
'icarus.experiment.922' : 'd789079',
|
|
140
|
+
'icarus.experiment.646' : 'd789080',
|
|
141
|
+
'icarus.experiment.618' : 'd789081',
|
|
142
|
+
'icarus.experiment.612' : 'd789082',
|
|
143
|
+
'icarus.experiment.691' : 'd789083',
|
|
144
|
+
'icarus.experiment.924' : 'd789084',
|
|
145
|
+
'icarus.experiment.920' : 'd789085',
|
|
146
|
+
'icarus.experiment.610' : 'd789086',
|
|
147
|
+
'icarus.experiment.692' : 'd789087',
|
|
148
|
+
'icarus.experiment.611' : 'd789088',
|
|
149
|
+
'icarus.experiment.225' : 'd789089',
|
|
150
|
+
'icarus.experiment.613' : 'd789090',
|
|
151
|
+
'icarus.experiment.921' : 'd789091',
|
|
152
|
+
'icarus.experiment.918' : 'd789092',
|
|
153
|
+
'icarus.experiment.617' : 'd789093',
|
|
154
|
+
'icarus.experiment.693' : 'd789094',
|
|
155
|
+
'icarus.experiment.220' : 'd789095',
|
|
156
|
+
'icarus.experiment.620' : 'd789096',
|
|
157
|
+
'icarus.experiment.694' : 'd789097',
|
|
158
|
+
'icarus.experiment.224' : 'd789098',
|
|
159
|
+
'icarus.experiment.919' : 'd789099',
|
|
160
|
+
'icarus.experiment.615' : 'd789100',
|
|
161
|
+
'icarus.experiment.221' : 'd789101',
|
|
162
|
+
'icarus.experiment.616' : 'd789102',
|
|
163
|
+
'icarus.experiment.543' : 'd789103',
|
|
164
|
+
'icarus.experiment.498' : 'd789104',
|
|
165
|
+
'icarus.experiment.871' : 'd789105',
|
|
166
|
+
'icarus.experiment.591' : 'd789106',
|
|
167
|
+
'icarus.experiment.215' : 'd789107',
|
|
168
|
+
'icarus.experiment.614' : 'd789108',
|
|
169
|
+
'icarus.experiment.621' : 'd789109',
|
|
170
|
+
'icarus.experiment.923' : 'd789117',
|
|
171
|
+
'icarus.experiment.872' : 'd789110',
|
|
172
|
+
'icarus.experiment.882' : 'd789111',
|
|
173
|
+
'icarus.experiment.529' : 'd789112',
|
|
174
|
+
'icarus.experiment.695' : 'd789113',
|
|
175
|
+
'icarus.experiment.501' : 'd789140',
|
|
176
|
+
'icarus.experiment.887' : 'd789118',
|
|
177
|
+
'icarus.experiment.588' : 'd789154',
|
|
178
|
+
'icarus.experiment.875' : 'd789155',
|
|
179
|
+
'icarus.experiment.589' : 'd789192',
|
|
180
|
+
'icarus.experiment.884' : 'd789193',
|
|
181
|
+
'icarus.experiment.587' : 'd789194',
|
|
182
|
+
'icarus.experiment.502' : 'd789195',
|
|
183
|
+
'icarus.experiment.148' : 'd789196',
|
|
184
|
+
'icarus.experiment.609' : 'd789197',
|
|
185
|
+
'icarus.experiment.885' : 'd789198',
|
|
186
|
+
'icarus.experiment.586' : 'd789199',
|
|
187
|
+
'icarus.experiment.585' : 'd789200',
|
|
188
|
+
'icarus.experiment.212' : 'd789201',
|
|
189
|
+
'icarus.experiment.497' : 'd789202',
|
|
190
|
+
'icarus.experiment.590' : 'd789203',
|
|
191
|
+
'icarus.experiment.889' : 'd789204',
|
|
192
|
+
'icarus.experiment.532' : 'd789245',
|
|
193
|
+
'icarus.experiment.873' : 'd789246',
|
|
194
|
+
'icarus.experiment.881' : 'd789247',
|
|
195
|
+
'icarus.experiment.876' : 'd789248',
|
|
196
|
+
'icarus.experiment.888' : 'd789249',
|
|
197
|
+
'icarus.experiment.879' : 'd789250',
|
|
198
|
+
'icarus.experiment.75' : 'd789251',
|
|
199
|
+
'icarus.experiment.149' : 'd789252',
|
|
200
|
+
'icarus.experiment.544' : 'd789253',
|
|
201
|
+
'icarus.experiment.622' : 'd789254',
|
|
202
|
+
'icarus.experiment.592' : 'd789255',
|
|
203
|
+
'icarus.experiment.302' : 'd789256',
|
|
204
|
+
'icarus.experiment.505' : 'd789257',
|
|
205
|
+
'icarus.experiment.880' : 'd789258',
|
|
206
|
+
'icarus.experiment.886' : 'd789259',
|
|
207
|
+
'icarus.experiment.593' : 'd789260',
|
|
208
|
+
'icarus.experiment.494' : 'd789261',
|
|
209
|
+
'icarus.experiment.696' : 'd789269',
|
|
210
|
+
'icarus.experiment.513' : 'd789281',
|
|
211
|
+
'icarus.experiment.883' : 'd789282',
|
|
212
|
+
'icarus.experiment.515' : 'd789283',
|
|
213
|
+
'icarus.experiment.447' : 'd789284',
|
|
214
|
+
'icarus.experiment.878' : 'd789285',
|
|
215
|
+
'icarus.experiment.484' : 'd789286',
|
|
216
|
+
'icarus.experiment.427' : 'd789287',
|
|
217
|
+
'icarus.experiment.442' : 'd789308',
|
|
218
|
+
'icarus.experiment.152' : 'd789309',
|
|
219
|
+
'icarus.experiment.486' : 'd789310',
|
|
220
|
+
'icarus.experiment.723' : 'd789311',
|
|
221
|
+
'icarus.experiment.597' : 'd789312',
|
|
222
|
+
'icarus.experiment.482' : 'd789313',
|
|
223
|
+
'icarus.experiment.534' : 'd789314',
|
|
224
|
+
'icarus.experiment.877' : 'd789315',
|
|
225
|
+
'icarus.experiment.413' : 'd789316',
|
|
226
|
+
'icarus.experiment.533' : 'd789317',
|
|
227
|
+
'icarus.experiment.415' : 'd789318',
|
|
228
|
+
'icarus.experiment.517' : 'd789319',
|
|
229
|
+
'icarus.experiment.440' : 'd789320',
|
|
230
|
+
'icarus.experiment.511' : 'd789321',
|
|
231
|
+
'icarus.experiment.443' : 'd789322',
|
|
232
|
+
'icarus.experiment.436' : 'd789323',
|
|
233
|
+
'icarus.experiment.428' : 'd789324',
|
|
234
|
+
'icarus.experiment.481' : 'd789325',
|
|
235
|
+
'icarus.experiment.699' : 'd789326',
|
|
236
|
+
'icarus.experiment.434' : 'd789334',
|
|
237
|
+
'icarus.experiment.437' : 'd789372',
|
|
238
|
+
'icarus.experiment.596' : 'd789373',
|
|
239
|
+
'icarus.experiment.432' : 'd789374',
|
|
240
|
+
'icarus.experiment.542' : 'd789375',
|
|
241
|
+
'icarus.experiment.175' : 'd789376',
|
|
242
|
+
'icarus.experiment.480' : 'd789377',
|
|
243
|
+
'icarus.experiment.441' : 'd789378',
|
|
244
|
+
'icarus.experiment.357' : 'd789379',
|
|
245
|
+
'icarus.experiment.204' : 'd789380',
|
|
246
|
+
'icarus.experiment.256' : 'd789381',
|
|
247
|
+
'icarus.experiment.358' : 'd789382',
|
|
248
|
+
'icarus.experiment.870' : 'd789383',
|
|
249
|
+
'icarus.experiment.435' : 'd789427',
|
|
250
|
+
'icarus.experiment.416' : 'd789428',
|
|
251
|
+
'icarus.experiment.500' : 'd789429',
|
|
252
|
+
'icarus.experiment.488' : 'd789430',
|
|
253
|
+
'icarus.experiment.174' : 'd789431',
|
|
254
|
+
'icarus.experiment.240' : 'd789432',
|
|
255
|
+
'icarus.experiment.176' : 'd789433',
|
|
256
|
+
'icarus.experiment.207' : 'd789434',
|
|
257
|
+
'icarus.experiment.483' : 'd789435',
|
|
258
|
+
'icarus.experiment.531' : 'd789436',
|
|
259
|
+
'icarus.experiment.420' : 'd789445',
|
|
260
|
+
'icarus.experiment.582' : 'd789446',
|
|
261
|
+
'icarus.experiment.425' : 'd789447',
|
|
262
|
+
'icarus.experiment.279' : 'd789448',
|
|
263
|
+
'icarus.experiment.277' : 'd789449',
|
|
264
|
+
'icarus.experiment.254' : 'd789450',
|
|
265
|
+
'icarus.experiment.418' : 'd789451',
|
|
266
|
+
'icarus.experiment.530' : 'd789452',
|
|
267
|
+
'icarus.experiment.424' : 'd789453',
|
|
268
|
+
'icarus.experiment.520' : 'd789454',
|
|
269
|
+
'icarus.experiment.701' : 'd789455',
|
|
270
|
+
'icarus.experiment.423' : 'd789456',
|
|
271
|
+
'icarus.experiment.539' : 'd789457',
|
|
272
|
+
'icarus.experiment.338' : 'd789458',
|
|
273
|
+
'icarus.experiment.322' : 'd789459',
|
|
274
|
+
'icarus.experiment.275' : 'd789460',
|
|
275
|
+
'icarus.experiment.433' : 'd789461',
|
|
276
|
+
'icarus.experiment.353' : 'd789462',
|
|
277
|
+
'icarus.experiment.541' : 'd789465',
|
|
278
|
+
'icarus.experiment.430' : 'd789466',
|
|
279
|
+
'icarus.experiment.448' : 'd789467',
|
|
280
|
+
'icarus.experiment.242' : 'd789468',
|
|
281
|
+
'icarus.experiment.492' : 'd789469',
|
|
282
|
+
'icarus.experiment.328' : 'd789470',
|
|
283
|
+
'icarus.experiment.354' : 'd789471',
|
|
284
|
+
'icarus.experiment.499' : 'd789472',
|
|
285
|
+
'icarus.experiment.283' : 'd789473',
|
|
286
|
+
'icarus.experiment.141' : 'd789474',
|
|
287
|
+
'icarus.experiment.727' : 'd789475',
|
|
288
|
+
'icarus.experiment.421' : 'd789476',
|
|
289
|
+
'icarus.experiment.334' : 'd789477',
|
|
290
|
+
'icarus.experiment.273' : 'd789479',
|
|
291
|
+
'icarus.experiment.506' : 'd789480',
|
|
292
|
+
'icarus.experiment.293' : 'd789481',
|
|
293
|
+
'icarus.experiment.475' : 'd789482',
|
|
294
|
+
'icarus.experiment.332' : 'd789483',
|
|
295
|
+
'icarus.experiment.336' : 'd789484',
|
|
296
|
+
'icarus.experiment.340' : 'd789485',
|
|
297
|
+
'icarus.experiment.471' : 'd789486',
|
|
298
|
+
'icarus.experiment.496' : 'd789487',
|
|
299
|
+
'icarus.experiment.438' : 'd789488',
|
|
300
|
+
'icarus.experiment.281' : 'd789489',
|
|
301
|
+
'icarus.experiment.330' : 'd789490',
|
|
302
|
+
'icarus.experiment.491' : 'd789491',
|
|
303
|
+
'icarus.experiment.595' : 'd789492',
|
|
304
|
+
'icarus.experiment.487' : 'd789493',
|
|
305
|
+
'icarus.experiment.493' : 'd789505',
|
|
306
|
+
'icarus.experiment.241' : 'd789506',
|
|
307
|
+
'icarus.experiment.388' : 'd789507',
|
|
308
|
+
'icarus.experiment.380' : 'd789508',
|
|
309
|
+
'icarus.experiment.381' : 'd789509',
|
|
310
|
+
'icarus.experiment.473' : 'd789510',
|
|
311
|
+
'icarus.experiment.575' : 'd789511',
|
|
312
|
+
'icarus.experiment.536' : 'd789512',
|
|
313
|
+
'icarus.experiment.431' : 'd789513',
|
|
314
|
+
'icarus.experiment.284' : 'd789514',
|
|
315
|
+
'icarus.experiment.323' : 'd789515',
|
|
316
|
+
'icarus.experiment.303' : 'd789516',
|
|
317
|
+
'icarus.experiment.326' : 'd789517',
|
|
318
|
+
'icarus.experiment.419' : 'd789518',
|
|
319
|
+
'icarus.experiment.378' : 'd789519',
|
|
320
|
+
'icarus.experiment.507' : 'd789520',
|
|
321
|
+
'icarus.experiment.102' : 'd789521',
|
|
322
|
+
'icarus.experiment.537' : 'd789522',
|
|
323
|
+
'icarus.experiment.255' : 'd789523',
|
|
324
|
+
'icarus.experiment.375' : 'd789524',
|
|
325
|
+
'icarus.experiment.853' : 'd789525',
|
|
326
|
+
'icarus.experiment.439' : 'd789526',
|
|
327
|
+
'icarus.experiment.538' : 'd789527',
|
|
328
|
+
'icarus.experiment.329' : 'd789528',
|
|
329
|
+
'icarus.experiment.150' : 'd789529',
|
|
330
|
+
'icarus.experiment.509' : 'd789530',
|
|
331
|
+
'icarus.experiment.258' : 'd789531',
|
|
332
|
+
'icarus.experiment.576' : 'd789532',
|
|
333
|
+
'icarus.experiment.253' : 'd789533',
|
|
334
|
+
'icarus.experiment.278' : 'd789542',
|
|
335
|
+
'icarus.experiment.297' : 'd789543',
|
|
336
|
+
'icarus.experiment.348' : 'd789544',
|
|
337
|
+
'icarus.experiment.855' : 'd789545',
|
|
338
|
+
'icarus.experiment.725' : 'd789546',
|
|
339
|
+
'icarus.experiment.623' : 'd789551',
|
|
340
|
+
'icarus.experiment.288' : 'd789552',
|
|
341
|
+
'icarus.experiment.426' : 'd789567',
|
|
342
|
+
'icarus.experiment.724' : 'd789568',
|
|
343
|
+
'icarus.experiment.257' : 'd789569',
|
|
344
|
+
'icarus.experiment.111' : 'd789570',
|
|
345
|
+
'icarus.experiment.259' : 'd789571',
|
|
346
|
+
'icarus.experiment.852' : 'd789572',
|
|
347
|
+
'icarus.experiment.349' : 'd789573',
|
|
348
|
+
'icarus.experiment.854' : 'd789574',
|
|
349
|
+
'icarus.experiment.389' : 'd789575',
|
|
350
|
+
'icarus.experiment.339' : 'd789576',
|
|
351
|
+
'icarus.experiment.490' : 'd789577',
|
|
352
|
+
'icarus.experiment.142' : 'd789578',
|
|
353
|
+
'icarus.experiment.540' : 'd789579',
|
|
354
|
+
'icarus.experiment.120' : 'd789580',
|
|
355
|
+
'icarus.experiment.298' : 'd789581',
|
|
356
|
+
'icarus.experiment.282' : 'd789582',
|
|
357
|
+
'icarus.experiment.516' : 'd789583',
|
|
358
|
+
'icarus.experiment.300' : 'd789584',
|
|
359
|
+
'icarus.experiment.573' : 'd789585',
|
|
360
|
+
'icarus.experiment.113' : 'd789586',
|
|
361
|
+
'icarus.experiment.849' : 'd789587',
|
|
362
|
+
'icarus.experiment.756' : 'd789588',
|
|
363
|
+
'icarus.experiment.755' : 'd789589',
|
|
364
|
+
'icarus.experiment.337' : 'd789590',
|
|
365
|
+
'icarus.experiment.151' : 'd789591',
|
|
366
|
+
'icarus.experiment.274' : 'd789592',
|
|
367
|
+
'icarus.experiment.371' : 'd789593',
|
|
368
|
+
'icarus.experiment.905' : 'd789594',
|
|
369
|
+
'icarus.experiment.295' : 'd789595',
|
|
370
|
+
'icarus.experiment.118' : 'd789596',
|
|
371
|
+
'icarus.experiment.583' : 'd789597',
|
|
372
|
+
'icarus.experiment.535' : 'd789598',
|
|
373
|
+
'icarus.experiment.276' : 'd789599',
|
|
374
|
+
'icarus.experiment.335' : 'd789600',
|
|
375
|
+
'icarus.experiment.292' : 'd789601',
|
|
376
|
+
'icarus.experiment.122' : 'd789602',
|
|
377
|
+
'icarus.experiment.296' : 'd789603',
|
|
378
|
+
'icarus.experiment.414' : 'd789604',
|
|
379
|
+
'icarus.experiment.333' : 'd789605',
|
|
380
|
+
'icarus.experiment.577' : 'd789606',
|
|
381
|
+
'icarus.experiment.512' : 'd789607',
|
|
382
|
+
'icarus.experiment.472' : 'd789608',
|
|
383
|
+
'icarus.experiment.376' : 'd789609',
|
|
384
|
+
'icarus.experiment.857' : 'd789610',
|
|
385
|
+
'icarus.experiment.299' : 'd789611',
|
|
386
|
+
'icarus.experiment.104' : 'd789612',
|
|
387
|
+
'icarus.experiment.514' : 'd789613',
|
|
388
|
+
'icarus.experiment.280' : 'd789614',
|
|
389
|
+
'icarus.experiment.79' : 'd789615',
|
|
390
|
+
'icarus.experiment.341' : 'd789616',
|
|
391
|
+
'icarus.experiment.271' : 'd789617',
|
|
392
|
+
'icarus.experiment.121' : 'd789618',
|
|
393
|
+
'icarus.experiment.386' : 'd789619',
|
|
394
|
+
'icarus.experiment.697' : 'd789620',
|
|
395
|
+
'icarus.experiment.485' : 'd789621',
|
|
396
|
+
'icarus.experiment.607' : 'd789622',
|
|
397
|
+
'icarus.experiment.594' : 'd789623',
|
|
398
|
+
'icarus.experiment.574' : 'd789624',
|
|
399
|
+
'icarus.experiment.291' : 'd789625',
|
|
400
|
+
'icarus.experiment.331' : 'd789626',
|
|
401
|
+
'icarus.experiment.369' : 'd789627',
|
|
402
|
+
'icarus.experiment.287' : 'd789628',
|
|
403
|
+
'icarus.experiment.384' : 'd789629',
|
|
404
|
+
'icarus.experiment.851' : 'd789630',
|
|
405
|
+
'icarus.experiment.355' : 'd789631',
|
|
406
|
+
'icarus.experiment.117' : 'd789632',
|
|
407
|
+
'icarus.experiment.201' : 'd789633',
|
|
408
|
+
'icarus.experiment.850' : 'd789634',
|
|
409
|
+
'icarus.experiment.301' : 'd789635',
|
|
410
|
+
'icarus.experiment.370' : 'd789636',
|
|
411
|
+
'icarus.experiment.856' : 'd789637',
|
|
412
|
+
'icarus.experiment.385' : 'd789638',
|
|
413
|
+
'icarus.experiment.726' : 'd789639',
|
|
414
|
+
'icarus.experiment.858' : 'd789640',
|
|
415
|
+
'icarus.experiment.114' : 'd789641',
|
|
416
|
+
'icarus.experiment.578' : 'd789642',
|
|
417
|
+
'icarus.experiment.387' : 'd789643',
|
|
418
|
+
'icarus.experiment.508' : 'd789644',
|
|
419
|
+
'icarus.experiment.860' : 'd789645',
|
|
420
|
+
'icarus.experiment.859' : 'd789646',
|
|
421
|
+
'icarus.experiment.290' : 'd789647',
|
|
422
|
+
'icarus.experiment.545' : 'd789648',
|
|
423
|
+
'icarus.experiment.519' : 'd789649',
|
|
424
|
+
'icarus.experiment.289' : 'd789650',
|
|
425
|
+
'icarus.experiment.568' : 'd789651',
|
|
426
|
+
'icarus.experiment.356' : 'd789652',
|
|
427
|
+
'icarus.experiment.372' : 'd789653',
|
|
428
|
+
'icarus.experiment.603' : 'd789654',
|
|
429
|
+
'icarus.experiment.272' : 'd789655',
|
|
430
|
+
'icarus.experiment.579' : 'd789656',
|
|
431
|
+
'icarus.experiment.286' : 'd789657',
|
|
432
|
+
'icarus.experiment.580' : 'd789658',
|
|
433
|
+
'icarus.experiment.203' : 'd789659',
|
|
434
|
+
'icarus.experiment.546' : 'd789660',
|
|
435
|
+
'icarus.experiment.624' : 'd789661',
|
|
436
|
+
'icarus.experiment.489' : 'd789662',
|
|
437
|
+
'icarus.experiment.119' : 'd789663',
|
|
438
|
+
'icarus.experiment.383' : 'd789664',
|
|
439
|
+
'icarus.experiment.703' : 'd789665',
|
|
440
|
+
'icarus.experiment.640' : 'd789566',
|
|
441
|
+
'icarus.experiment.294' : 'd789565',
|
|
442
|
+
'icarus.experiment.567' : 'd789564',
|
|
443
|
+
'icarus.experiment.629' : 'd789563',
|
|
444
|
+
'icarus.experiment.564' : 'd789562',
|
|
445
|
+
'icarus.experiment.638' : 'd789561',
|
|
446
|
+
'icarus.experiment.637' : 'd789560',
|
|
447
|
+
'icarus.experiment.633' : 'd789559',
|
|
448
|
+
'icarus.experiment.625' : 'd789558',
|
|
449
|
+
'icarus.experiment.495' : 'd789557',
|
|
450
|
+
'icarus.experiment.635' : 'd789556',
|
|
451
|
+
'icarus.experiment.604' : 'd789555',
|
|
452
|
+
'icarus.experiment.605' : 'd789554',
|
|
453
|
+
'icarus.experiment.598' : 'd789553',
|
|
454
|
+
'icarus.experiment.116' : 'd789550',
|
|
455
|
+
'icarus.experiment.457' : 'd789549',
|
|
456
|
+
'icarus.experiment.606' : 'd789548',
|
|
457
|
+
'icarus.experiment.566' : 'd789547',
|
|
458
|
+
'icarus.experiment.608' : 'd789541',
|
|
459
|
+
'icarus.experiment.469' : 'd789540',
|
|
460
|
+
'icarus.experiment.599' : 'd789539',
|
|
461
|
+
'icarus.experiment.864' : 'd789538',
|
|
462
|
+
'icarus.experiment.861' : 'd789537',
|
|
463
|
+
'icarus.experiment.866' : 'd789536',
|
|
464
|
+
'icarus.experiment.601' : 'd789535',
|
|
465
|
+
'icarus.experiment.565' : 'd789534',
|
|
466
|
+
'icarus.experiment.627' : 'd789504',
|
|
467
|
+
'icarus.experiment.862' : 'd789503',
|
|
468
|
+
'icarus.experiment.458' : 'd789502',
|
|
469
|
+
'icarus.experiment.639' : 'd789501',
|
|
470
|
+
'icarus.experiment.865' : 'd789500',
|
|
471
|
+
'icarus.experiment.602' : 'd789499',
|
|
472
|
+
'icarus.experiment.867' : 'd789498',
|
|
473
|
+
'icarus.experiment.327' : 'd789497',
|
|
474
|
+
'icarus.experiment.863' : 'd789496',
|
|
475
|
+
'icarus.experiment.868' : 'd789495',
|
|
476
|
+
'icarus.experiment.757' : 'd789494',
|
|
477
|
+
'icarus.experiment.759' : 'd789478',
|
|
478
|
+
'icarus.experiment.760' : 'd789464',
|
|
479
|
+
'icarus.experiment.937' : 'd789463',
|
|
480
|
+
'icarus.experiment.476' : 'd789444',
|
|
481
|
+
'icarus.experiment.477' : 'd789443',
|
|
482
|
+
'icarus.experiment.761' : 'd789442',
|
|
483
|
+
'icarus.experiment.106' : 'd789441',
|
|
484
|
+
'icarus.experiment.115' : 'd789440',
|
|
485
|
+
'icarus.experiment.874' : 'd789439',
|
|
486
|
+
'icarus.experiment.935' : 'd789438',
|
|
487
|
+
'icarus.experiment.930' : 'd789437',
|
|
488
|
+
'icarus.experiment.474' : 'd789426',
|
|
489
|
+
'icarus.experiment.933' : 'd789425',
|
|
490
|
+
'icarus.experiment.766' : 'd789424',
|
|
491
|
+
'icarus.experiment.758' : 'd789423',
|
|
492
|
+
'icarus.experiment.932' : 'd789422',
|
|
493
|
+
'icarus.experiment.934' : 'd789421',
|
|
494
|
+
'icarus.experiment.936' : 'd789420',
|
|
495
|
+
'icarus.experiment.929' : 'd789419',
|
|
496
|
+
'icarus.experiment.824' : 'd789418',
|
|
497
|
+
'icarus.experiment.478' : 'd789417',
|
|
498
|
+
'icarus.experiment.359' : 'd789416',
|
|
499
|
+
'icarus.experiment.325' : 'd789415',
|
|
500
|
+
'icarus.experiment.938' : 'd789414',
|
|
501
|
+
'icarus.experiment.931' : 'd789413',
|
|
502
|
+
'icarus.experiment.928' : 'd789412',
|
|
503
|
+
'icarus.experiment.470' : 'd789411',
|
|
504
|
+
'icarus.experiment.145' : 'd789410',
|
|
505
|
+
'icarus.experiment.767' : 'd789409',
|
|
506
|
+
'icarus.experiment.360' : 'd789408',
|
|
507
|
+
'icarus.experiment.351' : 'd789407',
|
|
508
|
+
'icarus.experiment.352' : 'd789406',
|
|
509
|
+
'icarus.experiment.362' : 'd789405',
|
|
510
|
+
'icarus.experiment.780' : 'd789404',
|
|
511
|
+
'icarus.experiment.772' : 'd789403',
|
|
512
|
+
'icarus.experiment.361' : 'd789402',
|
|
513
|
+
'icarus.experiment.781' : 'd789401',
|
|
514
|
+
'icarus.experiment.777' : 'd789400',
|
|
515
|
+
'icarus.experiment.830' : 'd789399',
|
|
516
|
+
'icarus.experiment.778' : 'd789398',
|
|
517
|
+
'icarus.experiment.765' : 'd789397',
|
|
518
|
+
'icarus.experiment.779' : 'd789396',
|
|
519
|
+
'icarus.experiment.776' : 'd789395',
|
|
520
|
+
'icarus.experiment.826' : 'd789394',
|
|
521
|
+
'icarus.experiment.762' : 'd789393',
|
|
522
|
+
'icarus.experiment.827' : 'd789392',
|
|
523
|
+
'icarus.experiment.764' : 'd789391',
|
|
524
|
+
'icarus.experiment.831' : 'd789390',
|
|
525
|
+
'icarus.experiment.510' : 'd789389',
|
|
526
|
+
'icarus.experiment.786' : 'd789388',
|
|
527
|
+
'icarus.experiment.828' : 'd789387',
|
|
528
|
+
'icarus.experiment.792' : 'd789386',
|
|
529
|
+
'icarus.experiment.373' : 'd789385',
|
|
530
|
+
'icarus.experiment.829' : 'd789384',
|
|
531
|
+
'icarus.experiment.869' : 'd789371',
|
|
532
|
+
'icarus.experiment.811' : 'd789370',
|
|
533
|
+
'icarus.experiment.81' : 'd789369',
|
|
534
|
+
'icarus.experiment.784' : 'd789368',
|
|
535
|
+
'icarus.experiment.785' : 'd789367',
|
|
536
|
+
'icarus.experiment.787' : 'd789366',
|
|
537
|
+
'icarus.experiment.794' : 'd789365',
|
|
538
|
+
'icarus.experiment.763' : 'd789364',
|
|
539
|
+
'icarus.experiment.791' : 'd789363',
|
|
540
|
+
'icarus.experiment.518' : 'd789362',
|
|
541
|
+
'icarus.experiment.771' : 'd789361',
|
|
542
|
+
'icarus.experiment.805' : 'd789360',
|
|
543
|
+
'icarus.experiment.808' : 'd789359',
|
|
544
|
+
'icarus.experiment.796' : 'd789358',
|
|
545
|
+
'icarus.experiment.795' : 'd789357',
|
|
546
|
+
'icarus.experiment.806' : 'd789356',
|
|
547
|
+
'icarus.experiment.783' : 'd789355',
|
|
548
|
+
'icarus.experiment.812' : 'd789354',
|
|
549
|
+
'icarus.experiment.809' : 'd789353',
|
|
550
|
+
'icarus.experiment.810' : 'd789352',
|
|
551
|
+
'icarus.experiment.82' : 'd789351',
|
|
552
|
+
'icarus.experiment.793' : 'd789350',
|
|
553
|
+
'icarus.experiment.790' : 'd789349',
|
|
554
|
+
'icarus.experiment.807' : 'd789348',
|
|
555
|
+
'icarus.experiment.789' : 'd789347',
|
|
556
|
+
'icarus.experiment.798' : 'd789346',
|
|
557
|
+
'icarus.experiment.800' : 'd789345',
|
|
558
|
+
'icarus.experiment.799' : 'd789344',
|
|
559
|
+
'icarus.experiment.788' : 'd789343',
|
|
560
|
+
'icarus.experiment.797' : 'd789342',
|
|
561
|
+
'icarus.experiment.782' : 'd789340',
|
|
562
|
+
'icarus.experiment.374' : 'd789339',
|
|
563
|
+
'icarus.experiment.801' : 'd789338',
|
|
564
|
+
'icarus.experiment.804' : 'd789337',
|
|
565
|
+
'icarus.experiment.802' : 'd789336',
|
|
566
|
+
'icarus.experiment.803' : 'd789335',
|
|
567
|
+
'icarus.experiment.743' : 'd789333',
|
|
568
|
+
'icarus.experiment.744' : 'd789332',
|
|
569
|
+
'icarus.experiment.735' : 'd789331',
|
|
570
|
+
'icarus.experiment.737' : 'd789330',
|
|
571
|
+
'icarus.experiment.736' : 'd789329',
|
|
572
|
+
'icarus.experiment.745' : 'd789328',
|
|
573
|
+
'icarus.experiment.746' : 'd789327',
|
|
574
|
+
'icarus.experiment.738' : 'd789307',
|
|
575
|
+
'icarus.experiment.749' : 'd789306',
|
|
576
|
+
'icarus.experiment.734' : 'd789305',
|
|
577
|
+
'icarus.experiment.666' : 'd789304',
|
|
578
|
+
'icarus.experiment.739' : 'd789303',
|
|
579
|
+
'icarus.experiment.663' : 'd789302',
|
|
580
|
+
'icarus.experiment.681' : 'd789301',
|
|
581
|
+
'icarus.experiment.690' : 'd789300',
|
|
582
|
+
'icarus.experiment.688' : 'd789299',
|
|
583
|
+
'icarus.experiment.747' : 'd789298',
|
|
584
|
+
'icarus.experiment.689' : 'd789297',
|
|
585
|
+
'icarus.experiment.684' : 'd789296',
|
|
586
|
+
'icarus.experiment.683' : 'd789295',
|
|
587
|
+
'icarus.experiment.679' : 'd789294',
|
|
588
|
+
'icarus.experiment.682' : 'd789293',
|
|
589
|
+
'icarus.experiment.685' : 'd789292',
|
|
590
|
+
'icarus.experiment.680' : 'd789291',
|
|
591
|
+
'icarus.experiment.678' : 'd789290',
|
|
592
|
+
'icarus.experiment.667' : 'd789289',
|
|
593
|
+
'icarus.experiment.674' : 'd789288',
|
|
594
|
+
'icarus.experiment.665' : 'd789280',
|
|
595
|
+
'icarus.experiment.672' : 'd789279',
|
|
596
|
+
'icarus.experiment.664' : 'd789278',
|
|
597
|
+
'icarus.experiment.676' : 'd789277',
|
|
598
|
+
'icarus.experiment.675' : 'd789276',
|
|
599
|
+
'icarus.experiment.671' : 'd789275',
|
|
600
|
+
'icarus.experiment.673' : 'd789274',
|
|
601
|
+
'icarus.experiment.748' : 'd789273',
|
|
602
|
+
'icarus.experiment.742' : 'd789272',
|
|
603
|
+
'icarus.experiment.750' : 'd789271',
|
|
604
|
+
'icarus.experiment.668' : 'd789270',
|
|
605
|
+
'icarus.experiment.662' : 'd789268',
|
|
606
|
+
'icarus.experiment.661' : 'd798267',
|
|
607
|
+
'icarus.experiment.660' : 'd789266',
|
|
608
|
+
'icarus.experiment.659' : 'd789265',
|
|
609
|
+
'icarus.experiment.669' : 'd789264',
|
|
610
|
+
'icarus.experiment.740' : 'd789263',
|
|
611
|
+
'icarus.experiment.741' : 'd789262',
|
|
612
|
+
'icarus.experiment.162' : 'd789244',
|
|
613
|
+
'icarus.experiment.687' : 'd789243',
|
|
614
|
+
'icarus.experiment.144' : 'd789242',
|
|
615
|
+
'icarus.experiment.837' : 'd789241',
|
|
616
|
+
'icarus.experiment.836' : 'd789240',
|
|
617
|
+
'icarus.experiment.835' : 'd789239',
|
|
618
|
+
'icarus.experiment.834' : 'd789238',
|
|
619
|
+
'icarus.experiment.833' : 'd789237',
|
|
620
|
+
'icarus.experiment.845' : 'd789236',
|
|
621
|
+
'icarus.experiment.844' : 'd789235',
|
|
622
|
+
'icarus.experiment.841' : 'd789234',
|
|
623
|
+
'icarus.experiment.842' : 'd789233',
|
|
624
|
+
'icarus.experiment.847' : 'd789232',
|
|
625
|
+
'icarus.experiment.843' : 'd789231',
|
|
626
|
+
'icarus.experiment.840' : 'd789230',
|
|
627
|
+
'icarus.experiment.848' : 'd789229',
|
|
628
|
+
'icarus.experiment.832' : 'd789228',
|
|
629
|
+
'icarus.experiment.846' : 'd789227',
|
|
630
|
+
'icarus.experiment.838' : 'd789226',
|
|
631
|
+
'icarus.experiment.839' : 'd789225',
|
|
632
|
+
'icarus.experiment.907' : 'd789224',
|
|
633
|
+
'icarus.experiment.754' : 'd789223',
|
|
634
|
+
'icarus.experiment.773' : 'd789222',
|
|
635
|
+
'icarus.experiment.306' : 'd789221',
|
|
636
|
+
'icarus.experiment.310' : 'd789220',
|
|
637
|
+
'icarus.experiment.305' : 'd789219',
|
|
638
|
+
'icarus.experiment.813' : 'd789218',
|
|
639
|
+
'icarus.experiment.304' : 'd789217',
|
|
640
|
+
'icarus.experiment.814' : 'd789216',
|
|
641
|
+
'icarus.experiment.308' : 'd789215',
|
|
642
|
+
'icarus.experiment.459' : 'd789214',
|
|
643
|
+
'icarus.experiment.311' : 'd789213',
|
|
644
|
+
'icarus.experiment.468' : 'd789212',
|
|
645
|
+
'icarus.experiment.463' : 'd789211',
|
|
646
|
+
'icarus.experiment.460' : 'd789210',
|
|
647
|
+
'icarus.experiment.465' : 'd789209',
|
|
648
|
+
'icarus.experiment.462' : 'd789208',
|
|
649
|
+
'icarus.experiment.467' : 'd789207',
|
|
650
|
+
'icarus.experiment.464' : 'd789206',
|
|
651
|
+
'icarus.experiment.466' : 'd789205',
|
|
652
|
+
'icarus.experiment.227' : 'd789191',
|
|
653
|
+
'icarus.experiment.318' : 'd789190',
|
|
654
|
+
'icarus.experiment.231' : 'd789189',
|
|
655
|
+
'icarus.experiment.312' : 'd789188',
|
|
656
|
+
'icarus.experiment.307' : 'd789187',
|
|
657
|
+
'icarus.experiment.229' : 'd789186',
|
|
658
|
+
'icarus.experiment.230' : 'd789185',
|
|
659
|
+
'icarus.experiment.309' : 'd789184',
|
|
660
|
+
'icarus.experiment.316' : 'd789183',
|
|
661
|
+
'icarus.experiment.315' : 'd789182',
|
|
662
|
+
'icarus.experiment.382' : 'd789181',
|
|
663
|
+
'icarus.experiment.167' : 'd789180',
|
|
664
|
+
'icarus.experiment.168' : 'd789179',
|
|
665
|
+
'icarus.experiment.317' : 'd789178',
|
|
666
|
+
'icarus.experiment.379' : 'd789177',
|
|
667
|
+
'icarus.experiment.314' : 'd789176',
|
|
668
|
+
'icarus.experiment.164' : 'd789175',
|
|
669
|
+
'icarus.experiment.163' : 'd789174',
|
|
670
|
+
'icarus.experiment.165' : 'd789173',
|
|
671
|
+
'icarus.experiment.143' : 'd789172',
|
|
672
|
+
'icarus.experiment.321' : 'd789171',
|
|
673
|
+
'icarus.experiment.166' : 'd789170',
|
|
674
|
+
'icarus.experiment.319' : 'd789169',
|
|
675
|
+
'icarus.experiment.161' : 'd789168',
|
|
676
|
+
'icarus.experiment.313' : 'd789167',
|
|
677
|
+
'icarus.experiment.147' : 'd789166',
|
|
678
|
+
'icarus.experiment.218' : 'd789165',
|
|
679
|
+
'icarus.experiment.217' : 'd789164',
|
|
680
|
+
'icarus.experiment.216' : 'd789163',
|
|
681
|
+
'icarus.experiment.213' : 'd789162',
|
|
682
|
+
'icarus.experiment.214' : 'd789161',
|
|
683
|
+
'icarus.experiment.320' : 'd789160',
|
|
684
|
+
'icarus.experiment.226' : 'd789159',
|
|
685
|
+
'icarus.experiment.109' : 'd789158',
|
|
686
|
+
'icarus.experiment.228' : 'd789157',
|
|
687
|
+
'icarus.experiment.108' : 'd789156',
|
|
688
|
+
'icarus.experiment.180' : 'd789153',
|
|
689
|
+
'icarus.experiment.77' : 'd789152',
|
|
690
|
+
'icarus.experiment.211' : 'd789151',
|
|
691
|
+
'icarus.experiment.187' : 'd789150',
|
|
692
|
+
'icarus.experiment.188' : 'd789149',
|
|
693
|
+
'icarus.experiment.185' : 'd789148',
|
|
694
|
+
'icarus.experiment.205' : 'd789147',
|
|
695
|
+
'icarus.experiment.189' : 'd789146',
|
|
696
|
+
'icarus.experiment.209' : 'd789145',
|
|
697
|
+
'icarus.experiment.186' : 'd789144',
|
|
698
|
+
'icarus.experiment.210' : 'd789143',
|
|
699
|
+
'icarus.experiment.206' : 'd789142',
|
|
700
|
+
'icarus.experiment.208' : 'd789141',
|
|
701
|
+
'icarus.experiment.177' : 'd789136',
|
|
702
|
+
'icarus.experiment.182' : 'd789133',
|
|
703
|
+
'icarus.experiment.184' : 'd789132',
|
|
704
|
+
'icarus.experiment.178' : 'd789131',
|
|
705
|
+
'icarus.experiment.183' : 'd789130',
|
|
706
|
+
'icarus.experiment.179' : 'd789129',
|
|
707
|
+
'icarus.experiment.196' : 'd789128',
|
|
708
|
+
'icarus.experiment.190' : 'd789127',
|
|
709
|
+
'icarus.experiment.181' : 'd789126',
|
|
710
|
+
'icarus.experiment.192' : 'd789125',
|
|
711
|
+
'icarus.experiment.193' : 'd789124',
|
|
712
|
+
'icarus.experiment.195' : 'd789123',
|
|
713
|
+
'icarus.experiment.200' : 'd789122',
|
|
714
|
+
'icarus.experiment.199' : 'd789121',
|
|
715
|
+
'icarus.experiment.191' : 'd789120',
|
|
716
|
+
'icarus.experiment.194' : 'd789119',
|
|
717
|
+
'icarus.experiment.198' : 'd789116',
|
|
718
|
+
'icarus.experiment.107' : 'd789115',
|
|
719
|
+
'icarus.experiment.197' : 'd789114'
|
|
720
|
+
}
|
|
721
|
+
|
|
722
|
+
ALLIDS = list(DSIDS.keys())
|
|
723
|
+
|
|
724
|
+
WFILES = {}
|
|
725
|
+
|
|
726
|
+
#
|
|
727
|
+
# main function to run this program
|
|
728
|
+
#
|
|
729
|
+
def main():
|
|
730
|
+
|
|
731
|
+
params = {} # array of input values
|
|
732
|
+
argv = sys.argv[1:]
|
|
733
|
+
opt = None
|
|
734
|
+
|
|
735
|
+
for arg in argv:
|
|
736
|
+
if arg == "-b":
|
|
737
|
+
PgLOG.PGLOG['BCKGRND'] = 1
|
|
738
|
+
elif re.match(r'^-[msNy]$', arg):
|
|
739
|
+
opt = arg[1]
|
|
740
|
+
params[opt] = []
|
|
741
|
+
elif re.match(r'^-', arg):
|
|
742
|
+
PgLOG.pglog(arg + ": Invalid Option", PgLOG.LGWNEX)
|
|
743
|
+
elif opt:
|
|
744
|
+
params[opt].append(arg)
|
|
745
|
+
else:
|
|
746
|
+
PgLOG.pglog(arg + ": Value passed in without leading option", PgLOG.LGWNEX)
|
|
747
|
+
|
|
748
|
+
if not opt:
|
|
749
|
+
PgLOG.show_usage('fillgdexusage')
|
|
750
|
+
elif 's' not in params:
|
|
751
|
+
PgLOG.pglog("-s: Missing dataset short name to gather GDEX metrics", PgLOG.LGWNEX)
|
|
752
|
+
elif len(params) < 2:
|
|
753
|
+
PgLOG.pglog("-(m|N|y): Missing Month, NumberDays or Year to gather GDEX metrics", PgLOG.LGWNEX)
|
|
754
|
+
|
|
755
|
+
|
|
756
|
+
PgLOG.cmdlog("fillgdexusage {}".format(' '.join(argv)))
|
|
757
|
+
dranges = get_date_ranges(params)
|
|
758
|
+
dsids = get_dataset_ids(params['s'])
|
|
759
|
+
if dranges and dsids: fill_gdex_usages(dsids, dranges)
|
|
760
|
+
PgLOG.pglog(None, PgLOG.LOGWRN|PgLOG.SNDEML) # send email out if any
|
|
761
|
+
|
|
762
|
+
sys.exit(0)
|
|
763
|
+
|
|
764
|
+
#
|
|
765
|
+
# connect to the gdex database esg-production
|
|
766
|
+
#
|
|
767
|
+
def gdex_dbname():
|
|
768
|
+
PgDBI.set_scname('gdex-production', 'metrics', 'gateway-reader', None, 'sagedbprodalma.ucar.edu')
|
|
769
|
+
|
|
770
|
+
#
|
|
771
|
+
# get datasets
|
|
772
|
+
#
|
|
773
|
+
def get_dataset_ids(dsnames):
|
|
774
|
+
|
|
775
|
+
gdex_dbname()
|
|
776
|
+
dsids = []
|
|
777
|
+
tbname = 'metadata.dataset'
|
|
778
|
+
for dsname in dsnames:
|
|
779
|
+
if re.match(r'^all$', dsname, re.I): return get_dataset_ids(ALLIDS)
|
|
780
|
+
if dsname not in DSIDS:
|
|
781
|
+
PgLOG.pglog(dsname + ": Unknown GDEX dataset short name", PgLOG.LOGWRN)
|
|
782
|
+
continue
|
|
783
|
+
bt = tm()
|
|
784
|
+
pgrec = PgDBI.pgget(tbname, 'id', "short_name = '{}'".format(dsname))
|
|
785
|
+
if not (pgrec and pgrec['id']): continue
|
|
786
|
+
rdaid = DSIDS[dsname]
|
|
787
|
+
strids = "{}-{}".format(dsname, rdaid)
|
|
788
|
+
gdexid = pgrec['id']
|
|
789
|
+
gdexids = [gdexid]
|
|
790
|
+
ccnt = 1
|
|
791
|
+
ccnt += recursive_dataset_ids(gdexid, gdexids)
|
|
792
|
+
dsids.append([dsname, rdaid, gdexids, strids])
|
|
793
|
+
rmsg = PgLOG.seconds_to_string_time(tm() - bt)
|
|
794
|
+
PgLOG.pglog("{}: Found {} GDEX dsid/subdsids in {} at {}".format(strids, ccnt, rmsg, PgLOG.current_datetime()), PgLOG.LOGWRN)
|
|
795
|
+
|
|
796
|
+
if not dsids: PgLOG.pglog("No Dataset Id identified to gather GDEX metrics", PgLOG.LOGWRN)
|
|
797
|
+
|
|
798
|
+
return dsids
|
|
799
|
+
|
|
800
|
+
#
|
|
801
|
+
# get gdexids recursivley
|
|
802
|
+
#
|
|
803
|
+
def recursive_dataset_ids(pgdexid, gdexids):
|
|
804
|
+
|
|
805
|
+
tbname = 'metadata.dataset'
|
|
806
|
+
pgrecs = PgDBI.pgmget(tbname, 'id', "parent_dataset_id = '{}'".format(pgdexid))
|
|
807
|
+
if not pgrecs: return 0
|
|
808
|
+
|
|
809
|
+
ccnt = 0
|
|
810
|
+
for gdexid in pgrecs['id']:
|
|
811
|
+
if gdexid in gdexids: continue
|
|
812
|
+
gdexids.append(gdexid)
|
|
813
|
+
ccnt += 1
|
|
814
|
+
ccnt += recursive_dataset_ids(gdexid, gdexids)
|
|
815
|
+
|
|
816
|
+
return ccnt
|
|
817
|
+
|
|
818
|
+
#
|
|
819
|
+
# get the date ranges for given condition
|
|
820
|
+
#
|
|
821
|
+
def get_date_ranges(inputs):
|
|
822
|
+
|
|
823
|
+
dranges = []
|
|
824
|
+
for opt in inputs:
|
|
825
|
+
for input in inputs[opt]:
|
|
826
|
+
# get date range
|
|
827
|
+
dates = []
|
|
828
|
+
if opt == 'N':
|
|
829
|
+
dates.append(PgUtil.adddate(USAGE['CDATE'], 0, 0, -int(input)))
|
|
830
|
+
dates.append(USAGE['CDATE'])
|
|
831
|
+
elif opt == 'm':
|
|
832
|
+
tms = input.split('-')
|
|
833
|
+
dates.append(PgUtil.fmtdate(int(tms[0]), int(tms[1]), 1))
|
|
834
|
+
dates.append(PgUtil.enddate(dates[0], 0, 'M'))
|
|
835
|
+
elif opt == 'y':
|
|
836
|
+
dates.append(input + "-01-01")
|
|
837
|
+
dates.append(input + "-12-31")
|
|
838
|
+
if dates: dranges.append(dates)
|
|
839
|
+
|
|
840
|
+
return dranges
|
|
841
|
+
|
|
842
|
+
#
|
|
843
|
+
# get file download records for given dsid
|
|
844
|
+
#
|
|
845
|
+
def get_dsid_records(gdexids, dates, strids):
|
|
846
|
+
|
|
847
|
+
gdex_dbname()
|
|
848
|
+
tbname = 'metrics.file_download'
|
|
849
|
+
fields = ('date_completed, remote_address, logical_file_size, logical_file_name, file_access_point_uri, user_agent_name, bytes_sent, '
|
|
850
|
+
'subset_file_size, range_request, dataset_file_size, dataset_file_name, dataset_file_file_access_point_uri')
|
|
851
|
+
dscnt = len(gdexids)
|
|
852
|
+
dscnd = "dataset_id "
|
|
853
|
+
if dscnt == 1:
|
|
854
|
+
dscnd += "= '{}'".format(gdexids[0])
|
|
855
|
+
else:
|
|
856
|
+
dscnd += "IN ('" + "','".join(gdexids) + "')"
|
|
857
|
+
dtcnd = "date_completed BETWEEN '{} 00:00:00' AND '{} 23:59:59'".format(dates[0], dates[1])
|
|
858
|
+
cond = "{} AND {} ORDER BY date_completed".format(dscnd, dtcnd)
|
|
859
|
+
PgLOG.pglog("{}: Query for {} GDEX dsid/subdsids between {} and {} at {}".format(strids, dscnt, dates[0], dates[1], PgLOG.current_datetime()), PgLOG.LOGWRN)
|
|
860
|
+
pgrecs = PgDBI.pgmget(tbname, fields, cond)
|
|
861
|
+
PgDBI.dssdb_dbname()
|
|
862
|
+
|
|
863
|
+
return pgrecs
|
|
864
|
+
|
|
865
|
+
#
|
|
866
|
+
# Fill TDS usages into table dssdb.tdsusage from gdex access records
|
|
867
|
+
#
|
|
868
|
+
def fill_gdex_usages(dsids, dranges):
|
|
869
|
+
|
|
870
|
+
allcnt = awcnt = atcnt = lcnt = 0
|
|
871
|
+
for dates in dranges:
|
|
872
|
+
for dsid in dsids:
|
|
873
|
+
lcnt += 1
|
|
874
|
+
dsname = dsid[0]
|
|
875
|
+
rdaid = dsid[1]
|
|
876
|
+
gdexids = dsid[2]
|
|
877
|
+
strids = dsid[3]
|
|
878
|
+
bt = tm()
|
|
879
|
+
pgrecs = get_dsid_records(gdexids, dates, strids)
|
|
880
|
+
pgcnt = len(pgrecs['dataset_file_name']) if pgrecs else 0
|
|
881
|
+
if pgcnt == 0:
|
|
882
|
+
PgLOG.pglog("{}: No record found to gather GDEX usage between {} and {}".format(strids, dates[0], dates[1]), PgLOG.LOGWRN)
|
|
883
|
+
continue
|
|
884
|
+
rmsg = PgLOG.seconds_to_string_time(tm() - bt)
|
|
885
|
+
PgLOG.pglog("{}: Got {} records in {} for processing GDEX usage at {}".format(strids, pgcnt, rmsg, PgLOG.current_datetime()), PgLOG.LOGWRN)
|
|
886
|
+
tcnt = wcnt = 0
|
|
887
|
+
pwkey = wrec = cdate = None
|
|
888
|
+
trecs = {}
|
|
889
|
+
bt = tm()
|
|
890
|
+
for i in range(pgcnt):
|
|
891
|
+
if (i+1)%20000 == 0:
|
|
892
|
+
PgLOG.pglog("{}/{}/{} GDEX/TDS/WEB records processed to add".format(i, tcnt, wcnt), PgLOG.WARNLG)
|
|
893
|
+
|
|
894
|
+
pgrec = PgUtil.onerecord(pgrecs, i)
|
|
895
|
+
dsize = pgrec['bytes_sent']
|
|
896
|
+
if not dsize: continue
|
|
897
|
+
(year, quarter, date, time) = get_record_date_time(pgrec['date_completed'])
|
|
898
|
+
url = pgrec['dataset_file_file_access_point_uri']
|
|
899
|
+
if not url: url = pgrec['file_access_point_uri']
|
|
900
|
+
ip = pgrec['remote_address']
|
|
901
|
+
engine = pgrec['user_agent_name']
|
|
902
|
+
wfile = pgrec['dataset_file_name']
|
|
903
|
+
if not wfile: wfile = pgrec['logic_file_name']
|
|
904
|
+
wfrec = get_wfile_record(rdaid, wfile)
|
|
905
|
+
if not wfrec: continue
|
|
906
|
+
dsid = wfrec['dsid']
|
|
907
|
+
fsize = pgrec['dataset_file_size']
|
|
908
|
+
if not fsize: fsize = pgrec['logic_file_size']
|
|
909
|
+
method = 'GDEX'
|
|
910
|
+
if pgrec['subset_file_size'] or pgrec['range_request'] or dsize < fsize:
|
|
911
|
+
wkey = "{}:{}:{}".format(ip, dsid, wfile)
|
|
912
|
+
else:
|
|
913
|
+
wkey = None
|
|
914
|
+
|
|
915
|
+
if wrec:
|
|
916
|
+
if wkey == pwkey:
|
|
917
|
+
wrec['size'] += dsize
|
|
918
|
+
continue
|
|
919
|
+
wcnt += add_webfile_usage(year, wrec)
|
|
920
|
+
wrec = {'ip' : ip, 'dsid' : dsid, 'wid' : wfrec['wid'], 'date' : date,
|
|
921
|
+
'time' : time, 'quarter' : quarter, 'size' : dsize,
|
|
922
|
+
'locflag' : 'C', 'method' : method}
|
|
923
|
+
pwkey = wkey
|
|
924
|
+
if not pwkey:
|
|
925
|
+
wcnt += add_webfile_usage(year, wrec)
|
|
926
|
+
wrec = None
|
|
927
|
+
|
|
928
|
+
if wrec: wcnt += add_webfile_usage(year, wrec)
|
|
929
|
+
atcnt += tcnt
|
|
930
|
+
awcnt += wcnt
|
|
931
|
+
allcnt += pgcnt
|
|
932
|
+
rmsg = PgLOG.seconds_to_string_time(tm() - bt)
|
|
933
|
+
PgLOG.pglog("{}: {}/{} TDS/WEB usage records added for {} GDEX entries in {}".format(strids, atcnt, awcnt, allcnt, rmsg), PgLOG.LOGWRN)
|
|
934
|
+
|
|
935
|
+
def get_record_date_time(ctime):
|
|
936
|
+
|
|
937
|
+
ms = re.search(r'^(\d+)-(\d+)-(\d+) (\d\d:\d\d:\d\d)', str(ctime))
|
|
938
|
+
if ms:
|
|
939
|
+
y = ms.group(1)
|
|
940
|
+
m = int(ms.group(2))
|
|
941
|
+
d = ms.group(3)
|
|
942
|
+
q = 1 + int((m-1)/3)
|
|
943
|
+
t = ms.group(4)
|
|
944
|
+
return (y, q, "{}-{:02}-{}".format(y, m, d), t)
|
|
945
|
+
else:
|
|
946
|
+
PgLOG.pglog(str(ctime) + ": Invalid time format", PgLOG.LGEREX)
|
|
947
|
+
|
|
948
|
+
#
|
|
949
|
+
# Fill usage of a single online data file into table dssdb.wusage of DSS PgSQL database
|
|
950
|
+
#
|
|
951
|
+
def add_webfile_usage(year, logrec):
|
|
952
|
+
|
|
953
|
+
table = "{}_{}".format(USAGE['WEBTBL'], year)
|
|
954
|
+
cdate = logrec['date']
|
|
955
|
+
ip = logrec['ip']
|
|
956
|
+
cond = "wid = {} AND method = '{}' AND date_read = '{}' AND time_read = '{}'".format(logrec['wid'], logrec['method'], cdate, logrec['time'])
|
|
957
|
+
if PgDBI.pgget(table, "", cond, PgLOG.LOGWRN): return 0
|
|
958
|
+
|
|
959
|
+
wurec = PgIPInfo.get_wuser_record(ip, cdate)
|
|
960
|
+
if not wurec: return 0
|
|
961
|
+
|
|
962
|
+
record = {'wid' : logrec['wid'], 'dsid' : logrec['dsid']}
|
|
963
|
+
record['wuid_read'] = wurec['wuid']
|
|
964
|
+
record['date_read'] = cdate
|
|
965
|
+
record['time_read'] = logrec['time']
|
|
966
|
+
record['size_read'] = logrec['size']
|
|
967
|
+
record['method'] = logrec['method']
|
|
968
|
+
record['locflag'] = logrec['locflag']
|
|
969
|
+
record['ip'] = ip
|
|
970
|
+
record['quarter'] = logrec['quarter']
|
|
971
|
+
|
|
972
|
+
if add_web_allusage(year, logrec, wurec):
|
|
973
|
+
return PgDBI.add_yearly_wusage(year, record)
|
|
974
|
+
else:
|
|
975
|
+
return 0
|
|
976
|
+
|
|
977
|
+
def add_web_allusage(year, logrec, wurec):
|
|
978
|
+
|
|
979
|
+
pgrec = {'source' : 'G'}
|
|
980
|
+
pgrec['email'] = wurec['email']
|
|
981
|
+
pgrec['org_type'] = wurec['org_type']
|
|
982
|
+
pgrec['country'] = wurec['country']
|
|
983
|
+
pgrec['region'] = wurec['region']
|
|
984
|
+
pgrec['dsid'] = logrec['dsid']
|
|
985
|
+
pgrec['date'] = logrec['date']
|
|
986
|
+
pgrec['quarter'] = logrec['quarter']
|
|
987
|
+
pgrec['time'] = logrec['time']
|
|
988
|
+
pgrec['size'] = logrec['size']
|
|
989
|
+
pgrec['method'] = logrec['method']
|
|
990
|
+
pgrec['ip'] = logrec['ip']
|
|
991
|
+
return PgDBI.add_yearly_allusage(year, pgrec)
|
|
992
|
+
|
|
993
|
+
#
|
|
994
|
+
# return wfile.wid upon success, 0 otherwise
|
|
995
|
+
#
|
|
996
|
+
def get_wfile_record(dsid, wfile):
|
|
997
|
+
|
|
998
|
+
wkey = "{}{}".format(dsid, wfile)
|
|
999
|
+
if wkey in WFILES: return WFILES[wkey]
|
|
1000
|
+
wfcond = "wfile LIKE '%{}'".format(wfile)
|
|
1001
|
+
pgrec = None
|
|
1002
|
+
pgrec = PgSplit.pgget_wfile(dsid, "wid", wfcond)
|
|
1003
|
+
if pgrec:
|
|
1004
|
+
pgrec['dsid'] = dsid
|
|
1005
|
+
wkey = "{}{}".format(dsid, wfile)
|
|
1006
|
+
WFILES[wkey] = pgrec
|
|
1007
|
+
return pgrec
|
|
1008
|
+
|
|
1009
|
+
pgrec = PgDBI.pgget("wfile_delete", "wid, dsid", "{} AND dsid = '{}'".format(wfcond, dsid))
|
|
1010
|
+
if not pgrec:
|
|
1011
|
+
mvrec = PgDBI.pgget("wmove", "wid, dsid", wfcond)
|
|
1012
|
+
if mvrec:
|
|
1013
|
+
pgrec = PgSplit.pgget_wfile(mvrec['dsid'], "wid", "wid = {}".format(pgrec['wid']))
|
|
1014
|
+
if pgrec: pgrec['dsid'] = mvrec['dsid']
|
|
1015
|
+
|
|
1016
|
+
WFILES[wkey] = pgrec
|
|
1017
|
+
return pgrec
|
|
1018
|
+
|
|
1019
|
+
#
|
|
1020
|
+
# call main() to start program
|
|
1021
|
+
#
|
|
1022
|
+
if __name__ == "__main__": main()
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
|
|
2
|
+
Retrieves GDEX usage information from GDEX metrics database to
|
|
3
|
+
fill 'webusage' in PostgreSQL database 'rdadb'.
|
|
4
|
+
|
|
5
|
+
Usage: fillgdexusage [-b] -s DatasetShortNames [-m MonthList] [-N NumberDays] [-y YearList]
|
|
6
|
+
|
|
7
|
+
select option -s and one of the options, -m, -N or -y each time to run
|
|
8
|
+
this application.
|
|
9
|
+
|
|
10
|
+
- Option -b, log process information into logfile only;
|
|
11
|
+
|
|
12
|
+
- Option -s, retrieve usage info for given dataset short names;
|
|
13
|
+
|
|
14
|
+
- Option -m, retrieve usage info in given months (YYYY-MM);
|
|
15
|
+
|
|
16
|
+
- Option -N, retrieve usage info in recent Number of days;
|
|
17
|
+
|
|
18
|
+
- Option -y, retrieve usage info in given years (YYYY).
|
rda_python_metrics/pgperson.py
CHANGED
rda_python_metrics/pgusername.py
CHANGED
|
@@ -1,14 +1,16 @@
|
|
|
1
|
-
rda_python_metrics/PgIPInfo.py,sha256
|
|
1
|
+
rda_python_metrics/PgIPInfo.py,sha256=CfcnJxD2zHyAhemhTisdnPB72wHwE8MxS0EH4EAKnfE,9151
|
|
2
2
|
rda_python_metrics/PgView.py,sha256=r6otb3DjfiaQJdg0z8bZQAOlhr4JnrXJzp9wgWh_8qQ,24369
|
|
3
3
|
rda_python_metrics/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
|
4
4
|
rda_python_metrics/fillawsusage.py,sha256=SZH9G4jruKvavqSr9DiVmHVW929E60tJGUkv8CK3GQI,8152
|
|
5
5
|
rda_python_metrics/fillawsusage.usg,sha256=pD_nYTfg1so9nvVEyPSWdgKvb9gWdtfHJAs3RsT2MMU,609
|
|
6
|
-
rda_python_metrics/fillcdgusage.py,sha256
|
|
6
|
+
rda_python_metrics/fillcdgusage.py,sha256=OYf4cJXqkHUgptvOa9TLmBRniGDCxr7KO6t9-JIeSuo,16381
|
|
7
7
|
rda_python_metrics/fillcdgusage.usg,sha256=5lkd4Zdi72nQcha-JtbaLnxl66V4wBDykKwQtUOtMrw,667
|
|
8
8
|
rda_python_metrics/fillcodusage.py,sha256=_RUcgY1Cf17dSjidhQCMOwVaPO1VL26s3bYGT8oSPWU,8322
|
|
9
9
|
rda_python_metrics/fillcodusage.usg,sha256=3B5IkQ4uwylqY8uEfUdnZ_MXqhYudeylMp5ulhUGXH8,678
|
|
10
10
|
rda_python_metrics/fillcountry.py,sha256=7i5LNi3scRoyRCT6t7aeNTGKOpxzJ2mA9tnvUqje2AU,2314
|
|
11
11
|
rda_python_metrics/fillendtime.py,sha256=skZttlpoY19g0dGwqGQI8t_1YPPTPEXwg3EfNlfL90I,2533
|
|
12
|
+
rda_python_metrics/fillgdexusage.py,sha256=21k19oUxcTOBv-Bu3az3mZyDYH2FnoJtRZrFvuODU6w,38085
|
|
13
|
+
rda_python_metrics/fillgdexusage.usg,sha256=mVYtK0pIYmvma0skT-wXM-NOEpkN_i3E61UdWgQWLfs,648
|
|
12
14
|
rda_python_metrics/fillglobususage.py,sha256=ahz8XnnJdD_AbSYqJ34lWmDuzws_-SNmCR8QE20aovA,8539
|
|
13
15
|
rda_python_metrics/fillglobususage.usg,sha256=1GgmCP22IQZdADwL5Mmkz3v8Ws-G7U3teQ1AxRJfV_4,637
|
|
14
16
|
rda_python_metrics/fillipinfo.py,sha256=BrboxAIs8Q5jhz_4GYW_GibMT5GyEbnQkYfFR8hc_yo,6747
|
|
@@ -25,9 +27,9 @@ rda_python_metrics/filluser.py,sha256=CvaMRaUPaR9nxJAExkLTb3Ci4sD7RQMOpWhWJdbyMF
|
|
|
25
27
|
rda_python_metrics/filluser.usg,sha256=Xgqi0QwA9-4jpYj3L4Q4TISpVwRlsomt2G7T0oYAFak,520
|
|
26
28
|
rda_python_metrics/logarch.py,sha256=cMty6vI5t4oBAKnA4VYZjEvfYoj_8cSUuW0orPOJOZQ,13721
|
|
27
29
|
rda_python_metrics/logarch.usg,sha256=6ksFxu_-lzhRpQE2YFAqgGVhwFP4PTweCbFBuGhhICU,960
|
|
28
|
-
rda_python_metrics/pgperson.py,sha256=
|
|
30
|
+
rda_python_metrics/pgperson.py,sha256=4XTDnVBFyF9drzx2I_DGCxQ-8I1bpclqTuQvOx3wST4,2088
|
|
29
31
|
rda_python_metrics/pgsyspath.py,sha256=DZhFp-r-LzN0qrHtfdCYfnGDnmD_R4ufuEduk0_vRIw,1039
|
|
30
|
-
rda_python_metrics/pgusername.py,sha256=
|
|
32
|
+
rda_python_metrics/pgusername.py,sha256=VoNJfXBsyzdfz49qwCypnUsqcDm4cUaqOYViJ-jzaKI,1265
|
|
31
33
|
rda_python_metrics/viewallusage.py,sha256=DlAvY2bieJWrrrYMPhCkKWgqdfSN492_LZCS95BhY0A,15753
|
|
32
34
|
rda_python_metrics/viewallusage.usg,sha256=Vfwc6aNIkpe1FBdBsV2htyUESgmx6ODhMMwAb1NX8-0,10297
|
|
33
35
|
rda_python_metrics/viewcheckusage.py,sha256=HougqjDAOVG6pYglFjyHQ-UdLBcYe7v_jzU1-80RqFA,12996
|
|
@@ -44,9 +46,9 @@ rda_python_metrics/viewwebfile.py,sha256=HSMNkQQawonu6W3blV7g9UbJuNy9VAOn9COqgmj
|
|
|
44
46
|
rda_python_metrics/viewwebfile.usg,sha256=lTNi8Yu8BUJuExEDJX-vsJyWUSUIQTS-DiiBEVFo33s,10054
|
|
45
47
|
rda_python_metrics/viewwebusage.py,sha256=ES2lI8NaCeCpTGi94HU-cDRBxHMiUBbplyYsZf2KqF0,16650
|
|
46
48
|
rda_python_metrics/viewwebusage.usg,sha256=OVDZ78p87E3HLW34ZhasNJ7Zmw8XXjmZPPWZfRhPLXo,9936
|
|
47
|
-
rda_python_metrics-1.0.
|
|
48
|
-
rda_python_metrics-1.0.
|
|
49
|
-
rda_python_metrics-1.0.
|
|
50
|
-
rda_python_metrics-1.0.
|
|
51
|
-
rda_python_metrics-1.0.
|
|
52
|
-
rda_python_metrics-1.0.
|
|
49
|
+
rda_python_metrics-1.0.31.dist-info/licenses/LICENSE,sha256=1dck4EAQwv8QweDWCXDx-4Or0S8YwiCstaso_H57Pno,1097
|
|
50
|
+
rda_python_metrics-1.0.31.dist-info/METADATA,sha256=lRRCyw5ONiMv2K-yq8ijEyvUgqaSL25htuBRuVIJlJM,761
|
|
51
|
+
rda_python_metrics-1.0.31.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
52
|
+
rda_python_metrics-1.0.31.dist-info/entry_points.txt,sha256=YfFLzlE3rdufSV471VsDnfYptnt1lR08aSrxPXlKqlY,1185
|
|
53
|
+
rda_python_metrics-1.0.31.dist-info/top_level.txt,sha256=aoBgbR_o70TP0QmMW0U6inRHYtfKld47OBmnWnLnDOs,19
|
|
54
|
+
rda_python_metrics-1.0.31.dist-info/RECORD,,
|
|
File without changes
|
{rda_python_metrics-1.0.28.dist-info → rda_python_metrics-1.0.31.dist-info}/entry_points.txt
RENAMED
|
File without changes
|
{rda_python_metrics-1.0.28.dist-info → rda_python_metrics-1.0.31.dist-info}/licenses/LICENSE
RENAMED
|
File without changes
|
|
File without changes
|