rda-python-metrics 1.0.32__tar.gz → 1.0.46__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. {rda_python_metrics-1.0.32/src/rda_python_metrics.egg-info → rda_python_metrics-1.0.46}/PKG-INFO +2 -1
  2. {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/pyproject.toml +7 -3
  3. rda_python_metrics-1.0.46/src/rda_python_metrics/fillawsusage.py +210 -0
  4. {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/fillawsusage.usg +1 -1
  5. {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/fillcdgusage.py +32 -27
  6. {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/fillgdexusage.py +51 -37
  7. {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/fillglobususage.py +9 -11
  8. {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/fillglobususage.usg +4 -2
  9. {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/fillosdfusage.py +57 -89
  10. {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/fillosdfusage.usg +1 -1
  11. {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/fillrdadb.py +1 -2
  12. rda_python_metrics-1.0.46/src/rda_python_metrics/fillzenodousage.py +408 -0
  13. {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/logarch.py +15 -8
  14. rda_python_metrics-1.0.46/src/rda_python_metrics/viewawsusage.py +321 -0
  15. rda_python_metrics-1.0.46/src/rda_python_metrics/viewawsusage.usg +190 -0
  16. rda_python_metrics-1.0.46/src/rda_python_metrics/viewosdfusage.py +321 -0
  17. rda_python_metrics-1.0.46/src/rda_python_metrics/viewosdfusage.usg +190 -0
  18. {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46/src/rda_python_metrics.egg-info}/PKG-INFO +2 -1
  19. {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics.egg-info/SOURCES.txt +5 -0
  20. {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics.egg-info/entry_points.txt +4 -1
  21. {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics.egg-info/requires.txt +1 -0
  22. rda_python_metrics-1.0.32/src/rda_python_metrics/fillawsusage.py +0 -254
  23. {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/LICENSE +0 -0
  24. {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/MANIFEST.in +0 -0
  25. {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/README.md +0 -0
  26. {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/setup.cfg +0 -0
  27. {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/PgIPInfo.py +0 -0
  28. {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/PgView.py +0 -0
  29. {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/__init__.py +0 -0
  30. {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/fillcdgusage.usg +0 -0
  31. {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/fillcodusage.py +0 -0
  32. {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/fillcodusage.usg +0 -0
  33. {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/fillcountry.py +0 -0
  34. {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/fillendtime.py +0 -0
  35. {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/fillgdexusage.usg +0 -0
  36. {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/fillipinfo.py +0 -0
  37. {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/fillipinfo.usg +0 -0
  38. {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/filloneorder.py +0 -0
  39. {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/filloneorder.usg +0 -0
  40. {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/fillrdadb.usg +0 -0
  41. {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/filltdsusage.py +0 -0
  42. {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/filltdsusage.usg +0 -0
  43. {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/filluser.py +0 -0
  44. {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/filluser.usg +0 -0
  45. {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/logarch.usg +0 -0
  46. {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/pgperson.py +0 -0
  47. {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/pgsyspath.py +0 -0
  48. {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/pgusername.py +0 -0
  49. {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/viewallusage.py +0 -0
  50. {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/viewallusage.usg +0 -0
  51. {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/viewcheckusage.py +0 -0
  52. {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/viewcheckusage.usg +0 -0
  53. {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/viewcodusage.py +0 -0
  54. {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/viewcodusage.usg +0 -0
  55. {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/viewordusage.py +0 -0
  56. {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/viewordusage.usg +0 -0
  57. {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/viewrqstusage.py +0 -0
  58. {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/viewrqstusage.usg +0 -0
  59. {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/viewtdsusage.py +0 -0
  60. {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/viewtdsusage.usg +0 -0
  61. {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/viewwebfile.py +0 -0
  62. {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/viewwebfile.usg +0 -0
  63. {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/viewwebusage.py +0 -0
  64. {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics/viewwebusage.usg +0 -0
  65. {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics.egg-info/dependency_links.txt +0 -0
  66. {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/src/rda_python_metrics.egg-info/top_level.txt +0 -0
  67. {rda_python_metrics-1.0.32 → rda_python_metrics-1.0.46}/tests/test_metrics.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: rda_python_metrics
3
- Version: 1.0.32
3
+ Version: 1.0.46
4
4
  Summary: RDA Python Package to gather and view data usage metrics
5
5
  Author-email: Zaihua Ji <zji@ucar.edu>
6
6
  Project-URL: Homepage, https://github.com/NCAR/rda-python-metrics
@@ -17,6 +17,7 @@ Requires-Dist: geoip2
17
17
  Requires-Dist: ipinfo
18
18
  Requires-Dist: httplib2
19
19
  Requires-Dist: dnspython
20
+ Requires-Dist: unidecode
20
21
  Dynamic: license-file
21
22
 
22
23
  RDA Python Package to gather and view data usage metrics.
@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
6
6
 
7
7
  [project]
8
8
  name = "rda_python_metrics"
9
- version = "1.0.32"
9
+ version = "1.0.46"
10
10
  authors = [
11
11
  { name="Zaihua Ji", email="zji@ucar.edu" },
12
12
  ]
@@ -25,7 +25,8 @@ dependencies = [
25
25
  "geoip2",
26
26
  "ipinfo",
27
27
  "httplib2",
28
- "dnspython"
28
+ "dnspython",
29
+ "unidecode"
29
30
  ]
30
31
 
31
32
  [tool.pytest.ini_options]
@@ -38,13 +39,14 @@ pythonpath = [
38
39
 
39
40
  [project.scripts]
40
41
  "logarch.py" = "rda_python_metrics.logarch:main"
41
- "fillawsusage" = "rda_python_metrics.fillaswusage:main"
42
+ "fillawsusage" = "rda_python_metrics.fillawsusage:main"
42
43
  "fillcdgusage" = "rda_python_metrics.fillcdgusage:main"
43
44
  "fillcodusage" = "rda_python_metrics.fillcodusage:main"
44
45
  "fillcountry" = "rda_python_metrics.fillcountry:main"
45
46
  "fillendtime" = "rda_python_metrics.fillendtime:main"
46
47
  "fillglobususage" = "rda_python_metrics.fillglobususage:main"
47
48
  "fillgdexusage" = "rda_python_metrics.fillgdexusage:main"
49
+ "fillzenodousage" = "rda_python_metrics.fillzenodousage:main"
48
50
  "fillipinfo" = "rda_python_metrics.fillipinfo:main"
49
51
  "filloneorder" = "rda_python_metrics.filloneorder:main"
50
52
  "fillosdfusage" = "rda_python_metrics.fillosdfusage:main"
@@ -54,9 +56,11 @@ pythonpath = [
54
56
  "pgperson" = "rda_python_metrics.pgperson:main"
55
57
  "pgusername" = "rda_python_metrics.pgusername:main"
56
58
  "viewallusage" = "rda_python_metrics.viewallusage:main"
59
+ "viewawsusage" = "rda_python_metrics.viewawsusage:main"
57
60
  "viewcheckusage" = "rda_python_metrics.viewcheckusage:main"
58
61
  "viewcodusage" = "rda_python_metrics.viewcodusage:main"
59
62
  "viewordusage" = "rda_python_metrics.viewordusage:main"
63
+ "viewosdfusage" = "rda_python_metrics.viewosdfusage:main"
60
64
  "viewrqstusage" = "rda_python_metrics.viewrqstusage:main"
61
65
  "viewtdsusage" = "rda_python_metrics.viewtdsusage:main"
62
66
  "viewwebfile" = "rda_python_metrics.viewwebfile:main"
@@ -0,0 +1,210 @@
1
+ #!/usr/bin/env python3
2
+ #
3
+ ###############################################################################
4
+ #
5
+ # Title : fillawsusage
6
+ # Author : Zaihua Ji, zji@ucar.edu
7
+ # Date : 03/11/2022
8
+ # 2025-03-26 transferred to package rda_python_metrics from
9
+ # https://github.com/NCAR/rda-database.git
10
+ # Purpose : python program to retrieve info from AWS logs
11
+ # and fill table wusages in PgSQL database dssdb.
12
+ #
13
+ # Github : https://github.com/NCAR/rda-pythn-metrics.git
14
+ #
15
+ ###############################################################################
16
+ #
17
+ import sys
18
+ import re
19
+ import glob
20
+ from os import path as op
21
+ from rda_python_common import PgLOG
22
+ from rda_python_common import PgUtil
23
+ from rda_python_common import PgFile
24
+ from rda_python_common import PgDBI
25
+ from . import PgIPInfo
26
+
27
+ USAGE = {
28
+ 'PGTBL' : "awsusage",
29
+ 'AWSDIR' : PgLOG.PGLOG["TRANSFER"] + "/AWSera5log",
30
+ 'AWSLOG' : "{}/{}-00-00-00-*",
31
+ 'PFMT' : "YYYY/MM/DD"
32
+ }
33
+
34
+ DSIDS = {'nsf-ncar-era5' : 'd633000'}
35
+
36
+ #
37
+ # main function to run this program
38
+ #
39
+ def main():
40
+
41
+ params = [] # array of input values
42
+ argv = sys.argv[1:]
43
+ option = None
44
+
45
+ for arg in argv:
46
+ ms = re.match(r'^-(b|d|p|N)$', arg)
47
+ if ms:
48
+ opt = ms.group(1)
49
+ if opt == 'b':
50
+ PgLOG.PGLOG['BCKGRND'] = 1
51
+ elif option:
52
+ PgLOG.pglog("{}: Option -{} is present already".format(arg, option), PgLOG.LGWNEX)
53
+ else:
54
+ option = opt
55
+ elif re.match(r'^-', arg):
56
+ PgLOG.pglog(arg + ": Invalid Option", PgLOG.LGWNEX)
57
+ elif option:
58
+ params.append(arg)
59
+ else:
60
+ PgLOG.pglog(arg + ": Invalid Parameter", PgLOG.LGWNEX)
61
+
62
+ if not (option and params): PgLOG.show_usage('fillawsusage')
63
+
64
+ PgDBI.dssdb_dbname()
65
+ cmdstr = "fillawsusage {}".format(' '.join(argv))
66
+ PgLOG.cmdlog(cmdstr)
67
+ PgFile.change_local_directory(USAGE['AWSDIR'])
68
+ filenames = get_log_file_names(option, params)
69
+ if filenames:
70
+ fill_aws_usages(filenames)
71
+ else:
72
+ PgLOG.pglog("No log file found for given command: " + cmdstr, PgLOG.LOGWRN)
73
+
74
+ PgLOG.pglog(None, PgLOG.LOGWRN)
75
+ sys.exit(0)
76
+
77
+ #
78
+ # get the log file dates
79
+ #
80
+ def get_log_file_names(option, params):
81
+
82
+ filenames = {}
83
+ if option == 'd':
84
+ for dt in params:
85
+ pdate = PgUtil.format_date(dt)
86
+ pd = PgUtil.format_date(pdate, USAGE['PFMT'])
87
+ fname = USAGE['AWSLOG'].format(pd, pdate)
88
+ fnames = glob.glob(fname)
89
+ if fnames: filenames[pdate] = sorted(fnames)
90
+ else:
91
+ if option == 'N':
92
+ edate = PgUtil.curdate()
93
+ pdate = PgUtil.adddate(edate, 0, 0, -int(params[0]))
94
+ else:
95
+ pdate = PgUtil.format_date(params[0])
96
+ if len(params) > 1:
97
+ edate = PgUtil.adddate(PgUtil.format_date(params[1]), 0, 0, 1)
98
+ else:
99
+ edate = PgUtil.curdate()
100
+ while pdate < edate:
101
+ pd = PgUtil.format_date(pdate, USAGE['PFMT'])
102
+ fname = USAGE['AWSLOG'].format(pd, pdate)
103
+ fnames = glob.glob(fname)
104
+ if fnames: filenames[pdate] = sorted(fnames)
105
+ pdate = PgUtil.adddate(pdate, 0, 0, 1)
106
+
107
+ return filenames
108
+
109
+ #
110
+ # Fill AWS usages into table dssdb.awsusage of DSS PgSQL database from aws access logs
111
+ #
112
+ def fill_aws_usages(filenames):
113
+
114
+ year = cntall = addall = 0
115
+ for pdate in filenames:
116
+ fnames = filenames[pdate]
117
+ fcnt = len(fnames)
118
+ PgLOG.pglog("{}: Gathering AWS usage info from {} log files at {}".format(pdate, fcnt, PgLOG.current_datetime()), PgLOG.LOGWRN)
119
+ records = {}
120
+ cntadd = entcnt = 0
121
+ for logfile in fnames:
122
+ if not op.isfile(logfile):
123
+ PgLOG.pglog("{}: Not exists for Gathering AWS usage".format(logfile), PgLOG.LOGWRN)
124
+ continue
125
+ aws = PgFile.open_local_file(logfile)
126
+ if not aws: continue
127
+ while True:
128
+ line = aws.readline()
129
+ if not line: break
130
+ entcnt += 1
131
+ if entcnt%20000 == 0:
132
+ dcnt = len(records)
133
+ PgLOG.pglog("{}: {}/{} AWS log entries processed/records to add".format(pdate, entcnt, dcnt), PgLOG.WARNLG)
134
+
135
+ ms = re.match(r'^\w+ ([\w-]+) \[(\S+).*\] ([\d\.]+) .+ REST\.GET\.OBJECT \S+ "GET.+" \d+ - (\d+) \d+ .* ".+" "(.+)" ', line)
136
+ if not ms: continue
137
+ values = list(ms.groups())
138
+ if values[0] not in DSIDS: continue
139
+ dsid = DSIDS[values[0]]
140
+ size = int(values[3])
141
+ ip = values[2]
142
+ engine = values[4]
143
+ moff = engine.find('/')
144
+ if moff > 0:
145
+ if moff > 20: moff = 20
146
+ method = engine[0:moff].upper()
147
+ else:
148
+ method = "AWS"
149
+ key = "{}:{}:{}".format(ip, dsid, method)
150
+ if key in records:
151
+ records[key]['size'] += size
152
+ records[key]['fcount'] += 1
153
+ else:
154
+ (year, quarter, date, time) = get_record_date_time(values[1])
155
+ iprec = PgIPInfo.get_missing_ipinfo(ip)
156
+ if not iprec: continue
157
+ records[key] = {'ip' : ip, 'dsid' : dsid, 'date' : date, 'time' : time, 'quarter' : quarter,
158
+ 'size' : size, 'fcount' : 1, 'method' : method, 'engine' : engine,
159
+ 'org_type' : iprec['org_type'], 'country' : iprec['country'],
160
+ 'region' : iprec['region'], 'email' : iprec['email']}
161
+ aws.close()
162
+ if records: cntadd = add_usage_records(records, year)
163
+ PgLOG.pglog("{}: {} AWS usage records added for {} entries at {}".format(pdate, cntadd, entcnt, PgLOG.current_datetime()), PgLOG.LOGWRN)
164
+ cntall += entcnt
165
+ if cntadd:
166
+ addall += cntadd
167
+ if addall > cntadd:
168
+ PgLOG.pglog("{} AWS usage records added for {} entries at {}".format(addall, cntall, PgLOG.current_datetime()), PgLOG.LOGWRN)
169
+
170
+ def get_record_date_time(ctime):
171
+
172
+ ms = re.search(r'^(\d+)/(\w+)/(\d+):(\d+:\d+:\d+)$', ctime)
173
+ if ms:
174
+ d = int(ms.group(1))
175
+ m = PgUtil.get_month(ms.group(2))
176
+ y = ms.group(3)
177
+ t = ms.group(4)
178
+ q = 1 + int((m-1)/3)
179
+ return (y, q, "{}-{:02}-{:02}".format(y, m, d), t)
180
+ else:
181
+ PgLOG.pglog(ctime + ": Invalid date/time format", PgLOG.LGEREX)
182
+
183
+ def add_usage_records(records, year):
184
+
185
+ cnt = 0
186
+ for key in records:
187
+ record = records[key]
188
+ cond = "date = '{}' AND time = '{}' AND ip = '{}'".format(record['date'], record['time'], record['ip'])
189
+ if PgDBI.pgget(USAGE['PGTBL'], '', cond, PgLOG.LGEREX): continue
190
+ if add_to_allusage(year, record):
191
+ cnt += PgDBI.pgadd(USAGE['PGTBL'], record, PgLOG.LOGWRN)
192
+
193
+ return cnt
194
+
195
+
196
+ def add_to_allusage(year, pgrec):
197
+
198
+ record = {'source' : 'A'}
199
+ flds = ['ip', 'dsid', 'date', 'time', 'quarter', 'size', 'method',
200
+ 'org_type', 'country', 'region', 'email']
201
+
202
+ for fld in flds:
203
+ record[fld] = pgrec[fld]
204
+
205
+ return PgDBI.add_yearly_allusage(year, record)
206
+
207
+ #
208
+ # call main() to start program
209
+ #
210
+ if __name__ == "__main__": main()
@@ -1,6 +1,6 @@
1
1
 
2
2
  Retrieves usage information from AWS Server logs under directory
3
- ../rda/transer/AWSera5log/ to fill table 'wusage' in database 'rdadb'.
3
+ ../rda/transer/AWSera5log/ to fill table 'awsusage' in database 'rdadb'.
4
4
 
5
5
  Usage: fillawsusage [-b] [-d LogFileDates] [-N NumberDay] [-p BeginDate [Enddate]]
6
6
 
@@ -31,29 +31,30 @@ USAGE = {
31
31
  }
32
32
 
33
33
  DSIDS = {
34
- # 'ucar.cgd.cesm2.cam6.prescribed_sst_amip' : ['d651010'],
35
- # 'ucar.cgd.ccsm4.CLM_LAND_ONLY' : ['d651011'],
36
- # 'ucar.cgd.artmip' : ['d651012', 'd651016', 'd651017', 'd651018'],
37
- # 'tamip' : ['d651013'],
38
- # 'ucar.cgd.ccsm4.CLIVAR_LE' : ['d651014'],
39
- # 'ucar.cgd.cesm2.Gettelman_CESM2_ECS' : ['d651015'],
40
- # 'ucar.cgd.ccsm4.geomip.ssp5' : ['d651024'],
41
- # 'ucar.cgd.ccsm4.IOD-PACEMAKER' : ['d651021'],
42
- # 'ucar.cgd.ccsm4.past2k_transient' : ['651023'],
43
- # 'ucar.cgd.ccsm4.lowwarming' : ['d651025'],
44
- # 'ucar.cgd.ccsm4.CESM_CAM5_BGC_ME' : ['d651000'],
45
- # 'ucar.cgd.ccsm4.iTRACE' : ['d651022'],
46
- # 'ucar.cgd.ccsm4.so2_geoeng' : ['d651026'],
47
- # 'ucar.cgd.ccsm4.cesmLE' : ['d651027'],
48
- # 'ucar.cgd.ccsm4.CESM1-CAM5-DP' : ['d651028'],
49
- # 'ucar.cgd.ccsm4.amv_lens' : ['d651031'],
50
- # 'ucar.cgd.ccsm4.ATL-PACEMAKER' : ['d651032'],
51
- # 'ucar.cgd.ccsm4.pac-pacemaker' : ['d651033'],
52
- # 'ucar.cgd.ccsm4.SD-WACCM-X_v2.1' : ['d651034'],
53
- # 'ucar.cgd.ccsm4.amv_lens' : ['d651035'],
54
- # 'ucar.cgd.cesm2.cism_ismip6' : ['d651036'],
55
- # 'ucar.cgd.ccsm4.pliomip2' : ['d651037'],
56
- # new added
34
+ 'pi_cesm2_atm_river_analysis' : ['d010073'],
35
+ 'na-cordex' : ['d316009'],
36
+ 'ucar.cgd.cesm2.cam6.prescribed_sst_amip' : ['d651010'],
37
+ 'ucar.cgd.ccsm4.CLM_LAND_ONLY' : ['d651011'],
38
+ 'ucar.cgd.artmip' : ['d651012', 'd651016', 'd651017', 'd651018'],
39
+ 'tamip' : ['d651013'],
40
+ 'ucar.cgd.ccsm4.CLIVAR_LE' : ['d651014'],
41
+ 'ucar.cgd.cesm2.Gettelman_CESM2_ECS' : ['d651015'],
42
+ 'ucar.cgd.ccsm4.geomip.ssp5' : ['d651024'],
43
+ 'ucar.cgd.ccsm4.IOD-PACEMAKER' : ['d651021'],
44
+ 'ucar.cgd.ccsm4.past2k_transient' : ['651023'],
45
+ 'ucar.cgd.ccsm4.lowwarming' : ['d651025'],
46
+ 'ucar.cgd.ccsm4.CESM_CAM5_BGC_ME' : ['d651000'],
47
+ 'ucar.cgd.ccsm4.iTRACE' : ['d651022'],
48
+ 'ucar.cgd.ccsm4.so2_geoeng' : ['d651026'],
49
+ 'ucar.cgd.ccsm4.cesmLE' : ['d651027'],
50
+ 'ucar.cgd.ccsm4.CESM1-CAM5-DP' : ['d651028'],
51
+ 'ucar.cgd.ccsm4.amv_lens' : ['d651031'],
52
+ 'ucar.cgd.ccsm4.ATL-PACEMAKER' : ['d651032'],
53
+ 'ucar.cgd.ccsm4.pac-pacemaker' : ['d651033'],
54
+ 'ucar.cgd.ccsm4.SD-WACCM-X_v2.1' : ['d651034'],
55
+ 'ucar.cgd.ccsm4.amv_lens' : ['d651035'],
56
+ 'ucar.cgd.cesm2.cism_ismip6' : ['d651036'],
57
+ 'ucar.cgd.ccsm4.pliomip2' : ['d651037'],
57
58
  'ucar.cgd.cesm2-waccm.s2s_hindcasts': ['d651040'],
58
59
  'ucar.cgd.CESM1.3_SH_storm_tracks': ['d651044'],
59
60
  'ucar.cgd.cesm2.waccm6.ssp245': ['d651045'],
@@ -67,13 +68,12 @@ DSIDS = {
67
68
  'ucar.cgd.ccsm4.PaleoIF' : ['d651052'],
68
69
  'ucar.cgd.ccsm4.b.e11.B20LE_fixedO3' : ['d651053'],
69
70
  'ucar.cgd.cesm2.single.forcing.large.ensemble' : ['d651055'],
70
- 'ucar.cgd.cesm2le.output': ['d651056'], # 2025-01 done
71
+ 'ucar.cgd.cesm2le.output': ['d651056'],
71
72
  'ucar.cgd.ccsm4.ARISE-SAI-1.5' : ['d651059'],
72
73
  'ucar.cgd.cesm2.s2s_hindcasts': ['d651060'],
73
74
  'ucar.cgd.cesm2.s2s_hindcasts.mjo': ['d651061'],
74
75
  'ucar.cgd.cesm2.s2s_hindcasts.tc_tracks': ['d651062'],
75
76
  'ucar.cgd.cesm2.s2s_hindcasts.cesm2.climo': ['d651063'],
76
- #add 2025-08-06
77
77
  'ucar.cgd.ccsm4.cesmLME' : ['d651058'],
78
78
  'ucar.cgd.ccsm4.GLENS' : ['d651064'],
79
79
  'ucar.cgd.ccsm4.CESM2-CISM2-LIGtransient' : ['d651066'],
@@ -85,8 +85,13 @@ DSIDS = {
85
85
  'ucar.cgd.cesm1.cldmod': ['d651069'],
86
86
  'ucar.cgd.cesm2.marine.biogeochemistry': ['d651071'],
87
87
  'ucar.cgd.nw2.mom6': ['d651072'],
88
- # 'ucar.cgd.cesm2.smyle': ['d651065'], # not done util the end of Aug.
89
-
88
+ 'ucar.cgd.cesm2.cam6.ppe': ['d651076'],
89
+ 'ucar.cgd.cesm2.smyle': ['d651065'],
90
+ # new added
91
+ 'gridded_precip_and_temp' : ['d010078'],
92
+ '29_newman' : ['d010079'],
93
+ 'waccm-x.ion.asymmetry' : ['d010081'],
94
+ 'NARCCAP' : ['d316015']
90
95
  }
91
96
 
92
97
  ALLIDS = list(DSIDS.keys())
@@ -30,44 +30,58 @@ USAGE = {
30
30
  }
31
31
 
32
32
  DSIDS = {
33
+ "14_schuster" : "d583138",
34
+ "384_duvivier" : "d583139",
35
+ "model_data_for_17_march_2015_storm_event" : "d583140",
36
+ "326_qingyu" : "d583141",
37
+ "354_fasullo" : "d583142",
38
+ "157_gaubert" : "d583143",
39
+ "370_gaubert" : "d583144",
40
+ "459" : "d583145",
41
+ "108_kristenk" : "d583146",
42
+ "82_abaker" : "d583147",
43
+ "215_grabow" : "d583148",
44
+ "222_duvivier" : "d583149",
45
+ "366_rberrios" : "d583150",
46
+ "210_dxyang" : "d583100",
47
+ "238_junkyung" : "d583101",
48
+ "361_islas" : "d583102",
49
+ "448" : "d583103",
50
+ "187_jaredlee" : "d583104",
51
+ "273_kdagon" : "d583105",
52
+ "149_liuh" : "d583106",
53
+ "279_schwantes" : "d583107",
54
+ "60_pinto" : "d583108",
55
+ "WACC_Dependence_on_Solar_Activity" : "d583109",
56
+ "275_domingom" : "d583110",
57
+ "445" : "d583111",
58
+ "TIE-GCM_2017_Solar_Storm" : "d583112",
59
+ "123_andrew" : "d583113",
60
+ "453" : "d583114",
61
+ "367_mclong" : "d583115",
62
+ "258_jiangzhu" : "d583116",
63
+ "136_hannay" : "d583117",
64
+ "173B_dcherian" : "d583118",
65
+ "407_islas" : "d583119",
66
+ "189_islas" : "d583120",
67
+ "289_shields" : "d583121",
68
+ "415_mcinerney" : "d583122",
69
+ "151_dll" : "d583123",
70
+ "460" : "d583124",
71
+ "117_phamkh" : "d583125",
72
+ "109_erichend" : "d583126",
73
+ "412" : "d583127",
74
+ "390_knocasio" : "d583128",
75
+ "382_kumar" : "d583129",
76
+ "470" : "d583130",
77
+ "371_abaker" : "d583131",
78
+ "CESM-cocco_CO2_experiments" : "d583132",
79
+ "fuel_moisture_content" : "d583133",
80
+ "81_rberrios" : "d583134",
81
+ "286_knocasio" : "d583135",
82
+ "383_stebbins" : "d583136",
83
+ "256_tilmes" : "d583137",
33
84
  '371_abaker' : 'd583131',
34
- 'CESM-cocco_CO2_experiments' : 'd583132',
35
- '256_tilmes' : 'd583137',
36
- 'fuel_moisture_content' : 'd583133',
37
- '81_rberrios' : 'd583134',
38
- '210_dxyang' : 'd583100',
39
- '286_knocasio' : 'd583135',
40
- '238_junkyung' : 'd583101',
41
- '361_islas' : 'd583102',
42
- '448' : 'd583103',
43
- '383_stebbins' : 'd583136',
44
- '187_jaredlee' : 'd583104',
45
- '273_kdagon' : 'd583105',
46
- '149_liuh' : 'd583106',
47
- '279_schwantes' : 'd583107',
48
- '60_pinto' : 'd583108',
49
- 'WACC_Dependence_on_Solar_Activity' : 'd583109',
50
- '275_domingom' : 'd583110',
51
- '445' : 'd583111',
52
- 'TIE-GCM_2017_Solar_Storm' : 'd583112',
53
- '123_andrew' : 'd583113',
54
- '453' : 'd583114',
55
- '367_mclong' : 'd583115',
56
- '258_jiangzhu' : 'd583116',
57
- '136_hannay' : 'd583117',
58
- '173B_dcherian' : 'd583118',
59
- '407_islas' : 'd583119',
60
- '189_islas' : 'd583120',
61
- '289_shields' : 'd583121',
62
- '415_mcinerney' : 'd583122',
63
- '151_dll' : 'd583123',
64
- '460' : 'd583124',
65
- '117_phamkh' : 'd583125',
66
- '109_erichend' : 'd583126',
67
- '412' : 'd583127',
68
- '390_knocasio' : 'd583128',
69
- '382_kumar' : 'd583129',
70
- '470' : 'd583130',
71
85
  # icarus
72
86
  'icarus.experiment.403' : 'd789010',
73
87
  'icarus.experiment.404' : 'd789011',
@@ -27,7 +27,7 @@ from . import PgIPInfo
27
27
 
28
28
  USAGE = {
29
29
  'PGTBL' : "wusage",
30
- 'GBSDIR' : PgLOG.PGLOG["DSSDATA"] + "/work/logs/gridftp/",
30
+ 'GBSDIR' : PgLOG.PGLOG["GDEXWORK"] + "/logs/gridftp/",
31
31
  'GBSLOG' : "access_log_gridftp0{}_{}",
32
32
  }
33
33
 
@@ -43,7 +43,7 @@ def main():
43
43
 
44
44
 
45
45
  for arg in argv:
46
- ms = re.match(r'^-(b|d|p|N)$', arg)
46
+ ms = re.match(r'^-(b|d|f|p|N)$', arg)
47
47
  if ms:
48
48
  opt = ms.group(1)
49
49
  if opt == 'b':
@@ -80,7 +80,9 @@ def main():
80
80
  def get_log_file_names(option, params, datelimits):
81
81
 
82
82
  filenames = []
83
- if option == 'd':
83
+ if option == 'f':
84
+ filenames = params
85
+ elif option == 'd':
84
86
  for pdate in params:
85
87
  fdate = PgUtil.format_date(pdate, 'MMDDYYYY')
86
88
  fname = USAGE['GBSLOG'].format('?', fdate)
@@ -147,18 +149,14 @@ def fill_globus_usages(fnames, datelimits):
147
149
  locflag = 'O' if re.match(r'^https://stratus\.', sline) else 'G'
148
150
  idx = wfile.find('?')
149
151
  if idx > -1: wfile = wfile[:idx]
150
-
151
- if re.match(r'^curl', engine, re.I):
152
- method = "CURL"
153
- elif re.match(r'^wget', engine, re.I):
154
- method = "WGET"
155
- elif re.match(r'^python', engine, re.I):
156
- method = "PYTHN"
152
+ moff = engine.find('/')
153
+ if moff > 0:
154
+ if moff > 20: moff = 20
155
+ method = engine[0:moff].upper()
157
156
  else:
158
157
  method = "WEB"
159
158
 
160
159
  key = "{}:{}:{}".format(ip, dsid, wfile) if stat == '206' else None
161
-
162
160
  if record:
163
161
  if key == pkey:
164
162
  record['size'] += size
@@ -3,14 +3,16 @@
3
3
  /gpfs/fs1/collections/rda/work/logs/gridftp/ to fill table 'wusage' in
4
4
  database 'rdadb'.
5
5
 
6
- Usage: fillglobususage [-b] [-d LogFileDates] [-N NumberDay] [-p BeginDate [Enddate]]
6
+ Usage: fillglobususage [-b] [-d LogFileDates] [-f LogFileNames] [-N NumberDay] [-p BeginDate [Enddate]]
7
7
 
8
- select option, -d, -N or -p to run this application.
8
+ select option, -d, -f, -N or -p to run this application.
9
9
 
10
10
  - Option -b, log process information into logfile only;
11
11
 
12
12
  - Option -d, retrieve usage info from given log file dates;
13
13
 
14
+ - Option -f, retrieve usage info from given log file names;
15
+
14
16
  - Option -N, retrieve usage info in recent NumberDay days;
15
17
 
16
18
  - Option -p, retrieve usage info between given period. For missing EndDate,