rda-python-metrics 2.0.6__tar.gz → 3.0.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rda_python_metrics-3.0.2/PKG-INFO +185 -0
- rda_python_metrics-3.0.2/README.md +160 -0
- {rda_python_metrics-2.0.6 → rda_python_metrics-3.0.2}/pyproject.toml +6 -3
- {rda_python_metrics-2.0.6 → rda_python_metrics-3.0.2}/src/rda_python_metrics/PgView.py +3 -3
- rda_python_metrics-3.0.2/src/rda_python_metrics/__init__.py +29 -0
- {rda_python_metrics-2.0.6 → rda_python_metrics-3.0.2}/src/rda_python_metrics/fill_awsusage.py +1 -1
- {rda_python_metrics-2.0.6 → rda_python_metrics-3.0.2}/src/rda_python_metrics/fill_cdgusage.py +2 -2
- {rda_python_metrics-2.0.6 → rda_python_metrics-3.0.2}/src/rda_python_metrics/fill_globususage.py +1 -1
- {rda_python_metrics-2.0.6 → rda_python_metrics-3.0.2}/src/rda_python_metrics/fill_osdfusage.py +1 -1
- {rda_python_metrics-2.0.6 → rda_python_metrics-3.0.2}/src/rda_python_metrics/fill_rdadb.py +1 -1
- {rda_python_metrics-2.0.6 → rda_python_metrics-3.0.2}/src/rda_python_metrics/fill_tdsusage.py +1 -1
- {rda_python_metrics-2.0.6 → rda_python_metrics-3.0.2}/src/rda_python_metrics/fillawsusage.py +11 -9
- rda_python_metrics-3.0.2/src/rda_python_metrics/fillawsusage.usg +27 -0
- {rda_python_metrics-2.0.6 → rda_python_metrics-3.0.2}/src/rda_python_metrics/fillcdgusage.py +17 -15
- rda_python_metrics-3.0.2/src/rda_python_metrics/fillcdgusage.usg +33 -0
- {rda_python_metrics-2.0.6 → rda_python_metrics-3.0.2}/src/rda_python_metrics/fillcodusage.py +11 -9
- rda_python_metrics-3.0.2/src/rda_python_metrics/fillcodusage.usg +32 -0
- {rda_python_metrics-2.0.6 → rda_python_metrics-3.0.2}/src/rda_python_metrics/fillcountry.py +8 -6
- {rda_python_metrics-2.0.6 → rda_python_metrics-3.0.2}/src/rda_python_metrics/fillendtime.py +7 -5
- rda_python_metrics-3.0.2/src/rda_python_metrics/fillgdexusage.py +1025 -0
- rda_python_metrics-3.0.2/src/rda_python_metrics/fillgdexusage.usg +33 -0
- {rda_python_metrics-2.0.6 → rda_python_metrics-3.0.2}/src/rda_python_metrics/fillglobususage.py +14 -12
- rda_python_metrics-3.0.2/src/rda_python_metrics/fillglobususage.usg +31 -0
- {rda_python_metrics-2.0.6 → rda_python_metrics-3.0.2}/src/rda_python_metrics/fillipinfo.py +12 -10
- rda_python_metrics-3.0.2/src/rda_python_metrics/fillipinfo.usg +38 -0
- {rda_python_metrics-2.0.6 → rda_python_metrics-3.0.2}/src/rda_python_metrics/filloneorder.py +9 -7
- rda_python_metrics-3.0.2/src/rda_python_metrics/filloneorder.usg +39 -0
- {rda_python_metrics-2.0.6 → rda_python_metrics-3.0.2}/src/rda_python_metrics/fillosdfusage.py +11 -9
- rda_python_metrics-3.0.2/src/rda_python_metrics/fillosdfusage.usg +27 -0
- {rda_python_metrics-2.0.6 → rda_python_metrics-3.0.2}/src/rda_python_metrics/fillrdadb.py +9 -7
- rda_python_metrics-3.0.2/src/rda_python_metrics/fillrdadb.usg +36 -0
- {rda_python_metrics-2.0.6 → rda_python_metrics-3.0.2}/src/rda_python_metrics/filltdsusage.py +42 -39
- rda_python_metrics-3.0.2/src/rda_python_metrics/filltdsusage.usg +27 -0
- {rda_python_metrics-2.0.6 → rda_python_metrics-3.0.2}/src/rda_python_metrics/filluser.py +8 -6
- rda_python_metrics-3.0.2/src/rda_python_metrics/filluser.usg +24 -0
- rda_python_metrics-3.0.2/src/rda_python_metrics/fillzenodousage.py +397 -0
- {rda_python_metrics-2.0.6 → rda_python_metrics-3.0.2}/src/rda_python_metrics/log_arch.py +2 -2
- {rda_python_metrics-2.0.6 → rda_python_metrics-3.0.2}/src/rda_python_metrics/logarch.py +5 -2
- rda_python_metrics-3.0.2/src/rda_python_metrics/logarch.usg +34 -0
- {rda_python_metrics-2.0.6 → rda_python_metrics-3.0.2}/src/rda_python_metrics/pg_ipinfo.py +29 -28
- {rda_python_metrics-2.0.6 → rda_python_metrics-3.0.2}/src/rda_python_metrics/pg_person.py +1 -1
- {rda_python_metrics-2.0.6 → rda_python_metrics-3.0.2}/src/rda_python_metrics/pg_username.py +1 -1
- {rda_python_metrics-2.0.6 → rda_python_metrics-3.0.2}/src/rda_python_metrics/pg_view.py +25 -30
- {rda_python_metrics-2.0.6 → rda_python_metrics-3.0.2}/src/rda_python_metrics/pgperson.py +4 -4
- {rda_python_metrics-2.0.6 → rda_python_metrics-3.0.2}/src/rda_python_metrics/pgsyspath.py +1 -1
- {rda_python_metrics-2.0.6 → rda_python_metrics-3.0.2}/src/rda_python_metrics/pgusername.py +2 -2
- {rda_python_metrics-2.0.6 → rda_python_metrics-3.0.2}/src/rda_python_metrics/view_allusage.py +9 -9
- {rda_python_metrics-2.0.6 → rda_python_metrics-3.0.2}/src/rda_python_metrics/view_awsusage.py +13 -13
- {rda_python_metrics-2.0.6 → rda_python_metrics-3.0.2}/src/rda_python_metrics/view_checkusage.py +10 -10
- {rda_python_metrics-2.0.6 → rda_python_metrics-3.0.2}/src/rda_python_metrics/view_codusage.py +8 -8
- {rda_python_metrics-2.0.6 → rda_python_metrics-3.0.2}/src/rda_python_metrics/view_ordusage.py +14 -14
- {rda_python_metrics-2.0.6 → rda_python_metrics-3.0.2}/src/rda_python_metrics/view_osdfusage.py +13 -13
- {rda_python_metrics-2.0.6 → rda_python_metrics-3.0.2}/src/rda_python_metrics/view_rqstusage.py +9 -9
- {rda_python_metrics-2.0.6 → rda_python_metrics-3.0.2}/src/rda_python_metrics/view_tdsusage.py +13 -13
- {rda_python_metrics-2.0.6 → rda_python_metrics-3.0.2}/src/rda_python_metrics/view_webfile.py +11 -13
- {rda_python_metrics-2.0.6 → rda_python_metrics-3.0.2}/src/rda_python_metrics/view_webusage.py +11 -11
- {rda_python_metrics-2.0.6 → rda_python_metrics-3.0.2}/src/rda_python_metrics/viewallusage.py +16 -15
- rda_python_metrics-3.0.2/src/rda_python_metrics/viewallusage.usg +150 -0
- {rda_python_metrics-2.0.6 → rda_python_metrics-3.0.2}/src/rda_python_metrics/viewawsusage.py +20 -19
- rda_python_metrics-3.0.2/src/rda_python_metrics/viewawsusage.usg +123 -0
- {rda_python_metrics-2.0.6 → rda_python_metrics-3.0.2}/src/rda_python_metrics/viewcheckusage.py +18 -17
- rda_python_metrics-3.0.2/src/rda_python_metrics/viewcheckusage.usg +131 -0
- rda_python_metrics-3.0.2/src/rda_python_metrics/viewcodusage.py +291 -0
- rda_python_metrics-3.0.2/src/rda_python_metrics/viewcodusage.usg +123 -0
- {rda_python_metrics-2.0.6 → rda_python_metrics-3.0.2}/src/rda_python_metrics/viewordusage.py +21 -20
- rda_python_metrics-3.0.2/src/rda_python_metrics/viewordusage.usg +131 -0
- {rda_python_metrics-2.0.6 → rda_python_metrics-3.0.2}/src/rda_python_metrics/viewosdfusage.py +20 -19
- rda_python_metrics-3.0.2/src/rda_python_metrics/viewosdfusage.usg +123 -0
- {rda_python_metrics-2.0.6 → rda_python_metrics-3.0.2}/src/rda_python_metrics/viewrqstusage.py +19 -18
- rda_python_metrics-3.0.2/src/rda_python_metrics/viewrqstusage.usg +142 -0
- {rda_python_metrics-2.0.6 → rda_python_metrics-3.0.2}/src/rda_python_metrics/viewtdsusage.py +20 -19
- rda_python_metrics-3.0.2/src/rda_python_metrics/viewtdsusage.usg +127 -0
- {rda_python_metrics-2.0.6 → rda_python_metrics-3.0.2}/src/rda_python_metrics/viewwebfile.py +17 -18
- rda_python_metrics-3.0.2/src/rda_python_metrics/viewwebfile.usg +143 -0
- {rda_python_metrics-2.0.6 → rda_python_metrics-3.0.2}/src/rda_python_metrics/viewwebusage.py +19 -18
- rda_python_metrics-3.0.2/src/rda_python_metrics/viewwebusage.usg +141 -0
- rda_python_metrics-3.0.2/src/rda_python_metrics.egg-info/PKG-INFO +185 -0
- {rda_python_metrics-2.0.6 → rda_python_metrics-3.0.2}/src/rda_python_metrics.egg-info/entry_points.txt +1 -1
- {rda_python_metrics-2.0.6 → rda_python_metrics-3.0.2}/src/rda_python_metrics.egg-info/requires.txt +3 -0
- rda_python_metrics-2.0.6/PKG-INFO +0 -23
- rda_python_metrics-2.0.6/README.md +0 -1
- rda_python_metrics-2.0.6/src/rda_python_metrics/__init__.py +0 -1
- rda_python_metrics-2.0.6/src/rda_python_metrics/fillawsusage.usg +0 -16
- rda_python_metrics-2.0.6/src/rda_python_metrics/fillcdgusage.usg +0 -18
- rda_python_metrics-2.0.6/src/rda_python_metrics/fillcodusage.usg +0 -21
- rda_python_metrics-2.0.6/src/rda_python_metrics/fillgdexusage.py +0 -1032
- rda_python_metrics-2.0.6/src/rda_python_metrics/fillgdexusage.usg +0 -18
- rda_python_metrics-2.0.6/src/rda_python_metrics/fillglobususage.usg +0 -19
- rda_python_metrics-2.0.6/src/rda_python_metrics/fillipinfo.usg +0 -20
- rda_python_metrics-2.0.6/src/rda_python_metrics/filloneorder.usg +0 -41
- rda_python_metrics-2.0.6/src/rda_python_metrics/fillosdfusage.usg +0 -17
- rda_python_metrics-2.0.6/src/rda_python_metrics/fillrdadb.usg +0 -32
- rda_python_metrics-2.0.6/src/rda_python_metrics/filltdsusage.usg +0 -16
- rda_python_metrics-2.0.6/src/rda_python_metrics/filluser.usg +0 -16
- rda_python_metrics-2.0.6/src/rda_python_metrics/fillzenodousage.py +0 -408
- rda_python_metrics-2.0.6/src/rda_python_metrics/logarch.usg +0 -29
- rda_python_metrics-2.0.6/src/rda_python_metrics/viewallusage.usg +0 -215
- rda_python_metrics-2.0.6/src/rda_python_metrics/viewawsusage.usg +0 -190
- rda_python_metrics-2.0.6/src/rda_python_metrics/viewcheckusage.usg +0 -185
- rda_python_metrics-2.0.6/src/rda_python_metrics/viewcodusage.py +0 -290
- rda_python_metrics-2.0.6/src/rda_python_metrics/viewcodusage.usg +0 -187
- rda_python_metrics-2.0.6/src/rda_python_metrics/viewordusage.usg +0 -223
- rda_python_metrics-2.0.6/src/rda_python_metrics/viewosdfusage.usg +0 -190
- rda_python_metrics-2.0.6/src/rda_python_metrics/viewrqstusage.usg +0 -217
- rda_python_metrics-2.0.6/src/rda_python_metrics/viewtdsusage.usg +0 -194
- rda_python_metrics-2.0.6/src/rda_python_metrics/viewwebfile.usg +0 -212
- rda_python_metrics-2.0.6/src/rda_python_metrics/viewwebusage.usg +0 -211
- rda_python_metrics-2.0.6/src/rda_python_metrics.egg-info/PKG-INFO +0 -23
- {rda_python_metrics-2.0.6 → rda_python_metrics-3.0.2}/LICENSE +0 -0
- {rda_python_metrics-2.0.6 → rda_python_metrics-3.0.2}/MANIFEST.in +0 -0
- {rda_python_metrics-2.0.6 → rda_python_metrics-3.0.2}/setup.cfg +0 -0
- {rda_python_metrics-2.0.6 → rda_python_metrics-3.0.2}/src/rda_python_metrics/PgIPInfo.py +0 -0
- {rda_python_metrics-2.0.6 → rda_python_metrics-3.0.2}/src/rda_python_metrics/fill_codusage.py +0 -0
- {rda_python_metrics-2.0.6 → rda_python_metrics-3.0.2}/src/rda_python_metrics/fill_country.py +0 -0
- {rda_python_metrics-2.0.6 → rda_python_metrics-3.0.2}/src/rda_python_metrics/fill_endtime.py +0 -0
- {rda_python_metrics-2.0.6 → rda_python_metrics-3.0.2}/src/rda_python_metrics/fill_ipinfo.py +0 -0
- {rda_python_metrics-2.0.6 → rda_python_metrics-3.0.2}/src/rda_python_metrics/fill_oneorder.py +0 -0
- {rda_python_metrics-2.0.6 → rda_python_metrics-3.0.2}/src/rda_python_metrics/fill_user.py +0 -0
- {rda_python_metrics-2.0.6 → rda_python_metrics-3.0.2}/src/rda_python_metrics.egg-info/SOURCES.txt +0 -0
- {rda_python_metrics-2.0.6 → rda_python_metrics-3.0.2}/src/rda_python_metrics.egg-info/dependency_links.txt +0 -0
- {rda_python_metrics-2.0.6 → rda_python_metrics-3.0.2}/src/rda_python_metrics.egg-info/top_level.txt +0 -0
- {rda_python_metrics-2.0.6 → rda_python_metrics-3.0.2}/tests/test_metrics.py +0 -0
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: rda_python_metrics
|
|
3
|
+
Version: 3.0.2
|
|
4
|
+
Summary: RDA Python Package to gather and view data usage metrics
|
|
5
|
+
Author-email: Zaihua Ji <zji@ucar.edu>
|
|
6
|
+
Project-URL: Homepage, https://github.com/NCAR/rda-python-metrics
|
|
7
|
+
Classifier: Programming Language :: Python :: 3
|
|
8
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
9
|
+
Classifier: Operating System :: OS Independent
|
|
10
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
11
|
+
Requires-Python: >=3.7
|
|
12
|
+
Description-Content-Type: text/markdown
|
|
13
|
+
License-File: LICENSE
|
|
14
|
+
Requires-Dist: rda_python_common
|
|
15
|
+
Requires-Dist: rda_python_setuid
|
|
16
|
+
Requires-Dist: geoip2
|
|
17
|
+
Requires-Dist: ipinfo
|
|
18
|
+
Requires-Dist: httplib2
|
|
19
|
+
Requires-Dist: dnspython
|
|
20
|
+
Requires-Dist: unidecode
|
|
21
|
+
Requires-Dist: urllib3>=2.5.0
|
|
22
|
+
Requires-Dist: requests>=2.33.0
|
|
23
|
+
Requires-Dist: idna>=3.10
|
|
24
|
+
Dynamic: license-file
|
|
25
|
+
|
|
26
|
+
# rda_python_metrics
|
|
27
|
+
|
|
28
|
+
RDA Python package to gather and view data usage metrics for the
|
|
29
|
+
[NSF NCAR Geoscience Data Exchange (GDEX)](https://gdex.ucar.edu).
|
|
30
|
+
|
|
31
|
+
## Programs
|
|
32
|
+
|
|
33
|
+
The package provides two categories of programs:
|
|
34
|
+
|
|
35
|
+
**Run as gdexdata via setuid (requires setup below):**
|
|
36
|
+
|
|
37
|
+
| Command | Connector script | Description |
|
|
38
|
+
|---------|-----------------|-------------|
|
|
39
|
+
| `logarch` | `setuid_logarch` | Archive RDA log files (web, TDS, AWS, OSDF, dssdb) to the DECSDATA area |
|
|
40
|
+
|
|
41
|
+
**Run as current user (no setuid required):**
|
|
42
|
+
|
|
43
|
+
| Command | Description |
|
|
44
|
+
|---------|-------------|
|
|
45
|
+
| `fillawsusage` | Fill AWS usage metrics into the database |
|
|
46
|
+
| `fillcdgusage` | Fill CDG usage metrics into the database |
|
|
47
|
+
| `fillcodusage` | Fill COD usage metrics into the database |
|
|
48
|
+
| `fillcountry` | Fill country information into the database |
|
|
49
|
+
| `fillendtime` | Fill end times for usage records |
|
|
50
|
+
| `fillgdexusage` | Fill GDEX usage metrics into the database |
|
|
51
|
+
| `fillglobususage` | Fill Globus transfer usage metrics into the database |
|
|
52
|
+
| `fillipinfo` | Fill IP geolocation info into the database |
|
|
53
|
+
| `filloneorder` | Fill metrics for a single order |
|
|
54
|
+
| `fillosdfusage` | Fill OSDF usage metrics into the database |
|
|
55
|
+
| `fillrdadb` | Fill RDA database usage metrics |
|
|
56
|
+
| `filltdsusage` | Fill TDS (THREDDS) usage metrics into the database |
|
|
57
|
+
| `filluser` | Fill user information into the database |
|
|
58
|
+
| `fillzenodousage` | Fill Zenodo usage metrics into the database |
|
|
59
|
+
| `pgperson` | Retrieve user info from the UCAR People API by field |
|
|
60
|
+
| `pgusername` | Retrieve user info from the UCAR People API by username |
|
|
61
|
+
| `viewallusage` | View combined usage statistics |
|
|
62
|
+
| `viewawsusage` | View AWS usage statistics |
|
|
63
|
+
| `viewcheckusage` | View usage check results |
|
|
64
|
+
| `viewcodusage` | View COD usage statistics |
|
|
65
|
+
| `viewordusage` | View order usage statistics |
|
|
66
|
+
| `viewosdfusage` | View OSDF usage statistics |
|
|
67
|
+
| `viewrqstusage` | View request usage statistics |
|
|
68
|
+
| `viewtdsusage` | View TDS usage statistics |
|
|
69
|
+
| `viewwebfile` | View web file access records |
|
|
70
|
+
| `viewwebusage` | View web usage statistics |
|
|
71
|
+
|
|
72
|
+
## Environment setup
|
|
73
|
+
|
|
74
|
+
Create a Python environment first; package installs in the next section run
|
|
75
|
+
inside whichever environment you activate here.
|
|
76
|
+
|
|
77
|
+
### Option A — Python venv (DECS machines)
|
|
78
|
+
|
|
79
|
+
```bash
|
|
80
|
+
python3 -m venv $ENVHOME # e.g. /glade/u/home/gdexdata/gdexmsenv
|
|
81
|
+
source $ENVHOME/bin/activate
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
### Option B — Conda (DAV/Casper)
|
|
85
|
+
|
|
86
|
+
```bash
|
|
87
|
+
conda create --prefix $ENVHOME python=3.12 # e.g. /glade/work/gdexdata/conda-envs/pg-gdex
|
|
88
|
+
conda activate $ENVHOME
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
## Installing rda-python-metrics
|
|
92
|
+
|
|
93
|
+
Pick whichever install mode fits your workflow. All four pull in the
|
|
94
|
+
transitive dependencies (`rda_python_common`, `rda_python_setuid`, `geoip2`,
|
|
95
|
+
`ipinfo`, `httplib2`, `dnspython`, `unidecode`, `urllib3>=2.5.0`,
|
|
96
|
+
`requests>=2.33.0`, `idna>=3.10`) automatically.
|
|
97
|
+
|
|
98
|
+
For local development, clone this repo alongside your project and install it
|
|
99
|
+
in editable mode so that changes are picked up without re-installing:
|
|
100
|
+
|
|
101
|
+
```bash
|
|
102
|
+
git clone https://github.com/NCAR/rda-python-metrics.git
|
|
103
|
+
cd rda-python-metrics
|
|
104
|
+
pip install -e .
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
To test a specific branch (e.g. an in-progress feature or fix branch), pass
|
|
108
|
+
`-b/--branch` to `git clone`:
|
|
109
|
+
|
|
110
|
+
```bash
|
|
111
|
+
git clone -b <branch-name> https://github.com/NCAR/rda-python-metrics.git
|
|
112
|
+
cd rda-python-metrics
|
|
113
|
+
pip install -e .
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
For a regular (non-editable) install from a checkout:
|
|
117
|
+
|
|
118
|
+
```bash
|
|
119
|
+
pip install /path/to/rda-python-metrics
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
For a production install on a system that uses the published distribution:
|
|
123
|
+
|
|
124
|
+
```bash
|
|
125
|
+
pip install rda_python_metrics
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
## Setuid Setup
|
|
129
|
+
|
|
130
|
+
`logarch` runs as the common user `PGLOG['COMMONUSER']` (default `gdexdata`)
|
|
131
|
+
via the `rda_python_setuid` mechanism, which is pulled in automatically as
|
|
132
|
+
a dependency. After `pip install` above, choose one of the wiring options
|
|
133
|
+
below.
|
|
134
|
+
|
|
135
|
+
> **Note:** If `rda_python_setuid` is already installed and fully set up in
|
|
136
|
+
> your environment, you can skip the compile step (`-c/--compile`) and the
|
|
137
|
+
> optional `pgstart` step (`-p/--pgstart`). The `-l/--link` step is still
|
|
138
|
+
> required to wire up this package's own setuid program.
|
|
139
|
+
|
|
140
|
+
### Full setuid install (requires sudo access to COMMONUSER)
|
|
141
|
+
|
|
142
|
+
Run these steps once per environment:
|
|
143
|
+
|
|
144
|
+
```bash
|
|
145
|
+
# 1. Compile the pywrapper C binary (once per environment):
|
|
146
|
+
pywrapper-install -c|--compile -n|--username gdexdata
|
|
147
|
+
|
|
148
|
+
# 2. Wire up logarch as a setuid entry (or use 'all' to link every setuid_* at once):
|
|
149
|
+
pywrapper-install -l|--link logarch
|
|
150
|
+
pywrapper-install -l|--link all
|
|
151
|
+
|
|
152
|
+
# 3. Optionally, install a pgstart_<loginname> binary so <loginname> (any
|
|
153
|
+
# user in the same group as PGLOG['COMMONUSER']) can run commands as
|
|
154
|
+
# themselves. Run either by PGLOG['ADMINUSER'] (default zji, if it has
|
|
155
|
+
# 'sudo -u <loginname>'), or by <loginname> directly:
|
|
156
|
+
pywrapper-install -p|--pgstart -n|--username <loginname>
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
`pywrapper-install` with no arguments displays the full user guide.
|
|
160
|
+
|
|
161
|
+
### Simple install (no sudo required, runs as current user)
|
|
162
|
+
|
|
163
|
+
Users who do not need the setuid mechanism can create a direct symlink instead:
|
|
164
|
+
|
|
165
|
+
```bash
|
|
166
|
+
pywrapper-install -l|--link logarch -s|--simple
|
|
167
|
+
pywrapper-install -l|--link all -s|--simple # or link every setuid_* at once
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
This creates `bin/logarch -> bin/setuid_logarch` and logarch runs as the
|
|
171
|
+
current user with no privilege change.
|
|
172
|
+
|
|
173
|
+
### Update an existing installation (no sudo required)
|
|
174
|
+
|
|
175
|
+
When the package is upgraded and a new `pywrapper.c` is bundled, recompile and
|
|
176
|
+
reinstall all setuid binaries using the existing `pgstart_*` binaries:
|
|
177
|
+
|
|
178
|
+
```bash
|
|
179
|
+
pywrapper-install -u|--update
|
|
180
|
+
```
|
|
181
|
+
|
|
182
|
+
### Setup guide
|
|
183
|
+
|
|
184
|
+
The shared setuid setup guide is shown automatically if `setuid_logarch` is
|
|
185
|
+
invoked directly before the setuid wrapper has been configured.
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
# rda_python_metrics
|
|
2
|
+
|
|
3
|
+
RDA Python package to gather and view data usage metrics for the
|
|
4
|
+
[NSF NCAR Geoscience Data Exchange (GDEX)](https://gdex.ucar.edu).
|
|
5
|
+
|
|
6
|
+
## Programs
|
|
7
|
+
|
|
8
|
+
The package provides two categories of programs:
|
|
9
|
+
|
|
10
|
+
**Run as gdexdata via setuid (requires setup below):**
|
|
11
|
+
|
|
12
|
+
| Command | Connector script | Description |
|
|
13
|
+
|---------|-----------------|-------------|
|
|
14
|
+
| `logarch` | `setuid_logarch` | Archive RDA log files (web, TDS, AWS, OSDF, dssdb) to the DECSDATA area |
|
|
15
|
+
|
|
16
|
+
**Run as current user (no setuid required):**
|
|
17
|
+
|
|
18
|
+
| Command | Description |
|
|
19
|
+
|---------|-------------|
|
|
20
|
+
| `fillawsusage` | Fill AWS usage metrics into the database |
|
|
21
|
+
| `fillcdgusage` | Fill CDG usage metrics into the database |
|
|
22
|
+
| `fillcodusage` | Fill COD usage metrics into the database |
|
|
23
|
+
| `fillcountry` | Fill country information into the database |
|
|
24
|
+
| `fillendtime` | Fill end times for usage records |
|
|
25
|
+
| `fillgdexusage` | Fill GDEX usage metrics into the database |
|
|
26
|
+
| `fillglobususage` | Fill Globus transfer usage metrics into the database |
|
|
27
|
+
| `fillipinfo` | Fill IP geolocation info into the database |
|
|
28
|
+
| `filloneorder` | Fill metrics for a single order |
|
|
29
|
+
| `fillosdfusage` | Fill OSDF usage metrics into the database |
|
|
30
|
+
| `fillrdadb` | Fill RDA database usage metrics |
|
|
31
|
+
| `filltdsusage` | Fill TDS (THREDDS) usage metrics into the database |
|
|
32
|
+
| `filluser` | Fill user information into the database |
|
|
33
|
+
| `fillzenodousage` | Fill Zenodo usage metrics into the database |
|
|
34
|
+
| `pgperson` | Retrieve user info from the UCAR People API by field |
|
|
35
|
+
| `pgusername` | Retrieve user info from the UCAR People API by username |
|
|
36
|
+
| `viewallusage` | View combined usage statistics |
|
|
37
|
+
| `viewawsusage` | View AWS usage statistics |
|
|
38
|
+
| `viewcheckusage` | View usage check results |
|
|
39
|
+
| `viewcodusage` | View COD usage statistics |
|
|
40
|
+
| `viewordusage` | View order usage statistics |
|
|
41
|
+
| `viewosdfusage` | View OSDF usage statistics |
|
|
42
|
+
| `viewrqstusage` | View request usage statistics |
|
|
43
|
+
| `viewtdsusage` | View TDS usage statistics |
|
|
44
|
+
| `viewwebfile` | View web file access records |
|
|
45
|
+
| `viewwebusage` | View web usage statistics |
|
|
46
|
+
|
|
47
|
+
## Environment setup
|
|
48
|
+
|
|
49
|
+
Create a Python environment first; package installs in the next section run
|
|
50
|
+
inside whichever environment you activate here.
|
|
51
|
+
|
|
52
|
+
### Option A — Python venv (DECS machines)
|
|
53
|
+
|
|
54
|
+
```bash
|
|
55
|
+
python3 -m venv $ENVHOME # e.g. /glade/u/home/gdexdata/gdexmsenv
|
|
56
|
+
source $ENVHOME/bin/activate
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
### Option B — Conda (DAV/Casper)
|
|
60
|
+
|
|
61
|
+
```bash
|
|
62
|
+
conda create --prefix $ENVHOME python=3.12 # e.g. /glade/work/gdexdata/conda-envs/pg-gdex
|
|
63
|
+
conda activate $ENVHOME
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
## Installing rda-python-metrics
|
|
67
|
+
|
|
68
|
+
Pick whichever install mode fits your workflow. All four pull in the
|
|
69
|
+
transitive dependencies (`rda_python_common`, `rda_python_setuid`, `geoip2`,
|
|
70
|
+
`ipinfo`, `httplib2`, `dnspython`, `unidecode`, `urllib3>=2.5.0`,
|
|
71
|
+
`requests>=2.33.0`, `idna>=3.10`) automatically.
|
|
72
|
+
|
|
73
|
+
For local development, clone this repo alongside your project and install it
|
|
74
|
+
in editable mode so that changes are picked up without re-installing:
|
|
75
|
+
|
|
76
|
+
```bash
|
|
77
|
+
git clone https://github.com/NCAR/rda-python-metrics.git
|
|
78
|
+
cd rda-python-metrics
|
|
79
|
+
pip install -e .
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
To test a specific branch (e.g. an in-progress feature or fix branch), pass
|
|
83
|
+
`-b/--branch` to `git clone`:
|
|
84
|
+
|
|
85
|
+
```bash
|
|
86
|
+
git clone -b <branch-name> https://github.com/NCAR/rda-python-metrics.git
|
|
87
|
+
cd rda-python-metrics
|
|
88
|
+
pip install -e .
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
For a regular (non-editable) install from a checkout:
|
|
92
|
+
|
|
93
|
+
```bash
|
|
94
|
+
pip install /path/to/rda-python-metrics
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
For a production install on a system that uses the published distribution:
|
|
98
|
+
|
|
99
|
+
```bash
|
|
100
|
+
pip install rda_python_metrics
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
## Setuid Setup
|
|
104
|
+
|
|
105
|
+
`logarch` runs as the common user `PGLOG['COMMONUSER']` (default `gdexdata`)
|
|
106
|
+
via the `rda_python_setuid` mechanism, which is pulled in automatically as
|
|
107
|
+
a dependency. After `pip install` above, choose one of the wiring options
|
|
108
|
+
below.
|
|
109
|
+
|
|
110
|
+
> **Note:** If `rda_python_setuid` is already installed and fully set up in
|
|
111
|
+
> your environment, you can skip the compile step (`-c/--compile`) and the
|
|
112
|
+
> optional `pgstart` step (`-p/--pgstart`). The `-l/--link` step is still
|
|
113
|
+
> required to wire up this package's own setuid program.
|
|
114
|
+
|
|
115
|
+
### Full setuid install (requires sudo access to COMMONUSER)
|
|
116
|
+
|
|
117
|
+
Run these steps once per environment:
|
|
118
|
+
|
|
119
|
+
```bash
|
|
120
|
+
# 1. Compile the pywrapper C binary (once per environment):
|
|
121
|
+
pywrapper-install -c|--compile -n|--username gdexdata
|
|
122
|
+
|
|
123
|
+
# 2. Wire up logarch as a setuid entry (or use 'all' to link every setuid_* at once):
|
|
124
|
+
pywrapper-install -l|--link logarch
|
|
125
|
+
pywrapper-install -l|--link all
|
|
126
|
+
|
|
127
|
+
# 3. Optionally, install a pgstart_<loginname> binary so <loginname> (any
|
|
128
|
+
# user in the same group as PGLOG['COMMONUSER']) can run commands as
|
|
129
|
+
# themselves. Run either by PGLOG['ADMINUSER'] (default zji, if it has
|
|
130
|
+
# 'sudo -u <loginname>'), or by <loginname> directly:
|
|
131
|
+
pywrapper-install -p|--pgstart -n|--username <loginname>
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
`pywrapper-install` with no arguments displays the full user guide.
|
|
135
|
+
|
|
136
|
+
### Simple install (no sudo required, runs as current user)
|
|
137
|
+
|
|
138
|
+
Users who do not need the setuid mechanism can create a direct symlink instead:
|
|
139
|
+
|
|
140
|
+
```bash
|
|
141
|
+
pywrapper-install -l|--link logarch -s|--simple
|
|
142
|
+
pywrapper-install -l|--link all -s|--simple # or link every setuid_* at once
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
This creates `bin/logarch -> bin/setuid_logarch` and logarch runs as the
|
|
146
|
+
current user with no privilege change.
|
|
147
|
+
|
|
148
|
+
### Update an existing installation (no sudo required)
|
|
149
|
+
|
|
150
|
+
When the package is upgraded and a new `pywrapper.c` is bundled, recompile and
|
|
151
|
+
reinstall all setuid binaries using the existing `pgstart_*` binaries:
|
|
152
|
+
|
|
153
|
+
```bash
|
|
154
|
+
pywrapper-install -u|--update
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
### Setup guide
|
|
158
|
+
|
|
159
|
+
The shared setuid setup guide is shown automatically if `setuid_logarch` is
|
|
160
|
+
invoked directly before the setuid wrapper has been configured.
|
|
@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
|
|
|
6
6
|
|
|
7
7
|
[project]
|
|
8
8
|
name = "rda_python_metrics"
|
|
9
|
-
version = "
|
|
9
|
+
version = "3.0.2"
|
|
10
10
|
authors = [
|
|
11
11
|
{ name="Zaihua Ji", email="zji@ucar.edu" },
|
|
12
12
|
]
|
|
@@ -26,7 +26,10 @@ dependencies = [
|
|
|
26
26
|
"ipinfo",
|
|
27
27
|
"httplib2",
|
|
28
28
|
"dnspython",
|
|
29
|
-
"unidecode"
|
|
29
|
+
"unidecode",
|
|
30
|
+
"urllib3>=2.5.0",
|
|
31
|
+
"requests>=2.33.0",
|
|
32
|
+
"idna>=3.10",
|
|
30
33
|
]
|
|
31
34
|
|
|
32
35
|
[tool.pytest.ini_options]
|
|
@@ -38,7 +41,7 @@ pythonpath = [
|
|
|
38
41
|
"Homepage" = "https://github.com/NCAR/rda-python-metrics"
|
|
39
42
|
|
|
40
43
|
[project.scripts]
|
|
41
|
-
"
|
|
44
|
+
"setuid_logarch" = "rda_python_metrics.logarch:main"
|
|
42
45
|
"fillawsusage" = "rda_python_metrics.fillawsusage:main"
|
|
43
46
|
"fillcdgusage" = "rda_python_metrics.fillcdgusage:main"
|
|
44
47
|
"fillcodusage" = "rda_python_metrics.fillcodusage:main"
|
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
# Date : 09/24/2020
|
|
7
7
|
# 2025-03-27 transferred to package rda_python_metrics from
|
|
8
8
|
# https://github.com/NCAR/rda-database.git
|
|
9
|
-
# Purpose : python library module to help
|
|
9
|
+
# Purpose : python library module to help routinely updates of new data
|
|
10
10
|
# for one or multiple datasets
|
|
11
11
|
#
|
|
12
12
|
# Github : https://github.com/NCAR/rda-python-metrics.git
|
|
@@ -32,7 +32,7 @@ def simple_output(params, flds, records, totals = None):
|
|
|
32
32
|
sep = params['L'][0] if 'L' in params else ' '
|
|
33
33
|
slen = len(sep)
|
|
34
34
|
|
|
35
|
-
# get total line length, dynamically evaluating column
|
|
35
|
+
# get total line length, dynamically evaluating column lengths if column 4 in %FLDS is zero
|
|
36
36
|
rcnt = linelen = 0
|
|
37
37
|
if 'A' in params: rcnt = int(params['A'][0])
|
|
38
38
|
for i in range(ccnt):
|
|
@@ -390,7 +390,7 @@ def join_query_tables(tblname, tablenames = '', joins = '', tbljoin = ''):
|
|
|
390
390
|
return (tablenames, joins)
|
|
391
391
|
|
|
392
392
|
#
|
|
393
|
-
# expand
|
|
393
|
+
# expand records via query action
|
|
394
394
|
#
|
|
395
395
|
def expand_query(expid, records, params, expand, vusg = None, sns = None, flds = None):
|
|
396
396
|
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
"""rda_python_metrics: RDA metrics, usage, and accounting utility package.
|
|
2
|
+
|
|
3
|
+
This package exposes two parallel APIs:
|
|
4
|
+
|
|
5
|
+
1. Legacy module-based API (back-compat). Import the capitalized
|
|
6
|
+
submodules and call their module-level functions, e.g.::
|
|
7
|
+
|
|
8
|
+
from rda_python_metrics import PgIPInfo, PgView
|
|
9
|
+
|
|
10
|
+
2. Class-based API (preferred for new code). Import the class from the
|
|
11
|
+
lower-case module and either instantiate or subclass it, e.g.::
|
|
12
|
+
|
|
13
|
+
from rda_python_metrics.pg_ipinfo import PgIPInfo
|
|
14
|
+
from rda_python_metrics.pg_view import PgView
|
|
15
|
+
|
|
16
|
+
The legacy submodules are eagerly imported below so that
|
|
17
|
+
``from rda_python_metrics import PgIPInfo`` continues to return the
|
|
18
|
+
module object that existing callers expect.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from . import PgIPInfo, PgView
|
|
22
|
+
|
|
23
|
+
__version__ = "2.0.6"
|
|
24
|
+
|
|
25
|
+
__all__ = [
|
|
26
|
+
"PgIPInfo",
|
|
27
|
+
"PgView",
|
|
28
|
+
"__version__",
|
|
29
|
+
]
|
{rda_python_metrics-2.0.6 → rda_python_metrics-3.0.2}/src/rda_python_metrics/fill_awsusage.py
RENAMED
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
# Purpose : python program to retrieve info from AWS logs
|
|
11
11
|
# and fill table wusages in PgSQL database dssdb.
|
|
12
12
|
#
|
|
13
|
-
# Github : https://github.com/NCAR/rda-
|
|
13
|
+
# Github : https://github.com/NCAR/rda-python-metrics.git
|
|
14
14
|
#
|
|
15
15
|
###############################################################################
|
|
16
16
|
#
|
{rda_python_metrics-2.0.6 → rda_python_metrics-3.0.2}/src/rda_python_metrics/fill_cdgusage.py
RENAMED
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
# Author : Zaihua Ji, zji@ucar.edu
|
|
7
7
|
# Date : 2025-04-14
|
|
8
8
|
# Purpose : python program to retrieve info from GDEX Postgres database for GDS
|
|
9
|
-
# file accesses and backup fill table tdsusage in PostgreSQL database
|
|
9
|
+
# file accesses and backup fill table tdsusage in PostgreSQL database rdadb.
|
|
10
10
|
#
|
|
11
11
|
# Github : https://github.com/NCAR/rda-python-metrics.git
|
|
12
12
|
#
|
|
@@ -173,7 +173,7 @@ def get_dataset_ids(dsnames):
|
|
|
173
173
|
return dsids
|
|
174
174
|
|
|
175
175
|
#
|
|
176
|
-
# get cdgids
|
|
176
|
+
# get cdgids recursively
|
|
177
177
|
#
|
|
178
178
|
def recursive_dataset_ids(pcdgid, cdgids):
|
|
179
179
|
|
{rda_python_metrics-2.0.6 → rda_python_metrics-3.0.2}/src/rda_python_metrics/fill_globususage.py
RENAMED
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
# Purpose : python program to retrieve info from Globus logs
|
|
11
11
|
# and fill table wusages in PgSQL database dssdb.
|
|
12
12
|
#
|
|
13
|
-
# Github : https://github.com/NCAR/rda-
|
|
13
|
+
# Github : https://github.com/NCAR/rda-python-metrics.git
|
|
14
14
|
#
|
|
15
15
|
###############################################################################
|
|
16
16
|
#
|
{rda_python_metrics-2.0.6 → rda_python_metrics-3.0.2}/src/rda_python_metrics/fill_osdfusage.py
RENAMED
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
# Purpose : python program to retrieve info from weekly OSDF logs
|
|
9
9
|
# and fill table wusages in PgSQL database dssdb.
|
|
10
10
|
#
|
|
11
|
-
# Github : https://github.com/NCAR/rda-
|
|
11
|
+
# Github : https://github.com/NCAR/rda-python-metrics.git
|
|
12
12
|
#
|
|
13
13
|
###############################################################################
|
|
14
14
|
#
|
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
# Purpose : python program to retrieve info from data logs, and fill tables
|
|
11
11
|
# in PostgreSQL database.schema rdadb.dssdb
|
|
12
12
|
#
|
|
13
|
-
# Github : https://github.com/NCAR/rda-
|
|
13
|
+
# Github : https://github.com/NCAR/rda-python-metrics.git
|
|
14
14
|
#
|
|
15
15
|
###############################################################################
|
|
16
16
|
#
|
{rda_python_metrics-2.0.6 → rda_python_metrics-3.0.2}/src/rda_python_metrics/fill_tdsusage.py
RENAMED
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
# 2025-03-26 transferred to package rda_python_metrics from
|
|
9
9
|
# https://github.com/NCAR/rda-database.git
|
|
10
10
|
# Purpose : python program to retrieve info from TDS logs
|
|
11
|
-
# and fill table tdsusage in PostgreSQL database
|
|
11
|
+
# and fill table tdsusage in PostgreSQL database rdadb.
|
|
12
12
|
#
|
|
13
13
|
# Github : https://github.com/NCAR/rda-python-metrics.git
|
|
14
14
|
#
|
{rda_python_metrics-2.0.6 → rda_python_metrics-3.0.2}/src/rda_python_metrics/fillawsusage.py
RENAMED
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
# 2025-12-16 converted to class FillAWSUsage
|
|
9
9
|
# Purpose : python program to retrieve info from AWS logs
|
|
10
10
|
# and fill table wusages in PgSQL database dssdb.
|
|
11
|
-
# Github : https://github.com/NCAR/rda-
|
|
11
|
+
# Github : https://github.com/NCAR/rda-python-metrics.git
|
|
12
12
|
###############################################################################
|
|
13
13
|
import sys
|
|
14
14
|
import re
|
|
@@ -19,6 +19,8 @@ from .pg_ipinfo import PgIPInfo
|
|
|
19
19
|
|
|
20
20
|
class FillAWSUsage(PgIPInfo, PgFile):
|
|
21
21
|
|
|
22
|
+
"""Retrieve info from AWS logs and fill table wusage in PostgreSQL database rdadb."""
|
|
23
|
+
|
|
22
24
|
def __init__(self):
|
|
23
25
|
super().__init__()
|
|
24
26
|
self.USAGE = {
|
|
@@ -31,8 +33,8 @@ class FillAWSUsage(PgIPInfo, PgFile):
|
|
|
31
33
|
self.option = self.cmdstr = None
|
|
32
34
|
self.params = [] # array of input values
|
|
33
35
|
|
|
34
|
-
# function to read parameters
|
|
35
36
|
def read_parameters(self):
|
|
37
|
+
"""Function to read parameters."""
|
|
36
38
|
argv = sys.argv[1:]
|
|
37
39
|
for arg in argv:
|
|
38
40
|
ms = re.match(r'^-(b|d|p|N)$', arg)
|
|
@@ -55,8 +57,8 @@ class FillAWSUsage(PgIPInfo, PgFile):
|
|
|
55
57
|
self.cmdstr = "fillawsusage {}".format(' '.join(argv))
|
|
56
58
|
self.cmdlog(self.cmdstr)
|
|
57
59
|
|
|
58
|
-
# function to start actions
|
|
59
60
|
def start_actions(self):
|
|
61
|
+
"""Function to start actions."""
|
|
60
62
|
self.change_local_directory(self.USAGE['AWSDIR'])
|
|
61
63
|
filenames = self.get_log_file_names()
|
|
62
64
|
if filenames:
|
|
@@ -65,8 +67,8 @@ class FillAWSUsage(PgIPInfo, PgFile):
|
|
|
65
67
|
self.pglog("No log file found for given command: " + self.cmdstr, self.LOGWRN)
|
|
66
68
|
self.pglog(None, self.LOGWRN)
|
|
67
69
|
|
|
68
|
-
# get the log file dates
|
|
69
70
|
def get_log_file_names(self):
|
|
71
|
+
"""Get the log file dates."""
|
|
70
72
|
filenames = {}
|
|
71
73
|
if self.option == 'd':
|
|
72
74
|
for dt in self.params:
|
|
@@ -93,8 +95,8 @@ class FillAWSUsage(PgIPInfo, PgFile):
|
|
|
93
95
|
pdate = self.adddate(pdate, 0, 0, 1)
|
|
94
96
|
return filenames
|
|
95
97
|
|
|
96
|
-
# Fill AWS usages into table dssdb.awsusage of DSS PgSQL database from aws access logs
|
|
97
98
|
def fill_aws_usages(self, filenames):
|
|
99
|
+
"""Fill AWS usages into table dssdb.awsusage of DSS PgSQL database from aws access logs."""
|
|
98
100
|
year = cntall = addall = 0
|
|
99
101
|
for pdate in filenames:
|
|
100
102
|
fnames = filenames[pdate]
|
|
@@ -147,8 +149,8 @@ class FillAWSUsage(PgIPInfo, PgFile):
|
|
|
147
149
|
if addall > cntadd:
|
|
148
150
|
self.pglog("{} AWS usage records added for {} entries at {}".format(addall, cntall, self.current_datetime()), self.LOGWRN)
|
|
149
151
|
|
|
150
|
-
# get date and time from record
|
|
151
152
|
def get_record_date_time(self, ctime):
|
|
153
|
+
"""Get date and time from record."""
|
|
152
154
|
ms = re.search(r'^(\d+)/(\w+)/(\d+):(\d+:\d+:\d+)$', ctime)
|
|
153
155
|
if ms:
|
|
154
156
|
d = int(ms.group(1))
|
|
@@ -160,8 +162,8 @@ class FillAWSUsage(PgIPInfo, PgFile):
|
|
|
160
162
|
else:
|
|
161
163
|
self.pglog(ctime + ": Invalid date/time format", self.LGEREX)
|
|
162
164
|
|
|
163
|
-
# add usage records for year
|
|
164
165
|
def add_usage_records(self, records, year):
|
|
166
|
+
"""Add usage records for year."""
|
|
165
167
|
cnt = 0
|
|
166
168
|
for key in records:
|
|
167
169
|
record = records[key]
|
|
@@ -171,8 +173,8 @@ class FillAWSUsage(PgIPInfo, PgFile):
|
|
|
171
173
|
cnt += self.pgadd(self.USAGE['PGTBL'], record, self.LOGWRN)
|
|
172
174
|
return cnt
|
|
173
175
|
|
|
174
|
-
# add record to allusage tables
|
|
175
176
|
def add_to_allusage(self, year, pgrec):
|
|
177
|
+
"""Add record to allusage tables."""
|
|
176
178
|
record = {'source' : 'A'}
|
|
177
179
|
flds = ['ip', 'dsid', 'date', 'time', 'quarter', 'size', 'method',
|
|
178
180
|
'org_type', 'country', 'region', 'email']
|
|
@@ -180,7 +182,7 @@ class FillAWSUsage(PgIPInfo, PgFile):
|
|
|
180
182
|
record[fld] = pgrec[fld]
|
|
181
183
|
return self.add_yearly_allusage(year, record)
|
|
182
184
|
|
|
183
|
-
# main function to
|
|
185
|
+
# main function to execute this script
|
|
184
186
|
def main():
|
|
185
187
|
object = FillAWSUsage()
|
|
186
188
|
object.read_parameters()
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
|
|
2
|
+
fillawsusage - Fill table 'awsusage' from AWS Server log files.
|
|
3
|
+
|
|
4
|
+
Reads AWS access logs from /gpfs/fs1/collections/rda/transer/AWSera5log/
|
|
5
|
+
and inserts usage records into the 'awsusage' table in PostgreSQL
|
|
6
|
+
database 'rdadb'.
|
|
7
|
+
|
|
8
|
+
USAGE:
|
|
9
|
+
fillawsusage [-b] (-d LogFileDates | -N NumberDays | -p BeginDate [EndDate])
|
|
10
|
+
|
|
11
|
+
OPTIONS (choose exactly one of -d, -N, or -p):
|
|
12
|
+
|
|
13
|
+
-d LogFileDates
|
|
14
|
+
Process log files for the given dates.
|
|
15
|
+
Date format: YYYY-MM-DD (e.g. 2024-01-15).
|
|
16
|
+
|
|
17
|
+
-N NumberDays
|
|
18
|
+
Process log files covering the most recent NumberDays days.
|
|
19
|
+
|
|
20
|
+
-p BeginDate [EndDate]
|
|
21
|
+
Process log files between BeginDate and EndDate, inclusive.
|
|
22
|
+
EndDate defaults to today if omitted.
|
|
23
|
+
|
|
24
|
+
-b Run in background mode (log to file only, no terminal output).
|
|
25
|
+
|
|
26
|
+
EXAMPLE:
|
|
27
|
+
fillawsusage -p 2024-01-01 2024-01-31
|