rda-python-metrics 2.0.5__tar.gz → 3.0.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (122) hide show
  1. rda_python_metrics-3.0.2/PKG-INFO +185 -0
  2. rda_python_metrics-3.0.2/README.md +160 -0
  3. {rda_python_metrics-2.0.5 → rda_python_metrics-3.0.2}/pyproject.toml +6 -3
  4. {rda_python_metrics-2.0.5 → rda_python_metrics-3.0.2}/src/rda_python_metrics/PgView.py +3 -3
  5. rda_python_metrics-3.0.2/src/rda_python_metrics/__init__.py +29 -0
  6. {rda_python_metrics-2.0.5 → rda_python_metrics-3.0.2}/src/rda_python_metrics/fill_awsusage.py +1 -1
  7. {rda_python_metrics-2.0.5 → rda_python_metrics-3.0.2}/src/rda_python_metrics/fill_cdgusage.py +2 -2
  8. {rda_python_metrics-2.0.5 → rda_python_metrics-3.0.2}/src/rda_python_metrics/fill_globususage.py +1 -1
  9. {rda_python_metrics-2.0.5 → rda_python_metrics-3.0.2}/src/rda_python_metrics/fill_osdfusage.py +1 -1
  10. {rda_python_metrics-2.0.5 → rda_python_metrics-3.0.2}/src/rda_python_metrics/fill_rdadb.py +1 -1
  11. {rda_python_metrics-2.0.5 → rda_python_metrics-3.0.2}/src/rda_python_metrics/fill_tdsusage.py +1 -1
  12. {rda_python_metrics-2.0.5 → rda_python_metrics-3.0.2}/src/rda_python_metrics/fillawsusage.py +11 -9
  13. rda_python_metrics-3.0.2/src/rda_python_metrics/fillawsusage.usg +27 -0
  14. {rda_python_metrics-2.0.5 → rda_python_metrics-3.0.2}/src/rda_python_metrics/fillcdgusage.py +17 -15
  15. rda_python_metrics-3.0.2/src/rda_python_metrics/fillcdgusage.usg +33 -0
  16. {rda_python_metrics-2.0.5 → rda_python_metrics-3.0.2}/src/rda_python_metrics/fillcodusage.py +11 -9
  17. rda_python_metrics-3.0.2/src/rda_python_metrics/fillcodusage.usg +32 -0
  18. {rda_python_metrics-2.0.5 → rda_python_metrics-3.0.2}/src/rda_python_metrics/fillcountry.py +8 -6
  19. {rda_python_metrics-2.0.5 → rda_python_metrics-3.0.2}/src/rda_python_metrics/fillendtime.py +7 -5
  20. rda_python_metrics-3.0.2/src/rda_python_metrics/fillgdexusage.py +1025 -0
  21. rda_python_metrics-3.0.2/src/rda_python_metrics/fillgdexusage.usg +33 -0
  22. {rda_python_metrics-2.0.5 → rda_python_metrics-3.0.2}/src/rda_python_metrics/fillglobususage.py +14 -12
  23. rda_python_metrics-3.0.2/src/rda_python_metrics/fillglobususage.usg +31 -0
  24. {rda_python_metrics-2.0.5 → rda_python_metrics-3.0.2}/src/rda_python_metrics/fillipinfo.py +15 -15
  25. rda_python_metrics-3.0.2/src/rda_python_metrics/fillipinfo.usg +38 -0
  26. {rda_python_metrics-2.0.5 → rda_python_metrics-3.0.2}/src/rda_python_metrics/filloneorder.py +9 -7
  27. rda_python_metrics-3.0.2/src/rda_python_metrics/filloneorder.usg +39 -0
  28. {rda_python_metrics-2.0.5 → rda_python_metrics-3.0.2}/src/rda_python_metrics/fillosdfusage.py +11 -9
  29. rda_python_metrics-3.0.2/src/rda_python_metrics/fillosdfusage.usg +27 -0
  30. {rda_python_metrics-2.0.5 → rda_python_metrics-3.0.2}/src/rda_python_metrics/fillrdadb.py +9 -7
  31. rda_python_metrics-3.0.2/src/rda_python_metrics/fillrdadb.usg +36 -0
  32. {rda_python_metrics-2.0.5 → rda_python_metrics-3.0.2}/src/rda_python_metrics/filltdsusage.py +42 -39
  33. rda_python_metrics-3.0.2/src/rda_python_metrics/filltdsusage.usg +27 -0
  34. {rda_python_metrics-2.0.5 → rda_python_metrics-3.0.2}/src/rda_python_metrics/filluser.py +8 -6
  35. rda_python_metrics-3.0.2/src/rda_python_metrics/filluser.usg +24 -0
  36. rda_python_metrics-3.0.2/src/rda_python_metrics/fillzenodousage.py +397 -0
  37. {rda_python_metrics-2.0.5 → rda_python_metrics-3.0.2}/src/rda_python_metrics/log_arch.py +2 -2
  38. {rda_python_metrics-2.0.5 → rda_python_metrics-3.0.2}/src/rda_python_metrics/logarch.py +5 -2
  39. rda_python_metrics-3.0.2/src/rda_python_metrics/logarch.usg +34 -0
  40. {rda_python_metrics-2.0.5 → rda_python_metrics-3.0.2}/src/rda_python_metrics/pg_ipinfo.py +29 -28
  41. {rda_python_metrics-2.0.5 → rda_python_metrics-3.0.2}/src/rda_python_metrics/pg_person.py +1 -1
  42. {rda_python_metrics-2.0.5 → rda_python_metrics-3.0.2}/src/rda_python_metrics/pg_username.py +1 -1
  43. {rda_python_metrics-2.0.5 → rda_python_metrics-3.0.2}/src/rda_python_metrics/pg_view.py +25 -30
  44. {rda_python_metrics-2.0.5 → rda_python_metrics-3.0.2}/src/rda_python_metrics/pgperson.py +4 -4
  45. {rda_python_metrics-2.0.5 → rda_python_metrics-3.0.2}/src/rda_python_metrics/pgsyspath.py +1 -1
  46. {rda_python_metrics-2.0.5 → rda_python_metrics-3.0.2}/src/rda_python_metrics/pgusername.py +2 -2
  47. {rda_python_metrics-2.0.5 → rda_python_metrics-3.0.2}/src/rda_python_metrics/view_allusage.py +9 -9
  48. {rda_python_metrics-2.0.5 → rda_python_metrics-3.0.2}/src/rda_python_metrics/view_awsusage.py +13 -13
  49. {rda_python_metrics-2.0.5 → rda_python_metrics-3.0.2}/src/rda_python_metrics/view_checkusage.py +10 -10
  50. {rda_python_metrics-2.0.5 → rda_python_metrics-3.0.2}/src/rda_python_metrics/view_codusage.py +8 -8
  51. {rda_python_metrics-2.0.5 → rda_python_metrics-3.0.2}/src/rda_python_metrics/view_ordusage.py +14 -14
  52. {rda_python_metrics-2.0.5 → rda_python_metrics-3.0.2}/src/rda_python_metrics/view_osdfusage.py +13 -13
  53. {rda_python_metrics-2.0.5 → rda_python_metrics-3.0.2}/src/rda_python_metrics/view_rqstusage.py +9 -9
  54. {rda_python_metrics-2.0.5 → rda_python_metrics-3.0.2}/src/rda_python_metrics/view_tdsusage.py +13 -13
  55. {rda_python_metrics-2.0.5 → rda_python_metrics-3.0.2}/src/rda_python_metrics/view_webfile.py +11 -13
  56. {rda_python_metrics-2.0.5 → rda_python_metrics-3.0.2}/src/rda_python_metrics/view_webusage.py +11 -11
  57. {rda_python_metrics-2.0.5 → rda_python_metrics-3.0.2}/src/rda_python_metrics/viewallusage.py +21 -16
  58. rda_python_metrics-3.0.2/src/rda_python_metrics/viewallusage.usg +150 -0
  59. {rda_python_metrics-2.0.5 → rda_python_metrics-3.0.2}/src/rda_python_metrics/viewawsusage.py +25 -20
  60. rda_python_metrics-3.0.2/src/rda_python_metrics/viewawsusage.usg +123 -0
  61. {rda_python_metrics-2.0.5 → rda_python_metrics-3.0.2}/src/rda_python_metrics/viewcheckusage.py +18 -17
  62. rda_python_metrics-3.0.2/src/rda_python_metrics/viewcheckusage.usg +131 -0
  63. rda_python_metrics-3.0.2/src/rda_python_metrics/viewcodusage.py +291 -0
  64. rda_python_metrics-3.0.2/src/rda_python_metrics/viewcodusage.usg +123 -0
  65. {rda_python_metrics-2.0.5 → rda_python_metrics-3.0.2}/src/rda_python_metrics/viewordusage.py +21 -20
  66. rda_python_metrics-3.0.2/src/rda_python_metrics/viewordusage.usg +131 -0
  67. {rda_python_metrics-2.0.5 → rda_python_metrics-3.0.2}/src/rda_python_metrics/viewosdfusage.py +27 -22
  68. rda_python_metrics-3.0.2/src/rda_python_metrics/viewosdfusage.usg +123 -0
  69. {rda_python_metrics-2.0.5 → rda_python_metrics-3.0.2}/src/rda_python_metrics/viewrqstusage.py +24 -19
  70. rda_python_metrics-3.0.2/src/rda_python_metrics/viewrqstusage.usg +142 -0
  71. {rda_python_metrics-2.0.5 → rda_python_metrics-3.0.2}/src/rda_python_metrics/viewtdsusage.py +25 -20
  72. rda_python_metrics-3.0.2/src/rda_python_metrics/viewtdsusage.usg +127 -0
  73. {rda_python_metrics-2.0.5 → rda_python_metrics-3.0.2}/src/rda_python_metrics/viewwebfile.py +17 -18
  74. rda_python_metrics-3.0.2/src/rda_python_metrics/viewwebfile.usg +143 -0
  75. {rda_python_metrics-2.0.5 → rda_python_metrics-3.0.2}/src/rda_python_metrics/viewwebusage.py +19 -18
  76. rda_python_metrics-3.0.2/src/rda_python_metrics/viewwebusage.usg +141 -0
  77. rda_python_metrics-3.0.2/src/rda_python_metrics.egg-info/PKG-INFO +185 -0
  78. {rda_python_metrics-2.0.5 → rda_python_metrics-3.0.2}/src/rda_python_metrics.egg-info/entry_points.txt +1 -1
  79. {rda_python_metrics-2.0.5 → rda_python_metrics-3.0.2}/src/rda_python_metrics.egg-info/requires.txt +3 -0
  80. rda_python_metrics-2.0.5/PKG-INFO +0 -23
  81. rda_python_metrics-2.0.5/README.md +0 -1
  82. rda_python_metrics-2.0.5/src/rda_python_metrics/__init__.py +0 -1
  83. rda_python_metrics-2.0.5/src/rda_python_metrics/fillawsusage.usg +0 -16
  84. rda_python_metrics-2.0.5/src/rda_python_metrics/fillcdgusage.usg +0 -18
  85. rda_python_metrics-2.0.5/src/rda_python_metrics/fillcodusage.usg +0 -21
  86. rda_python_metrics-2.0.5/src/rda_python_metrics/fillgdexusage.py +0 -1032
  87. rda_python_metrics-2.0.5/src/rda_python_metrics/fillgdexusage.usg +0 -18
  88. rda_python_metrics-2.0.5/src/rda_python_metrics/fillglobususage.usg +0 -19
  89. rda_python_metrics-2.0.5/src/rda_python_metrics/fillipinfo.usg +0 -20
  90. rda_python_metrics-2.0.5/src/rda_python_metrics/filloneorder.usg +0 -41
  91. rda_python_metrics-2.0.5/src/rda_python_metrics/fillosdfusage.usg +0 -17
  92. rda_python_metrics-2.0.5/src/rda_python_metrics/fillrdadb.usg +0 -32
  93. rda_python_metrics-2.0.5/src/rda_python_metrics/filltdsusage.usg +0 -16
  94. rda_python_metrics-2.0.5/src/rda_python_metrics/filluser.usg +0 -16
  95. rda_python_metrics-2.0.5/src/rda_python_metrics/fillzenodousage.py +0 -408
  96. rda_python_metrics-2.0.5/src/rda_python_metrics/logarch.usg +0 -29
  97. rda_python_metrics-2.0.5/src/rda_python_metrics/viewallusage.usg +0 -215
  98. rda_python_metrics-2.0.5/src/rda_python_metrics/viewawsusage.usg +0 -190
  99. rda_python_metrics-2.0.5/src/rda_python_metrics/viewcheckusage.usg +0 -185
  100. rda_python_metrics-2.0.5/src/rda_python_metrics/viewcodusage.py +0 -286
  101. rda_python_metrics-2.0.5/src/rda_python_metrics/viewcodusage.usg +0 -187
  102. rda_python_metrics-2.0.5/src/rda_python_metrics/viewordusage.usg +0 -223
  103. rda_python_metrics-2.0.5/src/rda_python_metrics/viewosdfusage.usg +0 -190
  104. rda_python_metrics-2.0.5/src/rda_python_metrics/viewrqstusage.usg +0 -217
  105. rda_python_metrics-2.0.5/src/rda_python_metrics/viewtdsusage.usg +0 -194
  106. rda_python_metrics-2.0.5/src/rda_python_metrics/viewwebfile.usg +0 -212
  107. rda_python_metrics-2.0.5/src/rda_python_metrics/viewwebusage.usg +0 -211
  108. rda_python_metrics-2.0.5/src/rda_python_metrics.egg-info/PKG-INFO +0 -23
  109. {rda_python_metrics-2.0.5 → rda_python_metrics-3.0.2}/LICENSE +0 -0
  110. {rda_python_metrics-2.0.5 → rda_python_metrics-3.0.2}/MANIFEST.in +0 -0
  111. {rda_python_metrics-2.0.5 → rda_python_metrics-3.0.2}/setup.cfg +0 -0
  112. {rda_python_metrics-2.0.5 → rda_python_metrics-3.0.2}/src/rda_python_metrics/PgIPInfo.py +0 -0
  113. {rda_python_metrics-2.0.5 → rda_python_metrics-3.0.2}/src/rda_python_metrics/fill_codusage.py +0 -0
  114. {rda_python_metrics-2.0.5 → rda_python_metrics-3.0.2}/src/rda_python_metrics/fill_country.py +0 -0
  115. {rda_python_metrics-2.0.5 → rda_python_metrics-3.0.2}/src/rda_python_metrics/fill_endtime.py +0 -0
  116. {rda_python_metrics-2.0.5 → rda_python_metrics-3.0.2}/src/rda_python_metrics/fill_ipinfo.py +0 -0
  117. {rda_python_metrics-2.0.5 → rda_python_metrics-3.0.2}/src/rda_python_metrics/fill_oneorder.py +0 -0
  118. {rda_python_metrics-2.0.5 → rda_python_metrics-3.0.2}/src/rda_python_metrics/fill_user.py +0 -0
  119. {rda_python_metrics-2.0.5 → rda_python_metrics-3.0.2}/src/rda_python_metrics.egg-info/SOURCES.txt +0 -0
  120. {rda_python_metrics-2.0.5 → rda_python_metrics-3.0.2}/src/rda_python_metrics.egg-info/dependency_links.txt +0 -0
  121. {rda_python_metrics-2.0.5 → rda_python_metrics-3.0.2}/src/rda_python_metrics.egg-info/top_level.txt +0 -0
  122. {rda_python_metrics-2.0.5 → rda_python_metrics-3.0.2}/tests/test_metrics.py +0 -0
@@ -0,0 +1,185 @@
1
+ Metadata-Version: 2.4
2
+ Name: rda_python_metrics
3
+ Version: 3.0.2
4
+ Summary: RDA Python Package to gather and view data usage metrics
5
+ Author-email: Zaihua Ji <zji@ucar.edu>
6
+ Project-URL: Homepage, https://github.com/NCAR/rda-python-metrics
7
+ Classifier: Programming Language :: Python :: 3
8
+ Classifier: License :: OSI Approved :: MIT License
9
+ Classifier: Operating System :: OS Independent
10
+ Classifier: Development Status :: 5 - Production/Stable
11
+ Requires-Python: >=3.7
12
+ Description-Content-Type: text/markdown
13
+ License-File: LICENSE
14
+ Requires-Dist: rda_python_common
15
+ Requires-Dist: rda_python_setuid
16
+ Requires-Dist: geoip2
17
+ Requires-Dist: ipinfo
18
+ Requires-Dist: httplib2
19
+ Requires-Dist: dnspython
20
+ Requires-Dist: unidecode
21
+ Requires-Dist: urllib3>=2.5.0
22
+ Requires-Dist: requests>=2.33.0
23
+ Requires-Dist: idna>=3.10
24
+ Dynamic: license-file
25
+
26
+ # rda_python_metrics
27
+
28
+ RDA Python package to gather and view data usage metrics for the
29
+ [NSF NCAR Geoscience Data Exchange (GDEX)](https://gdex.ucar.edu).
30
+
31
+ ## Programs
32
+
33
+ The package provides two categories of programs:
34
+
35
+ **Run as gdexdata via setuid (requires setup below):**
36
+
37
+ | Command | Connector script | Description |
38
+ |---------|-----------------|-------------|
39
+ | `logarch` | `setuid_logarch` | Archive RDA log files (web, TDS, AWS, OSDF, dssdb) to the DECSDATA area |
40
+
41
+ **Run as current user (no setuid required):**
42
+
43
+ | Command | Description |
44
+ |---------|-------------|
45
+ | `fillawsusage` | Fill AWS usage metrics into the database |
46
+ | `fillcdgusage` | Fill CDG usage metrics into the database |
47
+ | `fillcodusage` | Fill COD usage metrics into the database |
48
+ | `fillcountry` | Fill country information into the database |
49
+ | `fillendtime` | Fill end times for usage records |
50
+ | `fillgdexusage` | Fill GDEX usage metrics into the database |
51
+ | `fillglobususage` | Fill Globus transfer usage metrics into the database |
52
+ | `fillipinfo` | Fill IP geolocation info into the database |
53
+ | `filloneorder` | Fill metrics for a single order |
54
+ | `fillosdfusage` | Fill OSDF usage metrics into the database |
55
+ | `fillrdadb` | Fill RDA database usage metrics |
56
+ | `filltdsusage` | Fill TDS (THREDDS) usage metrics into the database |
57
+ | `filluser` | Fill user information into the database |
58
+ | `fillzenodousage` | Fill Zenodo usage metrics into the database |
59
+ | `pgperson` | Retrieve user info from the UCAR People API by field |
60
+ | `pgusername` | Retrieve user info from the UCAR People API by username |
61
+ | `viewallusage` | View combined usage statistics |
62
+ | `viewawsusage` | View AWS usage statistics |
63
+ | `viewcheckusage` | View usage check results |
64
+ | `viewcodusage` | View COD usage statistics |
65
+ | `viewordusage` | View order usage statistics |
66
+ | `viewosdfusage` | View OSDF usage statistics |
67
+ | `viewrqstusage` | View request usage statistics |
68
+ | `viewtdsusage` | View TDS usage statistics |
69
+ | `viewwebfile` | View web file access records |
70
+ | `viewwebusage` | View web usage statistics |
71
+
72
+ ## Environment setup
73
+
74
+ Create a Python environment first; package installs in the next section run
75
+ inside whichever environment you activate here.
76
+
77
+ ### Option A — Python venv (DECS machines)
78
+
79
+ ```bash
80
+ python3 -m venv $ENVHOME # e.g. /glade/u/home/gdexdata/gdexmsenv
81
+ source $ENVHOME/bin/activate
82
+ ```
83
+
84
+ ### Option B — Conda (DAV/Casper)
85
+
86
+ ```bash
87
+ conda create --prefix $ENVHOME python=3.12 # e.g. /glade/work/gdexdata/conda-envs/pg-gdex
88
+ conda activate $ENVHOME
89
+ ```
90
+
91
+ ## Installing rda-python-metrics
92
+
93
+ Pick whichever install mode fits your workflow. All four pull in the
94
+ transitive dependencies (`rda_python_common`, `rda_python_setuid`, `geoip2`,
95
+ `ipinfo`, `httplib2`, `dnspython`, `unidecode`, `urllib3>=2.5.0`,
96
+ `requests>=2.33.0`, `idna>=3.10`) automatically.
97
+
98
+ For local development, clone this repo alongside your project and install it
99
+ in editable mode so that changes are picked up without re-installing:
100
+
101
+ ```bash
102
+ git clone https://github.com/NCAR/rda-python-metrics.git
103
+ cd rda-python-metrics
104
+ pip install -e .
105
+ ```
106
+
107
+ To test a specific branch (e.g. an in-progress feature or fix branch), pass
108
+ `-b/--branch` to `git clone`:
109
+
110
+ ```bash
111
+ git clone -b <branch-name> https://github.com/NCAR/rda-python-metrics.git
112
+ cd rda-python-metrics
113
+ pip install -e .
114
+ ```
115
+
116
+ For a regular (non-editable) install from a checkout:
117
+
118
+ ```bash
119
+ pip install /path/to/rda-python-metrics
120
+ ```
121
+
122
+ For a production install on a system that uses the published distribution:
123
+
124
+ ```bash
125
+ pip install rda_python_metrics
126
+ ```
127
+
128
+ ## Setuid Setup
129
+
130
+ `logarch` runs as the common user `PGLOG['COMMONUSER']` (default `gdexdata`)
131
+ via the `rda_python_setuid` mechanism, which is pulled in automatically as
132
+ a dependency. After `pip install` above, choose one of the wiring options
133
+ below.
134
+
135
+ > **Note:** If `rda_python_setuid` is already installed and fully set up in
136
+ > your environment, you can skip the compile step (`-c/--compile`) and the
137
+ > optional `pgstart` step (`-p/--pgstart`). The `-l/--link` step is still
138
+ > required to wire up this package's own setuid program.
139
+
140
+ ### Full setuid install (requires sudo access to COMMONUSER)
141
+
142
+ Run these steps once per environment:
143
+
144
+ ```bash
145
+ # 1. Compile the pywrapper C binary (once per environment):
146
+ pywrapper-install -c|--compile -n|--username gdexdata
147
+
148
+ # 2. Wire up logarch as a setuid entry (or use 'all' to link every setuid_* at once):
149
+ pywrapper-install -l|--link logarch
150
+ pywrapper-install -l|--link all
151
+
152
+ # 3. Optionally, install a pgstart_<loginname> binary so <loginname> (any
153
+ # user in the same group as PGLOG['COMMONUSER']) can run commands as
154
+ # themselves. Run either by PGLOG['ADMINUSER'] (default zji, if it has
155
+ # 'sudo -u <loginname>'), or by <loginname> directly:
156
+ pywrapper-install -p|--pgstart -n|--username <loginname>
157
+ ```
158
+
159
+ `pywrapper-install` with no arguments displays the full user guide.
160
+
161
+ ### Simple install (no sudo required, runs as current user)
162
+
163
+ Users who do not need the setuid mechanism can create a direct symlink instead:
164
+
165
+ ```bash
166
+ pywrapper-install -l|--link logarch -s|--simple
167
+ pywrapper-install -l|--link all -s|--simple # or link every setuid_* at once
168
+ ```
169
+
170
+ This creates `bin/logarch -> bin/setuid_logarch` and logarch runs as the
171
+ current user with no privilege change.
172
+
173
+ ### Update an existing installation (no sudo required)
174
+
175
+ When the package is upgraded and a new `pywrapper.c` is bundled, recompile and
176
+ reinstall all setuid binaries using the existing `pgstart_*` binaries:
177
+
178
+ ```bash
179
+ pywrapper-install -u|--update
180
+ ```
181
+
182
+ ### Setup guide
183
+
184
+ The shared setuid setup guide is shown automatically if `setuid_logarch` is
185
+ invoked directly before the setuid wrapper has been configured.
@@ -0,0 +1,160 @@
1
+ # rda_python_metrics
2
+
3
+ RDA Python package to gather and view data usage metrics for the
4
+ [NSF NCAR Geoscience Data Exchange (GDEX)](https://gdex.ucar.edu).
5
+
6
+ ## Programs
7
+
8
+ The package provides two categories of programs:
9
+
10
+ **Run as gdexdata via setuid (requires setup below):**
11
+
12
+ | Command | Connector script | Description |
13
+ |---------|-----------------|-------------|
14
+ | `logarch` | `setuid_logarch` | Archive RDA log files (web, TDS, AWS, OSDF, dssdb) to the DECSDATA area |
15
+
16
+ **Run as current user (no setuid required):**
17
+
18
+ | Command | Description |
19
+ |---------|-------------|
20
+ | `fillawsusage` | Fill AWS usage metrics into the database |
21
+ | `fillcdgusage` | Fill CDG usage metrics into the database |
22
+ | `fillcodusage` | Fill COD usage metrics into the database |
23
+ | `fillcountry` | Fill country information into the database |
24
+ | `fillendtime` | Fill end times for usage records |
25
+ | `fillgdexusage` | Fill GDEX usage metrics into the database |
26
+ | `fillglobususage` | Fill Globus transfer usage metrics into the database |
27
+ | `fillipinfo` | Fill IP geolocation info into the database |
28
+ | `filloneorder` | Fill metrics for a single order |
29
+ | `fillosdfusage` | Fill OSDF usage metrics into the database |
30
+ | `fillrdadb` | Fill RDA database usage metrics |
31
+ | `filltdsusage` | Fill TDS (THREDDS) usage metrics into the database |
32
+ | `filluser` | Fill user information into the database |
33
+ | `fillzenodousage` | Fill Zenodo usage metrics into the database |
34
+ | `pgperson` | Retrieve user info from the UCAR People API by field |
35
+ | `pgusername` | Retrieve user info from the UCAR People API by username |
36
+ | `viewallusage` | View combined usage statistics |
37
+ | `viewawsusage` | View AWS usage statistics |
38
+ | `viewcheckusage` | View usage check results |
39
+ | `viewcodusage` | View COD usage statistics |
40
+ | `viewordusage` | View order usage statistics |
41
+ | `viewosdfusage` | View OSDF usage statistics |
42
+ | `viewrqstusage` | View request usage statistics |
43
+ | `viewtdsusage` | View TDS usage statistics |
44
+ | `viewwebfile` | View web file access records |
45
+ | `viewwebusage` | View web usage statistics |
46
+
47
+ ## Environment setup
48
+
49
+ Create a Python environment first; package installs in the next section run
50
+ inside whichever environment you activate here.
51
+
52
+ ### Option A — Python venv (DECS machines)
53
+
54
+ ```bash
55
+ python3 -m venv $ENVHOME # e.g. /glade/u/home/gdexdata/gdexmsenv
56
+ source $ENVHOME/bin/activate
57
+ ```
58
+
59
+ ### Option B — Conda (DAV/Casper)
60
+
61
+ ```bash
62
+ conda create --prefix $ENVHOME python=3.12 # e.g. /glade/work/gdexdata/conda-envs/pg-gdex
63
+ conda activate $ENVHOME
64
+ ```
65
+
66
+ ## Installing rda-python-metrics
67
+
68
+ Pick whichever install mode fits your workflow. All four pull in the
69
+ transitive dependencies (`rda_python_common`, `rda_python_setuid`, `geoip2`,
70
+ `ipinfo`, `httplib2`, `dnspython`, `unidecode`, `urllib3>=2.5.0`,
71
+ `requests>=2.33.0`, `idna>=3.10`) automatically.
72
+
73
+ For local development, clone this repo alongside your project and install it
74
+ in editable mode so that changes are picked up without re-installing:
75
+
76
+ ```bash
77
+ git clone https://github.com/NCAR/rda-python-metrics.git
78
+ cd rda-python-metrics
79
+ pip install -e .
80
+ ```
81
+
82
+ To test a specific branch (e.g. an in-progress feature or fix branch), pass
83
+ `-b/--branch` to `git clone`:
84
+
85
+ ```bash
86
+ git clone -b <branch-name> https://github.com/NCAR/rda-python-metrics.git
87
+ cd rda-python-metrics
88
+ pip install -e .
89
+ ```
90
+
91
+ For a regular (non-editable) install from a checkout:
92
+
93
+ ```bash
94
+ pip install /path/to/rda-python-metrics
95
+ ```
96
+
97
+ For a production install on a system that uses the published distribution:
98
+
99
+ ```bash
100
+ pip install rda_python_metrics
101
+ ```
102
+
103
+ ## Setuid Setup
104
+
105
+ `logarch` runs as the common user `PGLOG['COMMONUSER']` (default `gdexdata`)
106
+ via the `rda_python_setuid` mechanism, which is pulled in automatically as
107
+ a dependency. After `pip install` above, choose one of the wiring options
108
+ below.
109
+
110
+ > **Note:** If `rda_python_setuid` is already installed and fully set up in
111
+ > your environment, you can skip the compile step (`-c/--compile`) and the
112
+ > optional `pgstart` step (`-p/--pgstart`). The `-l/--link` step is still
113
+ > required to wire up this package's own setuid program.
114
+
115
+ ### Full setuid install (requires sudo access to COMMONUSER)
116
+
117
+ Run these steps once per environment:
118
+
119
+ ```bash
120
+ # 1. Compile the pywrapper C binary (once per environment):
121
+ pywrapper-install -c|--compile -n|--username gdexdata
122
+
123
+ # 2. Wire up logarch as a setuid entry (or use 'all' to link every setuid_* at once):
124
+ pywrapper-install -l|--link logarch
125
+ pywrapper-install -l|--link all
126
+
127
+ # 3. Optionally, install a pgstart_<loginname> binary so <loginname> (any
128
+ # user in the same group as PGLOG['COMMONUSER']) can run commands as
129
+ # themselves. Run either by PGLOG['ADMINUSER'] (default zji, if it has
130
+ # 'sudo -u <loginname>'), or by <loginname> directly:
131
+ pywrapper-install -p|--pgstart -n|--username <loginname>
132
+ ```
133
+
134
+ `pywrapper-install` with no arguments displays the full user guide.
135
+
136
+ ### Simple install (no sudo required, runs as current user)
137
+
138
+ Users who do not need the setuid mechanism can create a direct symlink instead:
139
+
140
+ ```bash
141
+ pywrapper-install -l|--link logarch -s|--simple
142
+ pywrapper-install -l|--link all -s|--simple # or link every setuid_* at once
143
+ ```
144
+
145
+ This creates `bin/logarch -> bin/setuid_logarch` and logarch runs as the
146
+ current user with no privilege change.
147
+
148
+ ### Update an existing installation (no sudo required)
149
+
150
+ When the package is upgraded and a new `pywrapper.c` is bundled, recompile and
151
+ reinstall all setuid binaries using the existing `pgstart_*` binaries:
152
+
153
+ ```bash
154
+ pywrapper-install -u|--update
155
+ ```
156
+
157
+ ### Setup guide
158
+
159
+ The shared setuid setup guide is shown automatically if `setuid_logarch` is
160
+ invoked directly before the setuid wrapper has been configured.
@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
6
6
 
7
7
  [project]
8
8
  name = "rda_python_metrics"
9
- version = "2.0.5"
9
+ version = "3.0.2"
10
10
  authors = [
11
11
  { name="Zaihua Ji", email="zji@ucar.edu" },
12
12
  ]
@@ -26,7 +26,10 @@ dependencies = [
26
26
  "ipinfo",
27
27
  "httplib2",
28
28
  "dnspython",
29
- "unidecode"
29
+ "unidecode",
30
+ "urllib3>=2.5.0",
31
+ "requests>=2.33.0",
32
+ "idna>=3.10",
30
33
  ]
31
34
 
32
35
  [tool.pytest.ini_options]
@@ -38,7 +41,7 @@ pythonpath = [
38
41
  "Homepage" = "https://github.com/NCAR/rda-python-metrics"
39
42
 
40
43
  [project.scripts]
41
- "logarch.py" = "rda_python_metrics.logarch:main"
44
+ "setuid_logarch" = "rda_python_metrics.logarch:main"
42
45
  "fillawsusage" = "rda_python_metrics.fillawsusage:main"
43
46
  "fillcdgusage" = "rda_python_metrics.fillcdgusage:main"
44
47
  "fillcodusage" = "rda_python_metrics.fillcodusage:main"
@@ -6,7 +6,7 @@
6
6
  # Date : 09/24/2020
7
7
  # 2025-03-27 transferred to package rda_python_metrics from
8
8
  # https://github.com/NCAR/rda-database.git
9
- # Purpose : python library module to help rountinely updates of new data
9
+ # Purpose : python library module to help routinely updates of new data
10
10
  # for one or multiple datasets
11
11
  #
12
12
  # Github : https://github.com/NCAR/rda-python-metrics.git
@@ -32,7 +32,7 @@ def simple_output(params, flds, records, totals = None):
32
32
  sep = params['L'][0] if 'L' in params else ' '
33
33
  slen = len(sep)
34
34
 
35
- # get total line length, dynamically evaluating column lengthes if column 4 in %FLDS is zero
35
+ # get total line length, dynamically evaluating column lengths if column 4 in %FLDS is zero
36
36
  rcnt = linelen = 0
37
37
  if 'A' in params: rcnt = int(params['A'][0])
38
38
  for i in range(ccnt):
@@ -390,7 +390,7 @@ def join_query_tables(tblname, tablenames = '', joins = '', tbljoin = ''):
390
390
  return (tablenames, joins)
391
391
 
392
392
  #
393
- # expand reocrds via query action
393
+ # expand records via query action
394
394
  #
395
395
  def expand_query(expid, records, params, expand, vusg = None, sns = None, flds = None):
396
396
 
@@ -0,0 +1,29 @@
1
+ """rda_python_metrics: RDA metrics, usage, and accounting utility package.
2
+
3
+ This package exposes two parallel APIs:
4
+
5
+ 1. Legacy module-based API (back-compat). Import the capitalized
6
+ submodules and call their module-level functions, e.g.::
7
+
8
+ from rda_python_metrics import PgIPInfo, PgView
9
+
10
+ 2. Class-based API (preferred for new code). Import the class from the
11
+ lower-case module and either instantiate or subclass it, e.g.::
12
+
13
+ from rda_python_metrics.pg_ipinfo import PgIPInfo
14
+ from rda_python_metrics.pg_view import PgView
15
+
16
+ The legacy submodules are eagerly imported below so that
17
+ ``from rda_python_metrics import PgIPInfo`` continues to return the
18
+ module object that existing callers expect.
19
+ """
20
+
21
+ from . import PgIPInfo, PgView
22
+
23
+ __version__ = "2.0.6"
24
+
25
+ __all__ = [
26
+ "PgIPInfo",
27
+ "PgView",
28
+ "__version__",
29
+ ]
@@ -10,7 +10,7 @@
10
10
  # Purpose : python program to retrieve info from AWS logs
11
11
  # and fill table wusages in PgSQL database dssdb.
12
12
  #
13
- # Github : https://github.com/NCAR/rda-pythn-metrics.git
13
+ # Github : https://github.com/NCAR/rda-python-metrics.git
14
14
  #
15
15
  ###############################################################################
16
16
  #
@@ -6,7 +6,7 @@
6
6
  # Author : Zaihua Ji, zji@ucar.edu
7
7
  # Date : 2025-04-14
8
8
  # Purpose : python program to retrieve info from GDEX Postgres database for GDS
9
- # file accesses and backup fill table tdsusage in PostgreSQL database dssdb.
9
+ # file accesses and backup fill table tdsusage in PostgreSQL database rdadb.
10
10
  #
11
11
  # Github : https://github.com/NCAR/rda-python-metrics.git
12
12
  #
@@ -173,7 +173,7 @@ def get_dataset_ids(dsnames):
173
173
  return dsids
174
174
 
175
175
  #
176
- # get cdgids recursivley
176
+ # get cdgids recursively
177
177
  #
178
178
  def recursive_dataset_ids(pcdgid, cdgids):
179
179
 
@@ -10,7 +10,7 @@
10
10
  # Purpose : python program to retrieve info from Globus logs
11
11
  # and fill table wusages in PgSQL database dssdb.
12
12
  #
13
- # Github : https://github.com/NCAR/rda-pythn-metrics.git
13
+ # Github : https://github.com/NCAR/rda-python-metrics.git
14
14
  #
15
15
  ###############################################################################
16
16
  #
@@ -8,7 +8,7 @@
8
8
  # Purpose : python program to retrieve info from weekly OSDF logs
9
9
  # and fill table wusages in PgSQL database dssdb.
10
10
  #
11
- # Github : https://github.com/NCAR/rda-pythn-metrics.git
11
+ # Github : https://github.com/NCAR/rda-python-metrics.git
12
12
  #
13
13
  ###############################################################################
14
14
  #
@@ -10,7 +10,7 @@
10
10
  # Purpose : python program to retrieve info from data logs, and fill tables
11
11
  # in PostgreSQL database.schema rdadb.dssdb
12
12
  #
13
- # Github : https://github.com/NCAR/rda-pythn-metrics.git
13
+ # Github : https://github.com/NCAR/rda-python-metrics.git
14
14
  #
15
15
  ###############################################################################
16
16
  #
@@ -8,7 +8,7 @@
8
8
  # 2025-03-26 transferred to package rda_python_metrics from
9
9
  # https://github.com/NCAR/rda-database.git
10
10
  # Purpose : python program to retrieve info from TDS logs
11
- # and fill table tdsusage in PostgreSQL database dssdb.
11
+ # and fill table tdsusage in PostgreSQL database rdadb.
12
12
  #
13
13
  # Github : https://github.com/NCAR/rda-python-metrics.git
14
14
  #
@@ -8,7 +8,7 @@
8
8
  # 2025-12-16 converted to class FillAWSUsage
9
9
  # Purpose : python program to retrieve info from AWS logs
10
10
  # and fill table wusages in PgSQL database dssdb.
11
- # Github : https://github.com/NCAR/rda-pythn-metrics.git
11
+ # Github : https://github.com/NCAR/rda-python-metrics.git
12
12
  ###############################################################################
13
13
  import sys
14
14
  import re
@@ -19,6 +19,8 @@ from .pg_ipinfo import PgIPInfo
19
19
 
20
20
  class FillAWSUsage(PgIPInfo, PgFile):
21
21
 
22
+ """Retrieve info from AWS logs and fill table wusage in PostgreSQL database rdadb."""
23
+
22
24
  def __init__(self):
23
25
  super().__init__()
24
26
  self.USAGE = {
@@ -31,8 +33,8 @@ class FillAWSUsage(PgIPInfo, PgFile):
31
33
  self.option = self.cmdstr = None
32
34
  self.params = [] # array of input values
33
35
 
34
- # function to read parameters
35
36
  def read_parameters(self):
37
+ """Function to read parameters."""
36
38
  argv = sys.argv[1:]
37
39
  for arg in argv:
38
40
  ms = re.match(r'^-(b|d|p|N)$', arg)
@@ -55,8 +57,8 @@ class FillAWSUsage(PgIPInfo, PgFile):
55
57
  self.cmdstr = "fillawsusage {}".format(' '.join(argv))
56
58
  self.cmdlog(self.cmdstr)
57
59
 
58
- # function to start actions
59
60
  def start_actions(self):
61
+ """Function to start actions."""
60
62
  self.change_local_directory(self.USAGE['AWSDIR'])
61
63
  filenames = self.get_log_file_names()
62
64
  if filenames:
@@ -65,8 +67,8 @@ class FillAWSUsage(PgIPInfo, PgFile):
65
67
  self.pglog("No log file found for given command: " + self.cmdstr, self.LOGWRN)
66
68
  self.pglog(None, self.LOGWRN)
67
69
 
68
- # get the log file dates
69
70
  def get_log_file_names(self):
71
+ """Get the log file dates."""
70
72
  filenames = {}
71
73
  if self.option == 'd':
72
74
  for dt in self.params:
@@ -93,8 +95,8 @@ class FillAWSUsage(PgIPInfo, PgFile):
93
95
  pdate = self.adddate(pdate, 0, 0, 1)
94
96
  return filenames
95
97
 
96
- # Fill AWS usages into table dssdb.awsusage of DSS PgSQL database from aws access logs
97
98
  def fill_aws_usages(self, filenames):
99
+ """Fill AWS usages into table dssdb.awsusage of DSS PgSQL database from aws access logs."""
98
100
  year = cntall = addall = 0
99
101
  for pdate in filenames:
100
102
  fnames = filenames[pdate]
@@ -147,8 +149,8 @@ class FillAWSUsage(PgIPInfo, PgFile):
147
149
  if addall > cntadd:
148
150
  self.pglog("{} AWS usage records added for {} entries at {}".format(addall, cntall, self.current_datetime()), self.LOGWRN)
149
151
 
150
- # get date and time from record
151
152
  def get_record_date_time(self, ctime):
153
+ """Get date and time from record."""
152
154
  ms = re.search(r'^(\d+)/(\w+)/(\d+):(\d+:\d+:\d+)$', ctime)
153
155
  if ms:
154
156
  d = int(ms.group(1))
@@ -160,8 +162,8 @@ class FillAWSUsage(PgIPInfo, PgFile):
160
162
  else:
161
163
  self.pglog(ctime + ": Invalid date/time format", self.LGEREX)
162
164
 
163
- # add usage records for year
164
165
  def add_usage_records(self, records, year):
166
+ """Add usage records for year."""
165
167
  cnt = 0
166
168
  for key in records:
167
169
  record = records[key]
@@ -171,8 +173,8 @@ class FillAWSUsage(PgIPInfo, PgFile):
171
173
  cnt += self.pgadd(self.USAGE['PGTBL'], record, self.LOGWRN)
172
174
  return cnt
173
175
 
174
- # add record to allusage tables
175
176
  def add_to_allusage(self, year, pgrec):
177
+ """Add record to allusage tables."""
176
178
  record = {'source' : 'A'}
177
179
  flds = ['ip', 'dsid', 'date', 'time', 'quarter', 'size', 'method',
178
180
  'org_type', 'country', 'region', 'email']
@@ -180,7 +182,7 @@ class FillAWSUsage(PgIPInfo, PgFile):
180
182
  record[fld] = pgrec[fld]
181
183
  return self.add_yearly_allusage(year, record)
182
184
 
183
- # main function to excecute this script
185
+ # main function to execute this script
184
186
  def main():
185
187
  object = FillAWSUsage()
186
188
  object.read_parameters()
@@ -0,0 +1,27 @@
1
+
2
+ fillawsusage - Fill table 'awsusage' from AWS Server log files.
3
+
4
+ Reads AWS access logs from /gpfs/fs1/collections/rda/transer/AWSera5log/
5
+ and inserts usage records into the 'awsusage' table in PostgreSQL
6
+ database 'rdadb'.
7
+
8
+ USAGE:
9
+ fillawsusage [-b] (-d LogFileDates | -N NumberDays | -p BeginDate [EndDate])
10
+
11
+ OPTIONS (choose exactly one of -d, -N, or -p):
12
+
13
+ -d LogFileDates
14
+ Process log files for the given dates.
15
+ Date format: YYYY-MM-DD (e.g. 2024-01-15).
16
+
17
+ -N NumberDays
18
+ Process log files covering the most recent NumberDays days.
19
+
20
+ -p BeginDate [EndDate]
21
+ Process log files between BeginDate and EndDate, inclusive.
22
+ EndDate defaults to today if omitted.
23
+
24
+ -b Run in background mode (log to file only, no terminal output).
25
+
26
+ EXAMPLE:
27
+ fillawsusage -p 2024-01-01 2024-01-31