waldur-site-agent-cscs-dwdi 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of waldur-site-agent-cscs-dwdi might be problematic. Click here for more details.
- waldur_site_agent_cscs_dwdi/__init__.py +1 -0
- waldur_site_agent_cscs_dwdi/backend.py +362 -0
- waldur_site_agent_cscs_dwdi/client.py +239 -0
- waldur_site_agent_cscs_dwdi-0.1.0.dist-info/METADATA +240 -0
- waldur_site_agent_cscs_dwdi-0.1.0.dist-info/RECORD +7 -0
- waldur_site_agent_cscs_dwdi-0.1.0.dist-info/WHEEL +4 -0
- waldur_site_agent_cscs_dwdi-0.1.0.dist-info/entry_points.txt +2 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""CSCS-DWDI reporting plugin for Waldur Site Agent."""
|
|
@@ -0,0 +1,362 @@
|
|
|
1
|
+
"""CSCS-DWDI backend implementation for usage reporting."""
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
from datetime import datetime, timezone
|
|
5
|
+
from typing import Any, Optional
|
|
6
|
+
|
|
7
|
+
from waldur_api_client.models.resource import Resource as WaldurResource
|
|
8
|
+
|
|
9
|
+
from waldur_site_agent.backend.backends import BaseBackend
|
|
10
|
+
|
|
11
|
+
from .client import CSCSDWDIClient
|
|
12
|
+
|
|
13
|
+
logger = logging.getLogger(__name__)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class CSCSDWDIBackend(BaseBackend):
|
|
17
|
+
"""Backend for reporting usage from CSCS-DWDI API."""
|
|
18
|
+
|
|
19
|
+
def __init__(
|
|
20
|
+
self, backend_settings: dict[str, Any], backend_components: dict[str, dict]
|
|
21
|
+
) -> None:
|
|
22
|
+
"""Initialize CSCS-DWDI backend.
|
|
23
|
+
|
|
24
|
+
Args:
|
|
25
|
+
backend_settings: Backend-specific settings from the offering
|
|
26
|
+
backend_components: Component configuration from the offering
|
|
27
|
+
"""
|
|
28
|
+
super().__init__(backend_settings, backend_components)
|
|
29
|
+
self.backend_type = "cscs-dwdi"
|
|
30
|
+
|
|
31
|
+
# Extract CSCS-DWDI specific configuration
|
|
32
|
+
self.api_url = backend_settings.get("cscs_dwdi_api_url", "")
|
|
33
|
+
self.client_id = backend_settings.get("cscs_dwdi_client_id", "")
|
|
34
|
+
self.client_secret = backend_settings.get("cscs_dwdi_client_secret", "")
|
|
35
|
+
|
|
36
|
+
# Required OIDC configuration
|
|
37
|
+
self.oidc_token_url = backend_settings.get("cscs_dwdi_oidc_token_url", "")
|
|
38
|
+
self.oidc_scope = backend_settings.get("cscs_dwdi_oidc_scope")
|
|
39
|
+
|
|
40
|
+
if not all([self.api_url, self.client_id, self.client_secret, self.oidc_token_url]):
|
|
41
|
+
msg = (
|
|
42
|
+
"CSCS-DWDI backend requires cscs_dwdi_api_url, cscs_dwdi_client_id, "
|
|
43
|
+
"cscs_dwdi_client_secret, and cscs_dwdi_oidc_token_url in backend_settings"
|
|
44
|
+
)
|
|
45
|
+
raise ValueError(msg)
|
|
46
|
+
|
|
47
|
+
self.cscs_client = CSCSDWDIClient(
|
|
48
|
+
api_url=self.api_url,
|
|
49
|
+
client_id=self.client_id,
|
|
50
|
+
client_secret=self.client_secret,
|
|
51
|
+
oidc_token_url=self.oidc_token_url,
|
|
52
|
+
oidc_scope=self.oidc_scope,
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
def ping(self, raise_exception: bool = False) -> bool: # noqa: ARG002
|
|
56
|
+
"""Check if CSCS-DWDI API is accessible.
|
|
57
|
+
|
|
58
|
+
Args:
|
|
59
|
+
raise_exception: Whether to raise an exception on failure
|
|
60
|
+
|
|
61
|
+
Returns:
|
|
62
|
+
True if API is accessible, False otherwise
|
|
63
|
+
"""
|
|
64
|
+
return self.cscs_client.ping()
|
|
65
|
+
|
|
66
|
+
def _get_usage_report(
|
|
67
|
+
self, resource_backend_ids: list[str]
|
|
68
|
+
) -> dict[str, dict[str, dict[str, float]]]:
|
|
69
|
+
"""Get usage report for specified resources.
|
|
70
|
+
|
|
71
|
+
This method queries the CSCS-DWDI API for the current month's usage
|
|
72
|
+
and formats it according to Waldur's expected structure.
|
|
73
|
+
|
|
74
|
+
Args:
|
|
75
|
+
resource_backend_ids: List of account identifiers to report on
|
|
76
|
+
|
|
77
|
+
Returns:
|
|
78
|
+
Dictionary mapping account names to usage data:
|
|
79
|
+
{
|
|
80
|
+
"account1": {
|
|
81
|
+
"TOTAL_ACCOUNT_USAGE": {
|
|
82
|
+
"nodeHours": 1234.56
|
|
83
|
+
},
|
|
84
|
+
"user1": {
|
|
85
|
+
"nodeHours": 500.00
|
|
86
|
+
},
|
|
87
|
+
"user2": {
|
|
88
|
+
"nodeHours": 734.56
|
|
89
|
+
}
|
|
90
|
+
},
|
|
91
|
+
...
|
|
92
|
+
}
|
|
93
|
+
"""
|
|
94
|
+
if not resource_backend_ids:
|
|
95
|
+
logger.warning("No resource backend IDs provided for usage report")
|
|
96
|
+
return {}
|
|
97
|
+
|
|
98
|
+
# Get current month's date range
|
|
99
|
+
today = datetime.now(tz=timezone.utc).date()
|
|
100
|
+
from_date = today.replace(day=1)
|
|
101
|
+
to_date = today
|
|
102
|
+
|
|
103
|
+
logger.info(
|
|
104
|
+
"Fetching usage report for %d accounts from %s to %s",
|
|
105
|
+
len(resource_backend_ids),
|
|
106
|
+
from_date,
|
|
107
|
+
to_date,
|
|
108
|
+
)
|
|
109
|
+
|
|
110
|
+
try:
|
|
111
|
+
# Query CSCS-DWDI API for usage data
|
|
112
|
+
response = self.cscs_client.get_usage_for_month(
|
|
113
|
+
accounts=resource_backend_ids,
|
|
114
|
+
from_date=from_date,
|
|
115
|
+
to_date=to_date,
|
|
116
|
+
)
|
|
117
|
+
|
|
118
|
+
# Process the response
|
|
119
|
+
usage_report = self._process_api_response(response)
|
|
120
|
+
|
|
121
|
+
# Filter to only include requested accounts
|
|
122
|
+
filtered_report = {
|
|
123
|
+
account: data
|
|
124
|
+
for account, data in usage_report.items()
|
|
125
|
+
if account in resource_backend_ids
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
logger.info(
|
|
129
|
+
"Successfully retrieved usage for %d accounts",
|
|
130
|
+
len(filtered_report),
|
|
131
|
+
)
|
|
132
|
+
|
|
133
|
+
return filtered_report
|
|
134
|
+
|
|
135
|
+
except Exception:
|
|
136
|
+
logger.exception("Failed to get usage report from CSCS-DWDI")
|
|
137
|
+
raise
|
|
138
|
+
|
|
139
|
+
def _process_api_response(
|
|
140
|
+
self, response: dict[str, Any]
|
|
141
|
+
) -> dict[str, dict[str, dict[str, float]]]:
|
|
142
|
+
"""Process CSCS-DWDI API response into Waldur format.
|
|
143
|
+
|
|
144
|
+
Args:
|
|
145
|
+
response: Raw API response from CSCS-DWDI
|
|
146
|
+
|
|
147
|
+
Returns:
|
|
148
|
+
Formatted usage report for Waldur with configured component mappings
|
|
149
|
+
"""
|
|
150
|
+
usage_report = {}
|
|
151
|
+
|
|
152
|
+
# The response has a "compute" field with list of account data
|
|
153
|
+
compute_data = response.get("compute", [])
|
|
154
|
+
|
|
155
|
+
for account_data in compute_data:
|
|
156
|
+
account_name = account_data.get("account")
|
|
157
|
+
if not account_name:
|
|
158
|
+
logger.warning("Account data missing account name, skipping")
|
|
159
|
+
continue
|
|
160
|
+
|
|
161
|
+
# Extract total account usage for all configured components
|
|
162
|
+
total_usage = self._extract_component_usage_from_account_data(account_data)
|
|
163
|
+
|
|
164
|
+
# Initialize account entry
|
|
165
|
+
usage_report[account_name] = {"TOTAL_ACCOUNT_USAGE": total_usage}
|
|
166
|
+
|
|
167
|
+
# Process per-user usage
|
|
168
|
+
users = account_data.get("users", [])
|
|
169
|
+
user_usage: dict[str, dict[str, float]] = {}
|
|
170
|
+
|
|
171
|
+
for user_data in users:
|
|
172
|
+
username = user_data.get("username")
|
|
173
|
+
if not username:
|
|
174
|
+
continue
|
|
175
|
+
|
|
176
|
+
# Extract user component usage
|
|
177
|
+
user_component_usage = self._extract_component_usage_from_user_data(user_data)
|
|
178
|
+
|
|
179
|
+
if username in user_usage:
|
|
180
|
+
# Aggregate usage for same user across different dates/clusters
|
|
181
|
+
for component_name, value in user_component_usage.items():
|
|
182
|
+
user_usage[username][component_name] = (
|
|
183
|
+
user_usage[username].get(component_name, 0.0) + value
|
|
184
|
+
)
|
|
185
|
+
else:
|
|
186
|
+
user_usage[username] = user_component_usage
|
|
187
|
+
|
|
188
|
+
# Add rounded user usage to report
|
|
189
|
+
for username, component_usage in user_usage.items():
|
|
190
|
+
rounded_usage = {comp: round(value, 2) for comp, value in component_usage.items()}
|
|
191
|
+
usage_report[account_name][username] = rounded_usage
|
|
192
|
+
|
|
193
|
+
return usage_report
|
|
194
|
+
|
|
195
|
+
def _extract_component_usage_from_account_data(
|
|
196
|
+
self, account_data: dict[str, Any]
|
|
197
|
+
) -> dict[str, float]:
|
|
198
|
+
"""Extract component usage from account-level data.
|
|
199
|
+
|
|
200
|
+
Args:
|
|
201
|
+
account_data: Account data from CSCS-DWDI API response
|
|
202
|
+
|
|
203
|
+
Returns:
|
|
204
|
+
Dictionary mapping component names to usage values
|
|
205
|
+
"""
|
|
206
|
+
usage = {}
|
|
207
|
+
|
|
208
|
+
for component_name, component_config in self.backend_components.items():
|
|
209
|
+
# Look for component usage in account data
|
|
210
|
+
# Try multiple naming patterns to match API response fields to component names
|
|
211
|
+
raw_value = 0.0
|
|
212
|
+
|
|
213
|
+
# Try exact match first (e.g., nodeHours -> nodeHours)
|
|
214
|
+
if component_name in account_data:
|
|
215
|
+
raw_value = account_data[component_name]
|
|
216
|
+
# Try account prefix preserving case (e.g., nodeHours -> accountNodeHours)
|
|
217
|
+
elif f"account{component_name[0].upper()}{component_name[1:]}" in account_data:
|
|
218
|
+
raw_value = account_data[f"account{component_name[0].upper()}{component_name[1:]}"]
|
|
219
|
+
# Try total prefix (e.g., nodeHours -> totalNodeHours)
|
|
220
|
+
elif f"total{component_name[0].upper()}{component_name[1:]}" in account_data:
|
|
221
|
+
raw_value = account_data[f"total{component_name[0].upper()}{component_name[1:]}"]
|
|
222
|
+
# Try lowercase with account prefix
|
|
223
|
+
elif f"account{component_name}" in account_data:
|
|
224
|
+
raw_value = account_data[f"account{component_name}"]
|
|
225
|
+
|
|
226
|
+
# Apply unit factor conversion
|
|
227
|
+
unit_factor = component_config.get("unit_factor", 1)
|
|
228
|
+
converted_value = raw_value * unit_factor
|
|
229
|
+
usage[component_name] = round(converted_value, 2)
|
|
230
|
+
|
|
231
|
+
return usage
|
|
232
|
+
|
|
233
|
+
def _extract_component_usage_from_user_data(
|
|
234
|
+
self, user_data: dict[str, Any]
|
|
235
|
+
) -> dict[str, float]:
|
|
236
|
+
"""Extract component usage from user-level data.
|
|
237
|
+
|
|
238
|
+
Args:
|
|
239
|
+
user_data: User data from CSCS-DWDI API response
|
|
240
|
+
|
|
241
|
+
Returns:
|
|
242
|
+
Dictionary mapping component names to usage values
|
|
243
|
+
"""
|
|
244
|
+
usage = {}
|
|
245
|
+
|
|
246
|
+
for component_name, component_config in self.backend_components.items():
|
|
247
|
+
# Look for component usage in user data
|
|
248
|
+
# The API should return fields that match component names
|
|
249
|
+
raw_value = user_data.get(component_name, 0.0)
|
|
250
|
+
|
|
251
|
+
# Apply unit factor conversion
|
|
252
|
+
unit_factor = component_config.get("unit_factor", 1)
|
|
253
|
+
converted_value = raw_value * unit_factor
|
|
254
|
+
usage[component_name] = converted_value
|
|
255
|
+
|
|
256
|
+
return usage
|
|
257
|
+
|
|
258
|
+
# Methods not implemented for reporting-only backend
|
|
259
|
+
def get_account(self, account_name: str) -> Optional[dict[str, Any]]:
|
|
260
|
+
"""Not implemented for reporting-only backend."""
|
|
261
|
+
msg = "CSCS-DWDI backend is reporting-only and does not support account management"
|
|
262
|
+
raise NotImplementedError(msg)
|
|
263
|
+
|
|
264
|
+
def create_account(self, account_data: dict) -> bool:
|
|
265
|
+
"""Not implemented for reporting-only backend."""
|
|
266
|
+
msg = "CSCS-DWDI backend is reporting-only and does not support account creation"
|
|
267
|
+
raise NotImplementedError(msg)
|
|
268
|
+
|
|
269
|
+
def delete_account(self, account_name: str) -> bool:
|
|
270
|
+
"""Not implemented for reporting-only backend."""
|
|
271
|
+
msg = "CSCS-DWDI backend is reporting-only and does not support account deletion"
|
|
272
|
+
raise NotImplementedError(msg)
|
|
273
|
+
|
|
274
|
+
def update_account_limit_deposit(
|
|
275
|
+
self,
|
|
276
|
+
account_name: str,
|
|
277
|
+
component_type: str,
|
|
278
|
+
component_amount: float,
|
|
279
|
+
offering_component_data: dict,
|
|
280
|
+
) -> bool:
|
|
281
|
+
"""Not implemented for reporting-only backend."""
|
|
282
|
+
msg = "CSCS-DWDI backend is reporting-only and does not support limit updates"
|
|
283
|
+
raise NotImplementedError(msg)
|
|
284
|
+
|
|
285
|
+
def reset_account_limit_deposit(
|
|
286
|
+
self,
|
|
287
|
+
account_name: str,
|
|
288
|
+
component_type: str,
|
|
289
|
+
offering_component_data: dict,
|
|
290
|
+
) -> bool:
|
|
291
|
+
"""Not implemented for reporting-only backend."""
|
|
292
|
+
msg = "CSCS-DWDI backend is reporting-only and does not support limit resets"
|
|
293
|
+
raise NotImplementedError(msg)
|
|
294
|
+
|
|
295
|
+
def add_account_users(self, account_name: str, user_backend_ids: list[str]) -> bool:
|
|
296
|
+
"""Not implemented for reporting-only backend."""
|
|
297
|
+
msg = "CSCS-DWDI backend is reporting-only and does not support user management"
|
|
298
|
+
raise NotImplementedError(msg)
|
|
299
|
+
|
|
300
|
+
def delete_account_users(self, account_name: str, user_backend_ids: list[str]) -> bool:
|
|
301
|
+
"""Not implemented for reporting-only backend."""
|
|
302
|
+
msg = "CSCS-DWDI backend is reporting-only and does not support user management"
|
|
303
|
+
raise NotImplementedError(msg)
|
|
304
|
+
|
|
305
|
+
def list_accounts(self) -> list[dict[str, Any]]:
|
|
306
|
+
"""Not implemented for reporting-only backend."""
|
|
307
|
+
msg = "CSCS-DWDI backend is reporting-only and does not support account listing"
|
|
308
|
+
raise NotImplementedError(msg)
|
|
309
|
+
|
|
310
|
+
def set_resource_limits(self, resource_backend_id: str, limits: dict[str, int]) -> None:
|
|
311
|
+
"""Not implemented for reporting-only backend."""
|
|
312
|
+
msg = "CSCS-DWDI backend is reporting-only and does not support resource limits"
|
|
313
|
+
raise NotImplementedError(msg)
|
|
314
|
+
|
|
315
|
+
def diagnostics(self) -> bool:
|
|
316
|
+
"""Get diagnostic information for the backend."""
|
|
317
|
+
logger.info(
|
|
318
|
+
"CSCS-DWDI Backend Diagnostics - Type: %s, API: %s, Components: %s, Ping: %s",
|
|
319
|
+
self.backend_type,
|
|
320
|
+
self.api_url,
|
|
321
|
+
list(self.backend_components.keys()),
|
|
322
|
+
self.ping(),
|
|
323
|
+
)
|
|
324
|
+
return self.ping()
|
|
325
|
+
|
|
326
|
+
def get_resource_metadata(self, resource_backend_id: str) -> dict[str, Any]:
|
|
327
|
+
"""Not implemented for reporting-only backend."""
|
|
328
|
+
msg = "CSCS-DWDI backend is reporting-only and does not support resource metadata"
|
|
329
|
+
raise NotImplementedError(msg)
|
|
330
|
+
|
|
331
|
+
def list_components(self) -> list[str]:
|
|
332
|
+
"""List configured components for this backend."""
|
|
333
|
+
return list(self.backend_components.keys())
|
|
334
|
+
|
|
335
|
+
def _collect_resource_limits(
|
|
336
|
+
self, waldur_resource: WaldurResource
|
|
337
|
+
) -> tuple[dict[str, int], dict[str, int]]:
|
|
338
|
+
"""Not implemented for reporting-only backend."""
|
|
339
|
+
msg = "CSCS-DWDI backend is reporting-only and does not support resource limits"
|
|
340
|
+
raise NotImplementedError(msg)
|
|
341
|
+
|
|
342
|
+
def _pre_create_resource(
|
|
343
|
+
self, waldur_resource: WaldurResource, user_context: Optional[dict] = None
|
|
344
|
+
) -> None:
|
|
345
|
+
"""Not implemented for reporting-only backend."""
|
|
346
|
+
msg = "CSCS-DWDI backend is reporting-only and does not support resource creation"
|
|
347
|
+
raise NotImplementedError(msg)
|
|
348
|
+
|
|
349
|
+
def pause_resource(self, resource_backend_id: str) -> bool:
|
|
350
|
+
"""Not implemented for reporting-only backend."""
|
|
351
|
+
msg = "CSCS-DWDI backend is reporting-only and does not support resource pausing"
|
|
352
|
+
raise NotImplementedError(msg)
|
|
353
|
+
|
|
354
|
+
def restore_resource(self, resource_backend_id: str) -> bool:
|
|
355
|
+
"""Not implemented for reporting-only backend."""
|
|
356
|
+
msg = "CSCS-DWDI backend is reporting-only and does not support resource restoration"
|
|
357
|
+
raise NotImplementedError(msg)
|
|
358
|
+
|
|
359
|
+
def downscale_resource(self, resource_backend_id: str) -> bool:
|
|
360
|
+
"""Not implemented for reporting-only backend."""
|
|
361
|
+
msg = "CSCS-DWDI backend is reporting-only and does not support resource downscaling"
|
|
362
|
+
raise NotImplementedError(msg)
|
|
@@ -0,0 +1,239 @@
|
|
|
1
|
+
"""CSCS-DWDI API client implementation."""
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
from datetime import date, datetime, timedelta, timezone
|
|
5
|
+
from typing import Any, Optional
|
|
6
|
+
|
|
7
|
+
import httpx
|
|
8
|
+
|
|
9
|
+
logger = logging.getLogger(__name__)
|
|
10
|
+
|
|
11
|
+
HTTP_OK = 200
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class CSCSDWDIClient:
|
|
15
|
+
"""Client for interacting with CSCS-DWDI API."""
|
|
16
|
+
|
|
17
|
+
def __init__(
|
|
18
|
+
self,
|
|
19
|
+
api_url: str,
|
|
20
|
+
client_id: str,
|
|
21
|
+
client_secret: str,
|
|
22
|
+
oidc_token_url: Optional[str] = None,
|
|
23
|
+
oidc_scope: Optional[str] = None,
|
|
24
|
+
) -> None:
|
|
25
|
+
"""Initialize CSCS-DWDI client.
|
|
26
|
+
|
|
27
|
+
Args:
|
|
28
|
+
api_url: Base URL for the CSCS-DWDI API
|
|
29
|
+
client_id: OIDC client ID for authentication
|
|
30
|
+
client_secret: OIDC client secret for authentication
|
|
31
|
+
oidc_token_url: OIDC token endpoint URL (required for authentication)
|
|
32
|
+
oidc_scope: OIDC scope to request (optional)
|
|
33
|
+
"""
|
|
34
|
+
self.api_url = api_url.rstrip("/")
|
|
35
|
+
self.client_id = client_id
|
|
36
|
+
self.client_secret = client_secret
|
|
37
|
+
self.oidc_token_url = oidc_token_url
|
|
38
|
+
self.oidc_scope = oidc_scope or "openid"
|
|
39
|
+
self._token: Optional[str] = None
|
|
40
|
+
self._token_expires_at: Optional[datetime] = None
|
|
41
|
+
|
|
42
|
+
def _get_auth_token(self) -> str:
|
|
43
|
+
"""Get or refresh OIDC authentication token.
|
|
44
|
+
|
|
45
|
+
Returns:
|
|
46
|
+
Valid authentication token
|
|
47
|
+
|
|
48
|
+
Raises:
|
|
49
|
+
httpx.HTTPError: If token acquisition fails
|
|
50
|
+
"""
|
|
51
|
+
# Check if we have a valid cached token
|
|
52
|
+
if (
|
|
53
|
+
self._token
|
|
54
|
+
and self._token_expires_at
|
|
55
|
+
and datetime.now(tz=timezone.utc) < self._token_expires_at
|
|
56
|
+
):
|
|
57
|
+
return self._token
|
|
58
|
+
|
|
59
|
+
# Fail if OIDC endpoint not configured
|
|
60
|
+
if not self.oidc_token_url:
|
|
61
|
+
error_msg = (
|
|
62
|
+
"OIDC authentication failed: cscs_dwdi_oidc_token_url not configured. "
|
|
63
|
+
"Set 'cscs_dwdi_oidc_token_url' in backend_settings for production use."
|
|
64
|
+
)
|
|
65
|
+
logger.error(error_msg)
|
|
66
|
+
raise ValueError(error_msg)
|
|
67
|
+
|
|
68
|
+
# Request new token from OIDC provider
|
|
69
|
+
return self._acquire_oidc_token()
|
|
70
|
+
|
|
71
|
+
def _acquire_oidc_token(self) -> str:
|
|
72
|
+
"""Acquire a new OIDC token from the configured provider.
|
|
73
|
+
|
|
74
|
+
Returns:
|
|
75
|
+
Valid authentication token
|
|
76
|
+
|
|
77
|
+
Raises:
|
|
78
|
+
httpx.HTTPError: If token acquisition fails
|
|
79
|
+
"""
|
|
80
|
+
logger.debug("Acquiring new OIDC token from %s", self.oidc_token_url)
|
|
81
|
+
|
|
82
|
+
token_data = {
|
|
83
|
+
"grant_type": "client_credentials",
|
|
84
|
+
"client_id": self.client_id,
|
|
85
|
+
"client_secret": self.client_secret,
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
# Add scope if specified
|
|
89
|
+
if self.oidc_scope:
|
|
90
|
+
token_data["scope"] = self.oidc_scope
|
|
91
|
+
|
|
92
|
+
headers = {"Content-Type": "application/x-www-form-urlencoded"}
|
|
93
|
+
|
|
94
|
+
with httpx.Client() as client:
|
|
95
|
+
response = client.post(
|
|
96
|
+
self.oidc_token_url, data=token_data, headers=headers, timeout=30.0
|
|
97
|
+
)
|
|
98
|
+
response.raise_for_status()
|
|
99
|
+
token_response = response.json()
|
|
100
|
+
|
|
101
|
+
# Extract token and expiry information
|
|
102
|
+
access_token = token_response.get("access_token")
|
|
103
|
+
if not access_token:
|
|
104
|
+
msg = f"No access_token in OIDC response: {token_response}"
|
|
105
|
+
raise ValueError(msg)
|
|
106
|
+
|
|
107
|
+
# Calculate token expiry time
|
|
108
|
+
expires_in = token_response.get("expires_in", 3600) # Default to 1 hour
|
|
109
|
+
# Subtract 5 minutes from expiry for safety margin
|
|
110
|
+
safe_expires_in = max(300, expires_in - 300)
|
|
111
|
+
|
|
112
|
+
self._token = access_token
|
|
113
|
+
self._token_expires_at = datetime.now(tz=timezone.utc) + timedelta(
|
|
114
|
+
seconds=safe_expires_in
|
|
115
|
+
)
|
|
116
|
+
|
|
117
|
+
logger.info("Successfully acquired OIDC token, expires in %d seconds", expires_in)
|
|
118
|
+
|
|
119
|
+
return self._token
|
|
120
|
+
|
|
121
|
+
def get_usage_for_month(
|
|
122
|
+
self, accounts: list[str], from_date: date, to_date: date
|
|
123
|
+
) -> dict[str, Any]:
|
|
124
|
+
"""Get usage data for multiple accounts for a month range.
|
|
125
|
+
|
|
126
|
+
Args:
|
|
127
|
+
accounts: List of account identifiers to query
|
|
128
|
+
from_date: Start date (beginning of month)
|
|
129
|
+
to_date: End date (end of month)
|
|
130
|
+
|
|
131
|
+
Returns:
|
|
132
|
+
API response with usage data grouped by account
|
|
133
|
+
|
|
134
|
+
Raises:
|
|
135
|
+
httpx.HTTPError: If API request fails
|
|
136
|
+
"""
|
|
137
|
+
token = self._get_auth_token()
|
|
138
|
+
|
|
139
|
+
# Format dates as YYYY-MM for month endpoints
|
|
140
|
+
from_month = from_date.strftime("%Y-%m")
|
|
141
|
+
to_month = to_date.strftime("%Y-%m")
|
|
142
|
+
|
|
143
|
+
params: dict[str, Any] = {
|
|
144
|
+
"from": from_month,
|
|
145
|
+
"to": to_month,
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
# Add account filters if provided
|
|
149
|
+
if accounts:
|
|
150
|
+
params["account"] = accounts
|
|
151
|
+
|
|
152
|
+
headers = {"Authorization": f"Bearer {token}"}
|
|
153
|
+
|
|
154
|
+
url = f"{self.api_url}/api/v1/compute/usage-month-multiaccount"
|
|
155
|
+
|
|
156
|
+
logger.debug(
|
|
157
|
+
"Fetching usage for accounts %s from %s to %s",
|
|
158
|
+
accounts,
|
|
159
|
+
from_month,
|
|
160
|
+
to_month,
|
|
161
|
+
)
|
|
162
|
+
|
|
163
|
+
with httpx.Client() as client:
|
|
164
|
+
response = client.get(url, params=params, headers=headers, timeout=30.0)
|
|
165
|
+
response.raise_for_status()
|
|
166
|
+
return response.json()
|
|
167
|
+
|
|
168
|
+
def get_usage_for_days(
|
|
169
|
+
self, accounts: list[str], from_date: date, to_date: date
|
|
170
|
+
) -> dict[str, Any]:
|
|
171
|
+
"""Get usage data for multiple accounts for a day range.
|
|
172
|
+
|
|
173
|
+
Args:
|
|
174
|
+
accounts: List of account identifiers to query
|
|
175
|
+
from_date: Start date
|
|
176
|
+
to_date: End date
|
|
177
|
+
|
|
178
|
+
Returns:
|
|
179
|
+
API response with usage data grouped by account
|
|
180
|
+
|
|
181
|
+
Raises:
|
|
182
|
+
httpx.HTTPError: If API request fails
|
|
183
|
+
"""
|
|
184
|
+
token = self._get_auth_token()
|
|
185
|
+
|
|
186
|
+
# Format dates as YYYY-MM-DD for day endpoints
|
|
187
|
+
from_day = from_date.strftime("%Y-%m-%d")
|
|
188
|
+
to_day = to_date.strftime("%Y-%m-%d")
|
|
189
|
+
|
|
190
|
+
params: dict[str, Any] = {
|
|
191
|
+
"from": from_day,
|
|
192
|
+
"to": to_day,
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
# Add account filters if provided
|
|
196
|
+
if accounts:
|
|
197
|
+
params["account"] = accounts
|
|
198
|
+
|
|
199
|
+
headers = {"Authorization": f"Bearer {token}"}
|
|
200
|
+
|
|
201
|
+
url = f"{self.api_url}/api/v1/compute/usage-day-multiaccount"
|
|
202
|
+
|
|
203
|
+
logger.debug(
|
|
204
|
+
"Fetching daily usage for accounts %s from %s to %s",
|
|
205
|
+
accounts,
|
|
206
|
+
from_day,
|
|
207
|
+
to_day,
|
|
208
|
+
)
|
|
209
|
+
|
|
210
|
+
with httpx.Client() as client:
|
|
211
|
+
response = client.get(url, params=params, headers=headers, timeout=30.0)
|
|
212
|
+
response.raise_for_status()
|
|
213
|
+
return response.json()
|
|
214
|
+
|
|
215
|
+
def ping(self) -> bool:
|
|
216
|
+
"""Check if CSCS-DWDI API is accessible.
|
|
217
|
+
|
|
218
|
+
Returns:
|
|
219
|
+
True if API is accessible, False otherwise
|
|
220
|
+
"""
|
|
221
|
+
try:
|
|
222
|
+
token = self._get_auth_token()
|
|
223
|
+
headers = {"Authorization": f"Bearer {token}"}
|
|
224
|
+
|
|
225
|
+
# Use a simple query to test connectivity
|
|
226
|
+
today = datetime.now(tz=timezone.utc).date()
|
|
227
|
+
params = {
|
|
228
|
+
"from": today.strftime("%Y-%m-%d"),
|
|
229
|
+
"to": today.strftime("%Y-%m-%d"),
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
url = f"{self.api_url}/api/v1/compute/usage-day-multiaccount"
|
|
233
|
+
|
|
234
|
+
with httpx.Client() as client:
|
|
235
|
+
response = client.get(url, params=params, headers=headers, timeout=10.0)
|
|
236
|
+
return response.status_code == HTTP_OK
|
|
237
|
+
except Exception:
|
|
238
|
+
logger.exception("Ping failed")
|
|
239
|
+
return False
|
|
@@ -0,0 +1,240 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: waldur-site-agent-cscs-dwdi
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: CSCS-DWDI reporting plugin for Waldur Site Agent
|
|
5
|
+
Author-email: OpenNode Team <info@opennodecloud.com>
|
|
6
|
+
Requires-Python: <4,>=3.9
|
|
7
|
+
Requires-Dist: httpx>=0.25.0
|
|
8
|
+
Requires-Dist: waldur-site-agent==0.1.0
|
|
9
|
+
Description-Content-Type: text/markdown
|
|
10
|
+
|
|
11
|
+
# CSCS-DWDI Plugin for Waldur Site Agent
|
|
12
|
+
|
|
13
|
+
This plugin provides reporting functionality for Waldur Site Agent by integrating with the CSCS-DWDI
|
|
14
|
+
(Data Warehouse and Data Intelligence) API.
|
|
15
|
+
|
|
16
|
+
## Overview
|
|
17
|
+
|
|
18
|
+
The CSCS-DWDI plugin is a **reporting-only backend** that fetches compute usage data from the CSCS-DWDI
|
|
19
|
+
service and reports it to Waldur. It supports node-hour usage tracking for multiple accounts and users.
|
|
20
|
+
|
|
21
|
+
## Features
|
|
22
|
+
|
|
23
|
+
- **Monthly Usage Reporting**: Fetches usage data for the current month
|
|
24
|
+
- **Multi-Account Support**: Reports usage for multiple accounts in a single API call
|
|
25
|
+
- **Per-User Usage**: Breaks down usage by individual users within each account
|
|
26
|
+
- **OIDC Authentication**: Uses OAuth2/OIDC for secure API access
|
|
27
|
+
- **Automatic Aggregation**: Combines usage across different clusters and time periods
|
|
28
|
+
|
|
29
|
+
## Configuration
|
|
30
|
+
|
|
31
|
+
Add the following configuration to your Waldur Site Agent offering:
|
|
32
|
+
|
|
33
|
+
```yaml
|
|
34
|
+
offerings:
|
|
35
|
+
- name: "CSCS HPC Offering"
|
|
36
|
+
reporting_backend: "cscs-dwdi"
|
|
37
|
+
backend_settings:
|
|
38
|
+
cscs_dwdi_api_url: "https://dwdi-api.cscs.ch"
|
|
39
|
+
cscs_dwdi_client_id: "your-oidc-client-id"
|
|
40
|
+
cscs_dwdi_client_secret: "your-oidc-client-secret"
|
|
41
|
+
# Optional OIDC configuration (for production use)
|
|
42
|
+
cscs_dwdi_oidc_token_url: "https://identity.cscs.ch/realms/cscs/protocol/openid-connect/token"
|
|
43
|
+
cscs_dwdi_oidc_scope: "cscs-dwdi:read"
|
|
44
|
+
|
|
45
|
+
backend_components:
|
|
46
|
+
nodeHours:
|
|
47
|
+
measured_unit: "node-hours"
|
|
48
|
+
unit_factor: 1
|
|
49
|
+
accounting_type: "usage"
|
|
50
|
+
label: "Node Hours"
|
|
51
|
+
storage:
|
|
52
|
+
measured_unit: "TB"
|
|
53
|
+
unit_factor: 1
|
|
54
|
+
accounting_type: "usage"
|
|
55
|
+
label: "Storage Usage"
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
### Configuration Parameters
|
|
59
|
+
|
|
60
|
+
#### Backend Settings
|
|
61
|
+
|
|
62
|
+
| Parameter | Required | Description |
|
|
63
|
+
|-----------|----------|-------------|
|
|
64
|
+
| `cscs_dwdi_api_url` | Yes | Base URL for the CSCS-DWDI API service |
|
|
65
|
+
| `cscs_dwdi_client_id` | Yes | OIDC client ID for authentication |
|
|
66
|
+
| `cscs_dwdi_client_secret` | Yes | OIDC client secret for authentication |
|
|
67
|
+
| `cscs_dwdi_oidc_token_url` | Yes | OIDC token endpoint URL (required for authentication) |
|
|
68
|
+
| `cscs_dwdi_oidc_scope` | No | OIDC scope to request (defaults to "openid") |
|
|
69
|
+
|
|
70
|
+
#### Backend Components
|
|
71
|
+
|
|
72
|
+
Components must match the field names returned by the CSCS-DWDI API. For example:
|
|
73
|
+
|
|
74
|
+
- `nodeHours` - Maps to the `nodeHours` field in API responses
|
|
75
|
+
- `storage` - Maps to the `storage` field in API responses (if available)
|
|
76
|
+
- `gpuHours` - Maps to the `gpuHours` field in API responses (if available)
|
|
77
|
+
|
|
78
|
+
Each component supports:
|
|
79
|
+
|
|
80
|
+
| Parameter | Description |
|
|
81
|
+
|-----------|-------------|
|
|
82
|
+
| `measured_unit` | Unit for display in Waldur (e.g., "node-hours", "TB") |
|
|
83
|
+
| `unit_factor` | Conversion factor from API units to measured units |
|
|
84
|
+
| `accounting_type` | Either "usage" for actual usage or "limit" for quotas |
|
|
85
|
+
| `label` | Display label in Waldur interface |
|
|
86
|
+
|
|
87
|
+
## Usage Data Format
|
|
88
|
+
|
|
89
|
+
The plugin reports usage for all configured components:
|
|
90
|
+
|
|
91
|
+
- **Component Types**: Configurable (e.g., `nodeHours`, `storage`, `gpuHours`)
|
|
92
|
+
- **Units**: Based on API response and `unit_factor` configuration
|
|
93
|
+
- **Granularity**: Monthly reporting with current month data
|
|
94
|
+
- **User Attribution**: Individual user usage within each account
|
|
95
|
+
- **Aggregation**: Automatically aggregates across clusters and time periods
|
|
96
|
+
|
|
97
|
+
## API Integration
|
|
98
|
+
|
|
99
|
+
The plugin uses the CSCS-DWDI API endpoints:
|
|
100
|
+
|
|
101
|
+
- `GET /api/v1/compute/usage-month-multiaccount` - Primary endpoint for monthly usage data
|
|
102
|
+
- Authentication via OIDC Bearer tokens
|
|
103
|
+
|
|
104
|
+
### Authentication
|
|
105
|
+
|
|
106
|
+
The plugin uses OAuth2/OIDC authentication with the following requirements:
|
|
107
|
+
|
|
108
|
+
- Requires `cscs_dwdi_oidc_token_url` in backend settings
|
|
109
|
+
- Uses OAuth2 `client_credentials` grant flow
|
|
110
|
+
- Automatically handles token caching and renewal
|
|
111
|
+
- Includes 5-minute safety margin for token expiry
|
|
112
|
+
- Fails with proper error logging if OIDC configuration is missing
|
|
113
|
+
|
|
114
|
+
### Data Processing
|
|
115
|
+
|
|
116
|
+
1. **Account Filtering**: Only reports on accounts that match Waldur resource backend IDs
|
|
117
|
+
2. **User Aggregation**: Combines usage for the same user across different dates and clusters
|
|
118
|
+
3. **Time Range**: Automatically queries from the first day of the current month to today
|
|
119
|
+
4. **Precision**: Rounds node-hours to 2 decimal places
|
|
120
|
+
|
|
121
|
+
## Installation
|
|
122
|
+
|
|
123
|
+
This plugin is part of the Waldur Site Agent workspace. To install:
|
|
124
|
+
|
|
125
|
+
```bash
|
|
126
|
+
# Install all workspace packages including cscs-dwdi plugin
|
|
127
|
+
uv sync --all-packages
|
|
128
|
+
|
|
129
|
+
# Install specific plugin for development
|
|
130
|
+
uv sync --extra cscs-dwdi
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
## Testing
|
|
134
|
+
|
|
135
|
+
Run the plugin tests:
|
|
136
|
+
|
|
137
|
+
```bash
|
|
138
|
+
# Run CSCS-DWDI plugin tests
|
|
139
|
+
uv run pytest plugins/cscs-dwdi/tests/
|
|
140
|
+
|
|
141
|
+
# Run with coverage
|
|
142
|
+
uv run pytest plugins/cscs-dwdi/tests/ --cov=waldur_site_agent_cscs_dwdi
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
## Limitations
|
|
146
|
+
|
|
147
|
+
This is a **reporting-only backend** that does not support:
|
|
148
|
+
|
|
149
|
+
- Account creation or deletion
|
|
150
|
+
- User management
|
|
151
|
+
- Resource limit management
|
|
152
|
+
- Order processing
|
|
153
|
+
- Membership synchronization
|
|
154
|
+
|
|
155
|
+
For these operations, use a different backend (e.g., SLURM) in combination with the CSCS-DWDI reporting backend:
|
|
156
|
+
|
|
157
|
+
```yaml
|
|
158
|
+
offerings:
|
|
159
|
+
- name: "Mixed Backend Offering"
|
|
160
|
+
order_processing_backend: "slurm" # Use SLURM for orders
|
|
161
|
+
reporting_backend: "cscs-dwdi" # Use CSCS-DWDI for reporting
|
|
162
|
+
membership_sync_backend: "slurm" # Use SLURM for membership
|
|
163
|
+
```
|
|
164
|
+
|
|
165
|
+
## Error Handling
|
|
166
|
+
|
|
167
|
+
The plugin includes comprehensive error handling:
|
|
168
|
+
|
|
169
|
+
- **API Connectivity**: Ping checks verify API availability
|
|
170
|
+
- **Authentication**: Token refresh and error handling
|
|
171
|
+
- **Data Validation**: Validates API responses and filters invalid data
|
|
172
|
+
- **Retry Logic**: Uses the framework's built-in retry mechanisms
|
|
173
|
+
|
|
174
|
+
## Development
|
|
175
|
+
|
|
176
|
+
### Project Structure
|
|
177
|
+
|
|
178
|
+
```text
|
|
179
|
+
plugins/cscs-dwdi/
|
|
180
|
+
├── pyproject.toml # Plugin configuration
|
|
181
|
+
├── README.md # This documentation
|
|
182
|
+
├── waldur_site_agent_cscs_dwdi/
|
|
183
|
+
│ ├── __init__.py # Package init
|
|
184
|
+
│ ├── backend.py # Main backend implementation
|
|
185
|
+
│ └── client.py # CSCS-DWDI API client
|
|
186
|
+
└── tests/
|
|
187
|
+
└── test_cscs_dwdi.py # Plugin tests
|
|
188
|
+
```
|
|
189
|
+
|
|
190
|
+
### Key Classes
|
|
191
|
+
|
|
192
|
+
- **`CSCSDWDIBackend`**: Main backend class implementing reporting functionality
|
|
193
|
+
- **`CSCSDWDIClient`**: HTTP client for CSCS-DWDI API communication
|
|
194
|
+
|
|
195
|
+
### Extension Points
|
|
196
|
+
|
|
197
|
+
To extend the plugin:
|
|
198
|
+
|
|
199
|
+
1. **Additional Endpoints**: Modify `CSCSDWDIClient` to support more API endpoints
|
|
200
|
+
2. **Authentication Methods**: Update authentication logic in `client.py`
|
|
201
|
+
3. **Data Processing**: Enhance `_process_api_response()` for additional data formats
|
|
202
|
+
|
|
203
|
+
## Troubleshooting
|
|
204
|
+
|
|
205
|
+
### Common Issues
|
|
206
|
+
|
|
207
|
+
#### Authentication Failures
|
|
208
|
+
|
|
209
|
+
- Verify OIDC client credentials
|
|
210
|
+
- Check API URL configuration
|
|
211
|
+
- Ensure proper token scopes
|
|
212
|
+
|
|
213
|
+
#### Missing Usage Data
|
|
214
|
+
|
|
215
|
+
- Verify account names match between Waldur and CSCS-DWDI
|
|
216
|
+
- Check date ranges and API response format
|
|
217
|
+
- Review API rate limits and quotas
|
|
218
|
+
|
|
219
|
+
#### Network Connectivity
|
|
220
|
+
|
|
221
|
+
- Test API connectivity with ping functionality
|
|
222
|
+
- Verify network access from agent deployment environment
|
|
223
|
+
- Check firewall and proxy settings
|
|
224
|
+
|
|
225
|
+
### Debugging
|
|
226
|
+
|
|
227
|
+
Enable debug logging for detailed API interactions:
|
|
228
|
+
|
|
229
|
+
```python
|
|
230
|
+
import logging
|
|
231
|
+
logging.getLogger('waldur_site_agent_cscs_dwdi').setLevel(logging.DEBUG)
|
|
232
|
+
```
|
|
233
|
+
|
|
234
|
+
## Support
|
|
235
|
+
|
|
236
|
+
For issues and questions:
|
|
237
|
+
|
|
238
|
+
- Check the [Waldur Site Agent documentation](../../docs/)
|
|
239
|
+
- Review plugin test cases for usage examples
|
|
240
|
+
- Create issues in the project repository
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
waldur_site_agent_cscs_dwdi/__init__.py,sha256=OHO1yF5NTGt0otI-GollR_ppPXP--aUZRCdaT5-8IWw,56
|
|
2
|
+
waldur_site_agent_cscs_dwdi/backend.py,sha256=V5T9H5b9utxcD73TmACvrc4dZm-GEORLDI0C0tUD_7s,14251
|
|
3
|
+
waldur_site_agent_cscs_dwdi/client.py,sha256=aAWNqrA0oBknWge_TQIU4u8PQi5YdIZtQTZ2XPREznQ,7555
|
|
4
|
+
waldur_site_agent_cscs_dwdi-0.1.0.dist-info/METADATA,sha256=nTtebzVzy0mWYqZejNICgI5hlHFDxbsPAk0woaSABHM,7772
|
|
5
|
+
waldur_site_agent_cscs_dwdi-0.1.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
|
6
|
+
waldur_site_agent_cscs_dwdi-0.1.0.dist-info/entry_points.txt,sha256=U3odcX7B4NmT9a98ov4uVxlng3JqiCRb5Ux3h4H_ZFE,93
|
|
7
|
+
waldur_site_agent_cscs_dwdi-0.1.0.dist-info/RECORD,,
|