waldur-site-agent-cscs-dwdi 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of waldur-site-agent-cscs-dwdi might be problematic. Click here for more details.

@@ -0,0 +1 @@
1
+ """CSCS-DWDI reporting plugin for Waldur Site Agent."""
@@ -0,0 +1,362 @@
1
+ """CSCS-DWDI backend implementation for usage reporting."""
2
+
3
+ import logging
4
+ from datetime import datetime, timezone
5
+ from typing import Any, Optional
6
+
7
+ from waldur_api_client.models.resource import Resource as WaldurResource
8
+
9
+ from waldur_site_agent.backend.backends import BaseBackend
10
+
11
+ from .client import CSCSDWDIClient
12
+
13
+ logger = logging.getLogger(__name__)
14
+
15
+
16
+ class CSCSDWDIBackend(BaseBackend):
17
+ """Backend for reporting usage from CSCS-DWDI API."""
18
+
19
+ def __init__(
20
+ self, backend_settings: dict[str, Any], backend_components: dict[str, dict]
21
+ ) -> None:
22
+ """Initialize CSCS-DWDI backend.
23
+
24
+ Args:
25
+ backend_settings: Backend-specific settings from the offering
26
+ backend_components: Component configuration from the offering
27
+ """
28
+ super().__init__(backend_settings, backend_components)
29
+ self.backend_type = "cscs-dwdi"
30
+
31
+ # Extract CSCS-DWDI specific configuration
32
+ self.api_url = backend_settings.get("cscs_dwdi_api_url", "")
33
+ self.client_id = backend_settings.get("cscs_dwdi_client_id", "")
34
+ self.client_secret = backend_settings.get("cscs_dwdi_client_secret", "")
35
+
36
+ # Required OIDC configuration
37
+ self.oidc_token_url = backend_settings.get("cscs_dwdi_oidc_token_url", "")
38
+ self.oidc_scope = backend_settings.get("cscs_dwdi_oidc_scope")
39
+
40
+ if not all([self.api_url, self.client_id, self.client_secret, self.oidc_token_url]):
41
+ msg = (
42
+ "CSCS-DWDI backend requires cscs_dwdi_api_url, cscs_dwdi_client_id, "
43
+ "cscs_dwdi_client_secret, and cscs_dwdi_oidc_token_url in backend_settings"
44
+ )
45
+ raise ValueError(msg)
46
+
47
+ self.cscs_client = CSCSDWDIClient(
48
+ api_url=self.api_url,
49
+ client_id=self.client_id,
50
+ client_secret=self.client_secret,
51
+ oidc_token_url=self.oidc_token_url,
52
+ oidc_scope=self.oidc_scope,
53
+ )
54
+
55
+ def ping(self, raise_exception: bool = False) -> bool: # noqa: ARG002
56
+ """Check if CSCS-DWDI API is accessible.
57
+
58
+ Args:
59
+ raise_exception: Whether to raise an exception on failure
60
+
61
+ Returns:
62
+ True if API is accessible, False otherwise
63
+ """
64
+ return self.cscs_client.ping()
65
+
66
+ def _get_usage_report(
67
+ self, resource_backend_ids: list[str]
68
+ ) -> dict[str, dict[str, dict[str, float]]]:
69
+ """Get usage report for specified resources.
70
+
71
+ This method queries the CSCS-DWDI API for the current month's usage
72
+ and formats it according to Waldur's expected structure.
73
+
74
+ Args:
75
+ resource_backend_ids: List of account identifiers to report on
76
+
77
+ Returns:
78
+ Dictionary mapping account names to usage data:
79
+ {
80
+ "account1": {
81
+ "TOTAL_ACCOUNT_USAGE": {
82
+ "nodeHours": 1234.56
83
+ },
84
+ "user1": {
85
+ "nodeHours": 500.00
86
+ },
87
+ "user2": {
88
+ "nodeHours": 734.56
89
+ }
90
+ },
91
+ ...
92
+ }
93
+ """
94
+ if not resource_backend_ids:
95
+ logger.warning("No resource backend IDs provided for usage report")
96
+ return {}
97
+
98
+ # Get current month's date range
99
+ today = datetime.now(tz=timezone.utc).date()
100
+ from_date = today.replace(day=1)
101
+ to_date = today
102
+
103
+ logger.info(
104
+ "Fetching usage report for %d accounts from %s to %s",
105
+ len(resource_backend_ids),
106
+ from_date,
107
+ to_date,
108
+ )
109
+
110
+ try:
111
+ # Query CSCS-DWDI API for usage data
112
+ response = self.cscs_client.get_usage_for_month(
113
+ accounts=resource_backend_ids,
114
+ from_date=from_date,
115
+ to_date=to_date,
116
+ )
117
+
118
+ # Process the response
119
+ usage_report = self._process_api_response(response)
120
+
121
+ # Filter to only include requested accounts
122
+ filtered_report = {
123
+ account: data
124
+ for account, data in usage_report.items()
125
+ if account in resource_backend_ids
126
+ }
127
+
128
+ logger.info(
129
+ "Successfully retrieved usage for %d accounts",
130
+ len(filtered_report),
131
+ )
132
+
133
+ return filtered_report
134
+
135
+ except Exception:
136
+ logger.exception("Failed to get usage report from CSCS-DWDI")
137
+ raise
138
+
139
+ def _process_api_response(
140
+ self, response: dict[str, Any]
141
+ ) -> dict[str, dict[str, dict[str, float]]]:
142
+ """Process CSCS-DWDI API response into Waldur format.
143
+
144
+ Args:
145
+ response: Raw API response from CSCS-DWDI
146
+
147
+ Returns:
148
+ Formatted usage report for Waldur with configured component mappings
149
+ """
150
+ usage_report = {}
151
+
152
+ # The response has a "compute" field with list of account data
153
+ compute_data = response.get("compute", [])
154
+
155
+ for account_data in compute_data:
156
+ account_name = account_data.get("account")
157
+ if not account_name:
158
+ logger.warning("Account data missing account name, skipping")
159
+ continue
160
+
161
+ # Extract total account usage for all configured components
162
+ total_usage = self._extract_component_usage_from_account_data(account_data)
163
+
164
+ # Initialize account entry
165
+ usage_report[account_name] = {"TOTAL_ACCOUNT_USAGE": total_usage}
166
+
167
+ # Process per-user usage
168
+ users = account_data.get("users", [])
169
+ user_usage: dict[str, dict[str, float]] = {}
170
+
171
+ for user_data in users:
172
+ username = user_data.get("username")
173
+ if not username:
174
+ continue
175
+
176
+ # Extract user component usage
177
+ user_component_usage = self._extract_component_usage_from_user_data(user_data)
178
+
179
+ if username in user_usage:
180
+ # Aggregate usage for same user across different dates/clusters
181
+ for component_name, value in user_component_usage.items():
182
+ user_usage[username][component_name] = (
183
+ user_usage[username].get(component_name, 0.0) + value
184
+ )
185
+ else:
186
+ user_usage[username] = user_component_usage
187
+
188
+ # Add rounded user usage to report
189
+ for username, component_usage in user_usage.items():
190
+ rounded_usage = {comp: round(value, 2) for comp, value in component_usage.items()}
191
+ usage_report[account_name][username] = rounded_usage
192
+
193
+ return usage_report
194
+
195
+ def _extract_component_usage_from_account_data(
196
+ self, account_data: dict[str, Any]
197
+ ) -> dict[str, float]:
198
+ """Extract component usage from account-level data.
199
+
200
+ Args:
201
+ account_data: Account data from CSCS-DWDI API response
202
+
203
+ Returns:
204
+ Dictionary mapping component names to usage values
205
+ """
206
+ usage = {}
207
+
208
+ for component_name, component_config in self.backend_components.items():
209
+ # Look for component usage in account data
210
+ # Try multiple naming patterns to match API response fields to component names
211
+ raw_value = 0.0
212
+
213
+ # Try exact match first (e.g., nodeHours -> nodeHours)
214
+ if component_name in account_data:
215
+ raw_value = account_data[component_name]
216
+ # Try account prefix preserving case (e.g., nodeHours -> accountNodeHours)
217
+ elif f"account{component_name[0].upper()}{component_name[1:]}" in account_data:
218
+ raw_value = account_data[f"account{component_name[0].upper()}{component_name[1:]}"]
219
+ # Try total prefix (e.g., nodeHours -> totalNodeHours)
220
+ elif f"total{component_name[0].upper()}{component_name[1:]}" in account_data:
221
+ raw_value = account_data[f"total{component_name[0].upper()}{component_name[1:]}"]
222
+ # Try lowercase with account prefix
223
+ elif f"account{component_name}" in account_data:
224
+ raw_value = account_data[f"account{component_name}"]
225
+
226
+ # Apply unit factor conversion
227
+ unit_factor = component_config.get("unit_factor", 1)
228
+ converted_value = raw_value * unit_factor
229
+ usage[component_name] = round(converted_value, 2)
230
+
231
+ return usage
232
+
233
+ def _extract_component_usage_from_user_data(
234
+ self, user_data: dict[str, Any]
235
+ ) -> dict[str, float]:
236
+ """Extract component usage from user-level data.
237
+
238
+ Args:
239
+ user_data: User data from CSCS-DWDI API response
240
+
241
+ Returns:
242
+ Dictionary mapping component names to usage values
243
+ """
244
+ usage = {}
245
+
246
+ for component_name, component_config in self.backend_components.items():
247
+ # Look for component usage in user data
248
+ # The API should return fields that match component names
249
+ raw_value = user_data.get(component_name, 0.0)
250
+
251
+ # Apply unit factor conversion
252
+ unit_factor = component_config.get("unit_factor", 1)
253
+ converted_value = raw_value * unit_factor
254
+ usage[component_name] = converted_value
255
+
256
+ return usage
257
+
258
+ # Methods not implemented for reporting-only backend
259
+ def get_account(self, account_name: str) -> Optional[dict[str, Any]]:
260
+ """Not implemented for reporting-only backend."""
261
+ msg = "CSCS-DWDI backend is reporting-only and does not support account management"
262
+ raise NotImplementedError(msg)
263
+
264
+ def create_account(self, account_data: dict) -> bool:
265
+ """Not implemented for reporting-only backend."""
266
+ msg = "CSCS-DWDI backend is reporting-only and does not support account creation"
267
+ raise NotImplementedError(msg)
268
+
269
+ def delete_account(self, account_name: str) -> bool:
270
+ """Not implemented for reporting-only backend."""
271
+ msg = "CSCS-DWDI backend is reporting-only and does not support account deletion"
272
+ raise NotImplementedError(msg)
273
+
274
+ def update_account_limit_deposit(
275
+ self,
276
+ account_name: str,
277
+ component_type: str,
278
+ component_amount: float,
279
+ offering_component_data: dict,
280
+ ) -> bool:
281
+ """Not implemented for reporting-only backend."""
282
+ msg = "CSCS-DWDI backend is reporting-only and does not support limit updates"
283
+ raise NotImplementedError(msg)
284
+
285
+ def reset_account_limit_deposit(
286
+ self,
287
+ account_name: str,
288
+ component_type: str,
289
+ offering_component_data: dict,
290
+ ) -> bool:
291
+ """Not implemented for reporting-only backend."""
292
+ msg = "CSCS-DWDI backend is reporting-only and does not support limit resets"
293
+ raise NotImplementedError(msg)
294
+
295
+ def add_account_users(self, account_name: str, user_backend_ids: list[str]) -> bool:
296
+ """Not implemented for reporting-only backend."""
297
+ msg = "CSCS-DWDI backend is reporting-only and does not support user management"
298
+ raise NotImplementedError(msg)
299
+
300
+ def delete_account_users(self, account_name: str, user_backend_ids: list[str]) -> bool:
301
+ """Not implemented for reporting-only backend."""
302
+ msg = "CSCS-DWDI backend is reporting-only and does not support user management"
303
+ raise NotImplementedError(msg)
304
+
305
+ def list_accounts(self) -> list[dict[str, Any]]:
306
+ """Not implemented for reporting-only backend."""
307
+ msg = "CSCS-DWDI backend is reporting-only and does not support account listing"
308
+ raise NotImplementedError(msg)
309
+
310
+ def set_resource_limits(self, resource_backend_id: str, limits: dict[str, int]) -> None:
311
+ """Not implemented for reporting-only backend."""
312
+ msg = "CSCS-DWDI backend is reporting-only and does not support resource limits"
313
+ raise NotImplementedError(msg)
314
+
315
+ def diagnostics(self) -> bool:
316
+ """Get diagnostic information for the backend."""
317
+ logger.info(
318
+ "CSCS-DWDI Backend Diagnostics - Type: %s, API: %s, Components: %s, Ping: %s",
319
+ self.backend_type,
320
+ self.api_url,
321
+ list(self.backend_components.keys()),
322
+ self.ping(),
323
+ )
324
+ return self.ping()
325
+
326
+ def get_resource_metadata(self, resource_backend_id: str) -> dict[str, Any]:
327
+ """Not implemented for reporting-only backend."""
328
+ msg = "CSCS-DWDI backend is reporting-only and does not support resource metadata"
329
+ raise NotImplementedError(msg)
330
+
331
+ def list_components(self) -> list[str]:
332
+ """List configured components for this backend."""
333
+ return list(self.backend_components.keys())
334
+
335
+ def _collect_resource_limits(
336
+ self, waldur_resource: WaldurResource
337
+ ) -> tuple[dict[str, int], dict[str, int]]:
338
+ """Not implemented for reporting-only backend."""
339
+ msg = "CSCS-DWDI backend is reporting-only and does not support resource limits"
340
+ raise NotImplementedError(msg)
341
+
342
+ def _pre_create_resource(
343
+ self, waldur_resource: WaldurResource, user_context: Optional[dict] = None
344
+ ) -> None:
345
+ """Not implemented for reporting-only backend."""
346
+ msg = "CSCS-DWDI backend is reporting-only and does not support resource creation"
347
+ raise NotImplementedError(msg)
348
+
349
+ def pause_resource(self, resource_backend_id: str) -> bool:
350
+ """Not implemented for reporting-only backend."""
351
+ msg = "CSCS-DWDI backend is reporting-only and does not support resource pausing"
352
+ raise NotImplementedError(msg)
353
+
354
+ def restore_resource(self, resource_backend_id: str) -> bool:
355
+ """Not implemented for reporting-only backend."""
356
+ msg = "CSCS-DWDI backend is reporting-only and does not support resource restoration"
357
+ raise NotImplementedError(msg)
358
+
359
+ def downscale_resource(self, resource_backend_id: str) -> bool:
360
+ """Not implemented for reporting-only backend."""
361
+ msg = "CSCS-DWDI backend is reporting-only and does not support resource downscaling"
362
+ raise NotImplementedError(msg)
@@ -0,0 +1,239 @@
1
+ """CSCS-DWDI API client implementation."""
2
+
3
+ import logging
4
+ from datetime import date, datetime, timedelta, timezone
5
+ from typing import Any, Optional
6
+
7
+ import httpx
8
+
9
+ logger = logging.getLogger(__name__)
10
+
11
+ HTTP_OK = 200
12
+
13
+
14
+ class CSCSDWDIClient:
15
+ """Client for interacting with CSCS-DWDI API."""
16
+
17
+ def __init__(
18
+ self,
19
+ api_url: str,
20
+ client_id: str,
21
+ client_secret: str,
22
+ oidc_token_url: Optional[str] = None,
23
+ oidc_scope: Optional[str] = None,
24
+ ) -> None:
25
+ """Initialize CSCS-DWDI client.
26
+
27
+ Args:
28
+ api_url: Base URL for the CSCS-DWDI API
29
+ client_id: OIDC client ID for authentication
30
+ client_secret: OIDC client secret for authentication
31
+ oidc_token_url: OIDC token endpoint URL (required for authentication)
32
+ oidc_scope: OIDC scope to request (optional)
33
+ """
34
+ self.api_url = api_url.rstrip("/")
35
+ self.client_id = client_id
36
+ self.client_secret = client_secret
37
+ self.oidc_token_url = oidc_token_url
38
+ self.oidc_scope = oidc_scope or "openid"
39
+ self._token: Optional[str] = None
40
+ self._token_expires_at: Optional[datetime] = None
41
+
42
+ def _get_auth_token(self) -> str:
43
+ """Get or refresh OIDC authentication token.
44
+
45
+ Returns:
46
+ Valid authentication token
47
+
48
+ Raises:
49
+ httpx.HTTPError: If token acquisition fails
50
+ """
51
+ # Check if we have a valid cached token
52
+ if (
53
+ self._token
54
+ and self._token_expires_at
55
+ and datetime.now(tz=timezone.utc) < self._token_expires_at
56
+ ):
57
+ return self._token
58
+
59
+ # Fail if OIDC endpoint not configured
60
+ if not self.oidc_token_url:
61
+ error_msg = (
62
+ "OIDC authentication failed: cscs_dwdi_oidc_token_url not configured. "
63
+ "Set 'cscs_dwdi_oidc_token_url' in backend_settings for production use."
64
+ )
65
+ logger.error(error_msg)
66
+ raise ValueError(error_msg)
67
+
68
+ # Request new token from OIDC provider
69
+ return self._acquire_oidc_token()
70
+
71
+ def _acquire_oidc_token(self) -> str:
72
+ """Acquire a new OIDC token from the configured provider.
73
+
74
+ Returns:
75
+ Valid authentication token
76
+
77
+ Raises:
78
+ httpx.HTTPError: If token acquisition fails
79
+ """
80
+ logger.debug("Acquiring new OIDC token from %s", self.oidc_token_url)
81
+
82
+ token_data = {
83
+ "grant_type": "client_credentials",
84
+ "client_id": self.client_id,
85
+ "client_secret": self.client_secret,
86
+ }
87
+
88
+ # Add scope if specified
89
+ if self.oidc_scope:
90
+ token_data["scope"] = self.oidc_scope
91
+
92
+ headers = {"Content-Type": "application/x-www-form-urlencoded"}
93
+
94
+ with httpx.Client() as client:
95
+ response = client.post(
96
+ self.oidc_token_url, data=token_data, headers=headers, timeout=30.0
97
+ )
98
+ response.raise_for_status()
99
+ token_response = response.json()
100
+
101
+ # Extract token and expiry information
102
+ access_token = token_response.get("access_token")
103
+ if not access_token:
104
+ msg = f"No access_token in OIDC response: {token_response}"
105
+ raise ValueError(msg)
106
+
107
+ # Calculate token expiry time
108
+ expires_in = token_response.get("expires_in", 3600) # Default to 1 hour
109
+ # Subtract 5 minutes from expiry for safety margin
110
+ safe_expires_in = max(300, expires_in - 300)
111
+
112
+ self._token = access_token
113
+ self._token_expires_at = datetime.now(tz=timezone.utc) + timedelta(
114
+ seconds=safe_expires_in
115
+ )
116
+
117
+ logger.info("Successfully acquired OIDC token, expires in %d seconds", expires_in)
118
+
119
+ return self._token
120
+
121
+ def get_usage_for_month(
122
+ self, accounts: list[str], from_date: date, to_date: date
123
+ ) -> dict[str, Any]:
124
+ """Get usage data for multiple accounts for a month range.
125
+
126
+ Args:
127
+ accounts: List of account identifiers to query
128
+ from_date: Start date (beginning of month)
129
+ to_date: End date (end of month)
130
+
131
+ Returns:
132
+ API response with usage data grouped by account
133
+
134
+ Raises:
135
+ httpx.HTTPError: If API request fails
136
+ """
137
+ token = self._get_auth_token()
138
+
139
+ # Format dates as YYYY-MM for month endpoints
140
+ from_month = from_date.strftime("%Y-%m")
141
+ to_month = to_date.strftime("%Y-%m")
142
+
143
+ params: dict[str, Any] = {
144
+ "from": from_month,
145
+ "to": to_month,
146
+ }
147
+
148
+ # Add account filters if provided
149
+ if accounts:
150
+ params["account"] = accounts
151
+
152
+ headers = {"Authorization": f"Bearer {token}"}
153
+
154
+ url = f"{self.api_url}/api/v1/compute/usage-month-multiaccount"
155
+
156
+ logger.debug(
157
+ "Fetching usage for accounts %s from %s to %s",
158
+ accounts,
159
+ from_month,
160
+ to_month,
161
+ )
162
+
163
+ with httpx.Client() as client:
164
+ response = client.get(url, params=params, headers=headers, timeout=30.0)
165
+ response.raise_for_status()
166
+ return response.json()
167
+
168
+ def get_usage_for_days(
169
+ self, accounts: list[str], from_date: date, to_date: date
170
+ ) -> dict[str, Any]:
171
+ """Get usage data for multiple accounts for a day range.
172
+
173
+ Args:
174
+ accounts: List of account identifiers to query
175
+ from_date: Start date
176
+ to_date: End date
177
+
178
+ Returns:
179
+ API response with usage data grouped by account
180
+
181
+ Raises:
182
+ httpx.HTTPError: If API request fails
183
+ """
184
+ token = self._get_auth_token()
185
+
186
+ # Format dates as YYYY-MM-DD for day endpoints
187
+ from_day = from_date.strftime("%Y-%m-%d")
188
+ to_day = to_date.strftime("%Y-%m-%d")
189
+
190
+ params: dict[str, Any] = {
191
+ "from": from_day,
192
+ "to": to_day,
193
+ }
194
+
195
+ # Add account filters if provided
196
+ if accounts:
197
+ params["account"] = accounts
198
+
199
+ headers = {"Authorization": f"Bearer {token}"}
200
+
201
+ url = f"{self.api_url}/api/v1/compute/usage-day-multiaccount"
202
+
203
+ logger.debug(
204
+ "Fetching daily usage for accounts %s from %s to %s",
205
+ accounts,
206
+ from_day,
207
+ to_day,
208
+ )
209
+
210
+ with httpx.Client() as client:
211
+ response = client.get(url, params=params, headers=headers, timeout=30.0)
212
+ response.raise_for_status()
213
+ return response.json()
214
+
215
+ def ping(self) -> bool:
216
+ """Check if CSCS-DWDI API is accessible.
217
+
218
+ Returns:
219
+ True if API is accessible, False otherwise
220
+ """
221
+ try:
222
+ token = self._get_auth_token()
223
+ headers = {"Authorization": f"Bearer {token}"}
224
+
225
+ # Use a simple query to test connectivity
226
+ today = datetime.now(tz=timezone.utc).date()
227
+ params = {
228
+ "from": today.strftime("%Y-%m-%d"),
229
+ "to": today.strftime("%Y-%m-%d"),
230
+ }
231
+
232
+ url = f"{self.api_url}/api/v1/compute/usage-day-multiaccount"
233
+
234
+ with httpx.Client() as client:
235
+ response = client.get(url, params=params, headers=headers, timeout=10.0)
236
+ return response.status_code == HTTP_OK
237
+ except Exception:
238
+ logger.exception("Ping failed")
239
+ return False
@@ -0,0 +1,240 @@
1
+ Metadata-Version: 2.4
2
+ Name: waldur-site-agent-cscs-dwdi
3
+ Version: 0.1.0
4
+ Summary: CSCS-DWDI reporting plugin for Waldur Site Agent
5
+ Author-email: OpenNode Team <info@opennodecloud.com>
6
+ Requires-Python: <4,>=3.9
7
+ Requires-Dist: httpx>=0.25.0
8
+ Requires-Dist: waldur-site-agent==0.1.0
9
+ Description-Content-Type: text/markdown
10
+
11
+ # CSCS-DWDI Plugin for Waldur Site Agent
12
+
13
+ This plugin provides reporting functionality for Waldur Site Agent by integrating with the CSCS-DWDI
14
+ (Data Warehouse and Data Intelligence) API.
15
+
16
+ ## Overview
17
+
18
+ The CSCS-DWDI plugin is a **reporting-only backend** that fetches compute usage data from the CSCS-DWDI
19
+ service and reports it to Waldur. It supports node-hour usage tracking for multiple accounts and users.
20
+
21
+ ## Features
22
+
23
+ - **Monthly Usage Reporting**: Fetches usage data for the current month
24
+ - **Multi-Account Support**: Reports usage for multiple accounts in a single API call
25
+ - **Per-User Usage**: Breaks down usage by individual users within each account
26
+ - **OIDC Authentication**: Uses OAuth2/OIDC for secure API access
27
+ - **Automatic Aggregation**: Combines usage across different clusters and time periods
28
+
29
+ ## Configuration
30
+
31
+ Add the following configuration to your Waldur Site Agent offering:
32
+
33
+ ```yaml
34
+ offerings:
35
+ - name: "CSCS HPC Offering"
36
+ reporting_backend: "cscs-dwdi"
37
+ backend_settings:
38
+ cscs_dwdi_api_url: "https://dwdi-api.cscs.ch"
39
+ cscs_dwdi_client_id: "your-oidc-client-id"
40
+ cscs_dwdi_client_secret: "your-oidc-client-secret"
41
+ # Optional OIDC configuration (for production use)
42
+ cscs_dwdi_oidc_token_url: "https://identity.cscs.ch/realms/cscs/protocol/openid-connect/token"
43
+ cscs_dwdi_oidc_scope: "cscs-dwdi:read"
44
+
45
+ backend_components:
46
+ nodeHours:
47
+ measured_unit: "node-hours"
48
+ unit_factor: 1
49
+ accounting_type: "usage"
50
+ label: "Node Hours"
51
+ storage:
52
+ measured_unit: "TB"
53
+ unit_factor: 1
54
+ accounting_type: "usage"
55
+ label: "Storage Usage"
56
+ ```
57
+
58
+ ### Configuration Parameters
59
+
60
+ #### Backend Settings
61
+
62
+ | Parameter | Required | Description |
63
+ |-----------|----------|-------------|
64
+ | `cscs_dwdi_api_url` | Yes | Base URL for the CSCS-DWDI API service |
65
+ | `cscs_dwdi_client_id` | Yes | OIDC client ID for authentication |
66
+ | `cscs_dwdi_client_secret` | Yes | OIDC client secret for authentication |
67
+ | `cscs_dwdi_oidc_token_url` | Yes | OIDC token endpoint URL (required for authentication) |
68
+ | `cscs_dwdi_oidc_scope` | No | OIDC scope to request (defaults to "openid") |
69
+
70
+ #### Backend Components
71
+
72
+ Components must match the field names returned by the CSCS-DWDI API. For example:
73
+
74
+ - `nodeHours` - Maps to the `nodeHours` field in API responses
75
+ - `storage` - Maps to the `storage` field in API responses (if available)
76
+ - `gpuHours` - Maps to the `gpuHours` field in API responses (if available)
77
+
78
+ Each component supports:
79
+
80
+ | Parameter | Description |
81
+ |-----------|-------------|
82
+ | `measured_unit` | Unit for display in Waldur (e.g., "node-hours", "TB") |
83
+ | `unit_factor` | Conversion factor from API units to measured units |
84
+ | `accounting_type` | Either "usage" for actual usage or "limit" for quotas |
85
+ | `label` | Display label in Waldur interface |
86
+
87
+ ## Usage Data Format
88
+
89
+ The plugin reports usage for all configured components:
90
+
91
+ - **Component Types**: Configurable (e.g., `nodeHours`, `storage`, `gpuHours`)
92
+ - **Units**: Based on API response and `unit_factor` configuration
93
+ - **Granularity**: Monthly reporting with current month data
94
+ - **User Attribution**: Individual user usage within each account
95
+ - **Aggregation**: Automatically aggregates across clusters and time periods
96
+
97
+ ## API Integration
98
+
99
+ The plugin uses the CSCS-DWDI API endpoints:
100
+
101
+ - `GET /api/v1/compute/usage-month-multiaccount` - Primary endpoint for monthly usage data
102
+ - Authentication via OIDC Bearer tokens
103
+
104
+ ### Authentication
105
+
106
+ The plugin uses OAuth2/OIDC authentication with the following requirements:
107
+
108
+ - Requires `cscs_dwdi_oidc_token_url` in backend settings
109
+ - Uses OAuth2 `client_credentials` grant flow
110
+ - Automatically handles token caching and renewal
111
+ - Includes 5-minute safety margin for token expiry
112
+ - Fails with proper error logging if OIDC configuration is missing
113
+
114
+ ### Data Processing
115
+
116
+ 1. **Account Filtering**: Only reports on accounts that match Waldur resource backend IDs
117
+ 2. **User Aggregation**: Combines usage for the same user across different dates and clusters
118
+ 3. **Time Range**: Automatically queries from the first day of the current month to today
119
+ 4. **Precision**: Rounds node-hours to 2 decimal places
120
+
121
+ ## Installation
122
+
123
+ This plugin is part of the Waldur Site Agent workspace. To install:
124
+
125
+ ```bash
126
+ # Install all workspace packages including cscs-dwdi plugin
127
+ uv sync --all-packages
128
+
129
+ # Install specific plugin for development
130
+ uv sync --extra cscs-dwdi
131
+ ```
132
+
133
+ ## Testing
134
+
135
+ Run the plugin tests:
136
+
137
+ ```bash
138
+ # Run CSCS-DWDI plugin tests
139
+ uv run pytest plugins/cscs-dwdi/tests/
140
+
141
+ # Run with coverage
142
+ uv run pytest plugins/cscs-dwdi/tests/ --cov=waldur_site_agent_cscs_dwdi
143
+ ```
144
+
145
+ ## Limitations
146
+
147
+ This is a **reporting-only backend** that does not support:
148
+
149
+ - Account creation or deletion
150
+ - User management
151
+ - Resource limit management
152
+ - Order processing
153
+ - Membership synchronization
154
+
155
+ For these operations, use a different backend (e.g., SLURM) in combination with the CSCS-DWDI reporting backend:
156
+
157
+ ```yaml
158
+ offerings:
159
+ - name: "Mixed Backend Offering"
160
+ order_processing_backend: "slurm" # Use SLURM for orders
161
+ reporting_backend: "cscs-dwdi" # Use CSCS-DWDI for reporting
162
+ membership_sync_backend: "slurm" # Use SLURM for membership
163
+ ```
164
+
165
+ ## Error Handling
166
+
167
+ The plugin includes comprehensive error handling:
168
+
169
+ - **API Connectivity**: Ping checks verify API availability
170
+ - **Authentication**: Token refresh and error handling
171
+ - **Data Validation**: Validates API responses and filters invalid data
172
+ - **Retry Logic**: Uses the framework's built-in retry mechanisms
173
+
174
+ ## Development
175
+
176
+ ### Project Structure
177
+
178
+ ```text
179
+ plugins/cscs-dwdi/
180
+ ├── pyproject.toml # Plugin configuration
181
+ ├── README.md # This documentation
182
+ ├── waldur_site_agent_cscs_dwdi/
183
+ │ ├── __init__.py # Package init
184
+ │ ├── backend.py # Main backend implementation
185
+ │ └── client.py # CSCS-DWDI API client
186
+ └── tests/
187
+ └── test_cscs_dwdi.py # Plugin tests
188
+ ```
189
+
190
+ ### Key Classes
191
+
192
+ - **`CSCSDWDIBackend`**: Main backend class implementing reporting functionality
193
+ - **`CSCSDWDIClient`**: HTTP client for CSCS-DWDI API communication
194
+
195
+ ### Extension Points
196
+
197
+ To extend the plugin:
198
+
199
+ 1. **Additional Endpoints**: Modify `CSCSDWDIClient` to support more API endpoints
200
+ 2. **Authentication Methods**: Update authentication logic in `client.py`
201
+ 3. **Data Processing**: Enhance `_process_api_response()` for additional data formats
202
+
203
+ ## Troubleshooting
204
+
205
+ ### Common Issues
206
+
207
+ #### Authentication Failures
208
+
209
+ - Verify OIDC client credentials
210
+ - Check API URL configuration
211
+ - Ensure proper token scopes
212
+
213
+ #### Missing Usage Data
214
+
215
+ - Verify account names match between Waldur and CSCS-DWDI
216
+ - Check date ranges and API response format
217
+ - Review API rate limits and quotas
218
+
219
+ #### Network Connectivity
220
+
221
+ - Test API connectivity with ping functionality
222
+ - Verify network access from agent deployment environment
223
+ - Check firewall and proxy settings
224
+
225
+ ### Debugging
226
+
227
+ Enable debug logging for detailed API interactions:
228
+
229
+ ```python
230
+ import logging
231
+ logging.getLogger('waldur_site_agent_cscs_dwdi').setLevel(logging.DEBUG)
232
+ ```
233
+
234
+ ## Support
235
+
236
+ For issues and questions:
237
+
238
+ - Check the [Waldur Site Agent documentation](../../docs/)
239
+ - Review plugin test cases for usage examples
240
+ - Create issues in the project repository
@@ -0,0 +1,7 @@
1
+ waldur_site_agent_cscs_dwdi/__init__.py,sha256=OHO1yF5NTGt0otI-GollR_ppPXP--aUZRCdaT5-8IWw,56
2
+ waldur_site_agent_cscs_dwdi/backend.py,sha256=V5T9H5b9utxcD73TmACvrc4dZm-GEORLDI0C0tUD_7s,14251
3
+ waldur_site_agent_cscs_dwdi/client.py,sha256=aAWNqrA0oBknWge_TQIU4u8PQi5YdIZtQTZ2XPREznQ,7555
4
+ waldur_site_agent_cscs_dwdi-0.1.0.dist-info/METADATA,sha256=nTtebzVzy0mWYqZejNICgI5hlHFDxbsPAk0woaSABHM,7772
5
+ waldur_site_agent_cscs_dwdi-0.1.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
6
+ waldur_site_agent_cscs_dwdi-0.1.0.dist-info/entry_points.txt,sha256=U3odcX7B4NmT9a98ov4uVxlng3JqiCRb5Ux3h4H_ZFE,93
7
+ waldur_site_agent_cscs_dwdi-0.1.0.dist-info/RECORD,,
@@ -0,0 +1,4 @@
1
+ Wheel-Version: 1.0
2
+ Generator: hatchling 1.27.0
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
@@ -0,0 +1,2 @@
1
+ [waldur_site_agent.backends]
2
+ cscs-dwdi = waldur_site_agent_cscs_dwdi.backend:CSCSDWDIBackend