waldur-site-agent-cscs-dwdi 0.1.0__py3-none-any.whl → 0.7.3__py3-none-any.whl

waldur_site_agent_cscs_dwdi/backend.py

@@ -1,4 +1,4 @@
-"""CSCS-DWDI backend implementation for usage reporting."""
+"""CSCS-DWDI backend implementations for compute and storage usage reporting."""
 
 import logging
 from datetime import datetime, timezone
@@ -13,8 +13,8 @@ from .client import CSCSDWDIClient
 logger = logging.getLogger(__name__)
 
 
-class CSCSDWDIBackend(BaseBackend):
-    """Backend for reporting usage from CSCS-DWDI API."""
+class CSCSDWDIComputeBackend(BaseBackend):
+    """Backend for reporting compute usage from CSCS-DWDI API."""
 
     def __init__(
         self, backend_settings: dict[str, Any], backend_components: dict[str, dict]
@@ -26,7 +26,7 @@ class CSCSDWDIBackend(BaseBackend):
             backend_components: Component configuration from the offering
         """
         super().__init__(backend_settings, backend_components)
-        self.backend_type = "cscs-dwdi"
+        self.backend_type = "cscs-dwdi-compute"
 
         # Extract CSCS-DWDI specific configuration
         self.api_url = backend_settings.get("cscs_dwdi_api_url", "")
@@ -37,6 +37,9 @@ class CSCSDWDIBackend(BaseBackend):
         self.oidc_token_url = backend_settings.get("cscs_dwdi_oidc_token_url", "")
         self.oidc_scope = backend_settings.get("cscs_dwdi_oidc_scope")
 
+        # Optional SOCKS proxy configuration
+        self.socks_proxy = backend_settings.get("socks_proxy")
+
         if not all([self.api_url, self.client_id, self.client_secret, self.oidc_token_url]):
             msg = (
                 "CSCS-DWDI backend requires cscs_dwdi_api_url, cscs_dwdi_client_id, "
@@ -50,8 +53,12 @@ class CSCSDWDIBackend(BaseBackend):
             client_secret=self.client_secret,
             oidc_token_url=self.oidc_token_url,
             oidc_scope=self.oidc_scope,
+            socks_proxy=self.socks_proxy,
         )
 
+        if self.socks_proxy:
+            logger.info("CSCS-DWDI Compute Backend: Using SOCKS proxy: %s", self.socks_proxy)
+
     def ping(self, raise_exception: bool = False) -> bool:  # noqa: ARG002
         """Check if CSCS-DWDI API is accessible.
 
@@ -360,3 +367,299 @@ class CSCSDWDIBackend(BaseBackend):
         """Not implemented for reporting-only backend."""
         msg = "CSCS-DWDI backend is reporting-only and does not support resource downscaling"
         raise NotImplementedError(msg)
+
+
+class CSCSDWDIStorageBackend(BaseBackend):
+    """Backend for reporting storage usage from CSCS-DWDI API."""
+
+    def __init__(
+        self, backend_settings: dict[str, Any], backend_components: dict[str, dict]
+    ) -> None:
+        """Initialize CSCS-DWDI storage backend.
+
+        Args:
+            backend_settings: Backend-specific settings from the offering
+            backend_components: Component configuration from the offering
+        """
+        super().__init__(backend_settings, backend_components)
+        self.backend_type = "cscs-dwdi-storage"
+
+        # Extract CSCS-DWDI specific configuration
+        self.api_url = backend_settings.get("cscs_dwdi_api_url", "")
+        self.client_id = backend_settings.get("cscs_dwdi_client_id", "")
+        self.client_secret = backend_settings.get("cscs_dwdi_client_secret", "")
+
+        # Required OIDC configuration
+        self.oidc_token_url = backend_settings.get("cscs_dwdi_oidc_token_url", "")
+        self.oidc_scope = backend_settings.get("cscs_dwdi_oidc_scope")
+
+        # Storage-specific settings
+        self.filesystem = backend_settings.get("storage_filesystem", "")
+        self.data_type = backend_settings.get("storage_data_type", "")
+        self.tenant = backend_settings.get("storage_tenant", "")
+        self.path_mapping = backend_settings.get("storage_path_mapping", {})
+
+        # Optional SOCKS proxy configuration
+        self.socks_proxy = backend_settings.get("socks_proxy")
+
+        if not all([self.api_url, self.client_id, self.client_secret, self.oidc_token_url]):
+            msg = (
+                "CSCS-DWDI storage backend requires cscs_dwdi_api_url, cscs_dwdi_client_id, "
+                "cscs_dwdi_client_secret, and cscs_dwdi_oidc_token_url in backend_settings"
+            )
+            raise ValueError(msg)
+
+        if not all([self.filesystem, self.data_type]):
+            msg = (
+                "CSCS-DWDI storage backend requires storage_filesystem and storage_data_type "
+                "in backend_settings"
+            )
+            raise ValueError(msg)
+
+        self.cscs_client = CSCSDWDIClient(
+            api_url=self.api_url,
+            client_id=self.client_id,
+            client_secret=self.client_secret,
+            oidc_token_url=self.oidc_token_url,
+            oidc_scope=self.oidc_scope,
+            socks_proxy=self.socks_proxy,
+        )
+
+        if self.socks_proxy:
+            logger.info("CSCS-DWDI Storage Backend: Using SOCKS proxy: %s", self.socks_proxy)
+
+    def ping(self, raise_exception: bool = False) -> bool:  # noqa: ARG002
+        """Check if CSCS-DWDI API is accessible.
+
+        Args:
+            raise_exception: Whether to raise an exception on failure
+
+        Returns:
+            True if API is accessible, False otherwise
+        """
+        return self.cscs_client.ping_storage()
+
+    def _get_usage_report(
+        self, resource_backend_ids: list[str]
+    ) -> dict[str, dict[str, dict[str, float]]]:
+        """Get storage usage report for specified resources.
+
+        This method queries the CSCS-DWDI storage API for the current month's usage
+        and formats it according to Waldur's expected structure.
+
+        Args:
+            resource_backend_ids: List of resource identifiers (paths or mapped IDs)
+
+        Returns:
+            Dictionary mapping resource IDs to usage data:
+            {
+                "resource1": {
+                    "TOTAL_ACCOUNT_USAGE": {
+                        "storage_space": 1234.56,  # GB
+                        "storage_inodes": 50000
+                    }
+                },
+                ...
+            }
+        """
+        if not resource_backend_ids:
+            logger.warning("No resource backend IDs provided for storage usage report")
+            return {}
+
+        # Get current month for reporting
+        today = datetime.now(tz=timezone.utc).date()
+        exact_month = today.strftime("%Y-%m")
+
+        logger.info(
+            "Fetching storage usage report for %d resources for month %s",
+            len(resource_backend_ids),
+            exact_month,
+        )
+
+        usage_report = {}
+
+        try:
+            # Map resource IDs to storage paths
+            paths_to_query = []
+            id_to_path_map = {}
+
+            for resource_id in resource_backend_ids:
+                # Check if we have a path mapping for this resource ID
+                if resource_id in self.path_mapping:
+                    path = self.path_mapping[resource_id]
+                    paths_to_query.append(path)
+                    id_to_path_map[path] = resource_id
+                else:
+                    # Assume the resource_id is the path itself
+                    paths_to_query.append(resource_id)
+                    id_to_path_map[resource_id] = resource_id
+
+            if not paths_to_query:
+                logger.warning("No paths to query after mapping")
+                return {}
+
+            # Query CSCS-DWDI storage API
+            response = self.cscs_client.get_storage_usage_for_month(
+                paths=paths_to_query,
+                tenant=self.tenant,
+                filesystem=self.filesystem,
+                data_type=self.data_type,
+                exact_month=exact_month,
+            )
+
+            # Process the response
+            storage_data = response.get("storage", [])
+
+            for storage_entry in storage_data:
+                path = storage_entry.get("path")
+                if not path:
+                    logger.warning("Storage entry missing path, skipping")
+                    continue
+
+                # Map path back to resource ID
+                resource_id = id_to_path_map.get(path, path)
+
+                # Extract storage metrics
+                space_used_bytes = storage_entry.get("spaceUsed", 0)
+                inodes_used = storage_entry.get("inodesUsed", 0)
+
+                # Convert bytes to configured units (typically GB)
+                storage_usage = {}
+                for component_name, component_config in self.backend_components.items():
+                    if (
+                        "storage_space" in component_name.lower()
+                        or "space" in component_name.lower()
+                    ):
+                        # Apply unit factor for space (e.g., bytes to GB)
+                        unit_factor = component_config.get("unit_factor", 1)
+                        storage_usage[component_name] = round(space_used_bytes * unit_factor, 2)
+                    elif "inode" in component_name.lower() or "file" in component_name.lower():
+                        # Inodes typically don't need conversion
+                        unit_factor = component_config.get("unit_factor", 1)
+                        storage_usage[component_name] = round(inodes_used * unit_factor, 2)
+
+                usage_report[resource_id] = {"TOTAL_ACCOUNT_USAGE": storage_usage}
+
+            logger.info(
+                "Successfully retrieved storage usage for %d resources",
+                len(usage_report),
+            )
+
+            return usage_report
+
+        except Exception:
+            logger.exception("Failed to get storage usage report from CSCS-DWDI")
+            raise
+
+    # Methods not implemented for reporting-only backend
+    def get_account(self, account_name: str) -> Optional[dict[str, Any]]:
+        """Not implemented for reporting-only backend."""
+        msg = "CSCS-DWDI storage backend is reporting-only and does not support account management"
+        raise NotImplementedError(msg)
+
+    def create_account(self, account_data: dict) -> bool:
+        """Not implemented for reporting-only backend."""
+        msg = "CSCS-DWDI storage backend is reporting-only and does not support account creation"
+        raise NotImplementedError(msg)
+
+    def delete_account(self, account_name: str) -> bool:
+        """Not implemented for reporting-only backend."""
+        msg = "CSCS-DWDI storage backend is reporting-only and does not support account deletion"
+        raise NotImplementedError(msg)
+
+    def update_account_limit_deposit(
+        self,
+        account_name: str,
+        component_type: str,
+        component_amount: float,
+        offering_component_data: dict,
+    ) -> bool:
+        """Not implemented for reporting-only backend."""
+        msg = "CSCS-DWDI storage backend is reporting-only and does not support limit updates"
+        raise NotImplementedError(msg)
+
+    def reset_account_limit_deposit(
+        self,
+        account_name: str,
+        component_type: str,
+        offering_component_data: dict,
+    ) -> bool:
+        """Not implemented for reporting-only backend."""
+        msg = "CSCS-DWDI storage backend is reporting-only and does not support limit resets"
+        raise NotImplementedError(msg)
+
+    def add_account_users(self, account_name: str, user_backend_ids: list[str]) -> bool:
+        """Not implemented for reporting-only backend."""
+        msg = "CSCS-DWDI storage backend is reporting-only and does not support user management"
+        raise NotImplementedError(msg)
+
+    def delete_account_users(self, account_name: str, user_backend_ids: list[str]) -> bool:
+        """Not implemented for reporting-only backend."""
+        msg = "CSCS-DWDI storage backend is reporting-only and does not support user management"
+        raise NotImplementedError(msg)
+
+    def list_accounts(self) -> list[dict[str, Any]]:
+        """Not implemented for reporting-only backend."""
+        msg = "CSCS-DWDI storage backend is reporting-only and does not support account listing"
+        raise NotImplementedError(msg)
+
+    def set_resource_limits(self, resource_backend_id: str, limits: dict[str, int]) -> None:
+        """Not implemented for reporting-only backend."""
+        msg = "CSCS-DWDI storage backend is reporting-only and does not support resource limits"
+        raise NotImplementedError(msg)
+
+    def diagnostics(self) -> bool:
+        """Get diagnostic information for the backend."""
+        logger.info(
+            "CSCS-DWDI Storage Backend Diagnostics - Type: %s, API: %s, "
+            "Filesystem: %s, DataType: %s, Components: %s, Ping: %s",
+            self.backend_type,
+            self.api_url,
+            self.filesystem,
+            self.data_type,
+            list(self.backend_components.keys()),
+            self.ping(),
+        )
+        return self.ping()
+
+    def get_resource_metadata(self, resource_backend_id: str) -> dict[str, Any]:
+        """Not implemented for reporting-only backend."""
+        msg = "CSCS-DWDI storage backend is reporting-only and does not support resource metadata"
+        raise NotImplementedError(msg)
+
+    def list_components(self) -> list[str]:
+        """List configured components for this backend."""
+        return list(self.backend_components.keys())
+
+    def _collect_resource_limits(
+        self, waldur_resource: WaldurResource
+    ) -> tuple[dict[str, int], dict[str, int]]:
+        """Not implemented for reporting-only backend."""
+        msg = "CSCS-DWDI storage backend is reporting-only and does not support resource limits"
+        raise NotImplementedError(msg)
+
+    def _pre_create_resource(
+        self, waldur_resource: WaldurResource, user_context: Optional[dict] = None
+    ) -> None:
+        """Not implemented for reporting-only backend."""
+        msg = "CSCS-DWDI storage backend is reporting-only and does not support resource creation"
+        raise NotImplementedError(msg)
+
+    def pause_resource(self, resource_backend_id: str) -> bool:
+        """Not implemented for reporting-only backend."""
+        msg = "CSCS-DWDI storage backend is reporting-only and does not support resource pausing"
+        raise NotImplementedError(msg)
+
+    def restore_resource(self, resource_backend_id: str) -> bool:
+        """Not implemented for reporting-only backend."""
+        msg = (
+            "CSCS-DWDI storage backend is reporting-only and does not support resource restoration"
+        )
+        raise NotImplementedError(msg)
+
+    def downscale_resource(self, resource_backend_id: str) -> bool:
+        """Not implemented for reporting-only backend."""
+        msg = (
+            "CSCS-DWDI storage backend is reporting-only and does not support resource downscaling"
+        )
+        raise NotImplementedError(msg)

waldur_site_agent_cscs_dwdi/client.py

@@ -21,6 +21,7 @@ class CSCSDWDIClient:
         client_secret: str,
         oidc_token_url: Optional[str] = None,
         oidc_scope: Optional[str] = None,
+        socks_proxy: Optional[str] = None,
     ) -> None:
         """Initialize CSCS-DWDI client.
 
@@ -30,12 +31,14 @@ class CSCSDWDIClient:
             client_secret: OIDC client secret for authentication
             oidc_token_url: OIDC token endpoint URL (required for authentication)
             oidc_scope: OIDC scope to request (optional)
+            socks_proxy: SOCKS proxy URL (e.g., "socks5://localhost:12345")
         """
         self.api_url = api_url.rstrip("/")
         self.client_id = client_id
         self.client_secret = client_secret
         self.oidc_token_url = oidc_token_url
         self.oidc_scope = oidc_scope or "openid"
+        self.socks_proxy = socks_proxy
         self._token: Optional[str] = None
         self._token_expires_at: Optional[datetime] = None
 
@@ -91,10 +94,14 @@ class CSCSDWDIClient:
 
         headers = {"Content-Type": "application/x-www-form-urlencoded"}
 
-        with httpx.Client() as client:
-            response = client.post(
-                self.oidc_token_url, data=token_data, headers=headers, timeout=30.0
-            )
+        # Configure httpx client with SOCKS proxy if specified
+        client_args: dict[str, Any] = {"timeout": 30.0}
+        if self.socks_proxy:
+            client_args["proxy"] = self.socks_proxy
+            logger.debug("Using SOCKS proxy for token acquisition: %s", self.socks_proxy)
+
+        with httpx.Client(**client_args) as client:
+            response = client.post(self.oidc_token_url, data=token_data, headers=headers)
             response.raise_for_status()
             token_response = response.json()
 
@@ -160,8 +167,14 @@ class CSCSDWDIClient:
             to_month,
         )
 
-        with httpx.Client() as client:
-            response = client.get(url, params=params, headers=headers, timeout=30.0)
+        # Configure httpx client with SOCKS proxy if specified
+        client_args: dict[str, Any] = {"timeout": 30.0}
+        if self.socks_proxy:
+            client_args["proxy"] = self.socks_proxy
+            logger.debug("Using SOCKS proxy for API request: %s", self.socks_proxy)
+
+        with httpx.Client(**client_args) as client:
+            response = client.get(url, params=params, headers=headers)
             response.raise_for_status()
             return response.json()
 
@@ -207,13 +220,139 @@ class CSCSDWDIClient:
             to_day,
         )
 
-        with httpx.Client() as client:
-            response = client.get(url, params=params, headers=headers, timeout=30.0)
+        # Configure httpx client with SOCKS proxy if specified
+        client_args: dict[str, Any] = {"timeout": 30.0}
+        if self.socks_proxy:
+            client_args["proxy"] = self.socks_proxy
+            logger.debug("Using SOCKS proxy for API request: %s", self.socks_proxy)
+
+        with httpx.Client(**client_args) as client:
+            response = client.get(url, params=params, headers=headers)
+            response.raise_for_status()
+            return response.json()
+
+    def get_storage_usage_for_month(
+        self,
+        paths: list[str],
+        tenant: Optional[str],
+        filesystem: str,
+        data_type: str,
+        exact_month: str,
+    ) -> dict[str, Any]:
+        """Get storage usage data for specified paths for a month.
+
+        Args:
+            paths: List of storage paths to query
+            tenant: Tenant identifier (optional)
+            filesystem: Filesystem name (e.g., "lustre")
+            data_type: Data type (e.g., "projects")
+            exact_month: Month in YYYY-MM format
+
+        Returns:
+            API response with storage usage data
+
+        Raises:
+            httpx.HTTPError: If API request fails
+        """
+        token = self._get_auth_token()
+
+        params: dict[str, Any] = {
+            "exact-month": exact_month,
+        }
+
+        # Add optional parameters
+        if paths:
+            params["paths"] = paths
+        if tenant:
+            params["tenant"] = tenant
+        if filesystem:
+            params["filesystem"] = filesystem
+        if data_type:
+            params["data_type"] = data_type
+
+        headers = {"Authorization": f"Bearer {token}"}
+
+        url = f"{self.api_url}/api/v1/storage/usage-month/filesystem_name/data_type"
+
+        logger.debug(
+            "Fetching storage usage for paths %s for month %s",
+            paths,
+            exact_month,
+        )
+
+        # Configure httpx client with SOCKS proxy if specified
+        client_args: dict[str, Any] = {"timeout": 30.0}
+        if self.socks_proxy:
+            client_args["proxy"] = self.socks_proxy
+            logger.debug("Using SOCKS proxy for API request: %s", self.socks_proxy)
+
+        with httpx.Client(**client_args) as client:
+            response = client.get(url, params=params, headers=headers)
+            response.raise_for_status()
+            return response.json()
+
+    def get_storage_usage_for_day(
+        self,
+        paths: list[str],
+        tenant: Optional[str],
+        filesystem: str,
+        data_type: str,
+        exact_date: date,
+    ) -> dict[str, Any]:
+        """Get storage usage data for specified paths for a specific day.
+
+        Args:
+            paths: List of storage paths to query
+            tenant: Tenant identifier (optional)
+            filesystem: Filesystem name (e.g., "lustre")
+            data_type: Data type (e.g., "projects")
+            exact_date: Specific date
+
+        Returns:
+            API response with storage usage data
+
+        Raises:
+            httpx.HTTPError: If API request fails
+        """
+        token = self._get_auth_token()
+
+        params: dict[str, Any] = {
+            "exact-date": exact_date.strftime("%Y-%m-%d"),
+        }
+
+        # Add optional parameters
+        if paths:
+            params["paths"] = paths
+        if tenant:
+            params["tenant"] = tenant
+        if filesystem:
+            params["filesystem"] = filesystem
+        if data_type:
+            params["data_type"] = data_type
+
+        headers = {"Authorization": f"Bearer {token}"}
+
+        url = f"{self.api_url}/api/v1/storage/usage-day/filesystem_name/data_type"
+
+        logger.debug(
+            "Fetching storage usage for paths %s for date %s",
+            paths,
+            exact_date,
+        )
+
+        # Configure httpx client with SOCKS proxy if specified
+        client_args: dict[str, Any] = {"timeout": 30.0}
+        if self.socks_proxy:
+            client_args["proxy"] = self.socks_proxy
+            logger.debug("Using SOCKS proxy for API request: %s", self.socks_proxy)
+
+        with httpx.Client(**client_args) as client:
+            response = client.get(url, params=params, headers=headers)
             response.raise_for_status()
             return response.json()
 
     def ping(self) -> bool:
-        """Check if CSCS-DWDI API is accessible.
+        """Check if CSCS-DWDI compute API is accessible.
 
         Returns:
             True if API is accessible, False otherwise
@@ -231,9 +370,46 @@ class CSCSDWDIClient:
 
             url = f"{self.api_url}/api/v1/compute/usage-day-multiaccount"
 
-            with httpx.Client() as client:
-                response = client.get(url, params=params, headers=headers, timeout=10.0)
+            # Configure httpx client with SOCKS proxy if specified
+            client_args: dict[str, Any] = {"timeout": 10.0}
+            if self.socks_proxy:
+                client_args["proxy"] = self.socks_proxy
+                logger.debug("Using SOCKS proxy for ping: %s", self.socks_proxy)
+
+            with httpx.Client(**client_args) as client:
+                response = client.get(url, params=params, headers=headers)
+                return response.status_code == HTTP_OK
+        except Exception:
+            logger.exception("Compute ping failed")
+            return False
+
+    def ping_storage(self) -> bool:
+        """Check if CSCS-DWDI storage API is accessible.
+
+        Returns:
+            True if API is accessible, False otherwise
+        """
+        try:
+            token = self._get_auth_token()
+            headers = {"Authorization": f"Bearer {token}"}
+
+            # Use a simple query to test connectivity
+            today = datetime.now(tz=timezone.utc).date()
+            params = {
+                "exact-date": today.strftime("%Y-%m-%d"),
+            }
+
+            url = f"{self.api_url}/api/v1/storage/usage-day/filesystem_name/data_type"
+
+            # Configure httpx client with SOCKS proxy if specified
+            client_args: dict[str, Any] = {"timeout": 10.0}
+            if self.socks_proxy:
+                client_args["proxy"] = self.socks_proxy
+                logger.debug("Using SOCKS proxy for storage ping: %s", self.socks_proxy)
+
+            with httpx.Client(**client_args) as client:
+                response = client.get(url, params=params, headers=headers)
                 return response.status_code == HTTP_OK
         except Exception:
-            logger.exception("Ping failed")
+            logger.exception("Storage ping failed")
             return False

waldur_site_agent_cscs_dwdi-0.7.3.dist-info/METADATA

@@ -0,0 +1,284 @@
+Metadata-Version: 2.4
+Name: waldur-site-agent-cscs-dwdi
+Version: 0.7.3
+Summary: CSCS-DWDI reporting plugin for Waldur Site Agent
+Author-email: OpenNode Team <info@opennodecloud.com>
+Requires-Python: <4,>=3.9
+Requires-Dist: httpx>=0.25.0
+Requires-Dist: waldur-site-agent>=0.7.0
+Description-Content-Type: text/markdown
+
+# CSCS-DWDI Plugin for Waldur Site Agent
+
+This plugin provides integration with the CSCS Data Warehouse Data Intelligence (DWDI) system to report both
+computational and storage usage data to Waldur.
+
+## Overview
+
+The plugin implements two separate backends to handle different types of accounting data:
+
+- **Compute Backend** (`cscs-dwdi-compute`): Reports CPU and node hour usage from HPC clusters
+- **Storage Backend** (`cscs-dwdi-storage`): Reports storage space and inode usage from filesystems
+
+## Backend Types
+
+### Compute Backend
+
+The compute backend queries the DWDI API for computational resource usage and reports:
+
+- Node hours consumed by accounts and users
+- CPU hours consumed by accounts and users
+- Account-level and user-level usage aggregation
+
+**API Endpoints Used:**
+
+- `/api/v1/compute/usage-month/account` - Monthly usage data
+- `/api/v1/compute/usage-day/account` - Daily usage data
+
+### Storage Backend
+
+The storage backend queries the DWDI API for storage resource usage and reports:
+
+- Storage space used (converted from bytes to configured units)
+- Inode (file count) usage
+- Path-based resource identification
+
+**API Endpoints Used:**
+
+- `/api/v1/storage/usage-month/filesystem_name/data_type` - Monthly storage usage
+- `/api/v1/storage/usage-day/filesystem_name/data_type` - Daily storage usage
+
+## Configuration
+
+### Compute Backend Configuration
+
+```yaml
+backend_type: "cscs-dwdi-compute"
+
+backend_settings:
+  cscs_dwdi_api_url: "https://dwdi.cscs.ch"
+  cscs_dwdi_client_id: "your_oidc_client_id"
+  cscs_dwdi_client_secret: "your_oidc_client_secret"
+  cscs_dwdi_oidc_token_url: "https://auth.cscs.ch/realms/cscs/protocol/openid-connect/token"
+  cscs_dwdi_oidc_scope: "openid"  # Optional
+
+backend_components:
+  nodeHours:
+    measured_unit: "node-hours"
+    unit_factor: 1
+    accounting_type: "usage"
+    label: "Node Hours"
+
+  cpuHours:
+    measured_unit: "cpu-hours"
+    unit_factor: 1
+    accounting_type: "usage"
+    label: "CPU Hours"
+```
+
+### Storage Backend Configuration
+
+```yaml
+backend_type: "cscs-dwdi-storage"
+
+backend_settings:
+  cscs_dwdi_api_url: "https://dwdi.cscs.ch"
+  cscs_dwdi_client_id: "your_oidc_client_id"
+  cscs_dwdi_client_secret: "your_oidc_client_secret"
+  cscs_dwdi_oidc_token_url: "https://auth.cscs.ch/realms/cscs/protocol/openid-connect/token"
+
+  # Storage-specific settings
+  storage_filesystem: "lustre"
+  storage_data_type: "projects"
+  storage_tenant: "cscs"  # Optional
+
+  # Map Waldur resource IDs to storage paths
+  storage_path_mapping:
+    "project_123": "/store/projects/proj123"
+    "project_456": "/store/projects/proj456"
+
+backend_components:
+  storage_space:
+    measured_unit: "GB"
+    unit_factor: 0.000000001  # Convert bytes to GB
+    accounting_type: "usage"
+    label: "Storage Space (GB)"
+
+  storage_inodes:
+    measured_unit: "count"
+    unit_factor: 1
+    accounting_type: "usage"
+    label: "File Count"
+```
+
+## Authentication
+
+Both backends use OIDC client credentials flow for authentication with the DWDI API. You need:
+
+- `cscs_dwdi_client_id`: OIDC client identifier
+- `cscs_dwdi_client_secret`: OIDC client secret
+- `cscs_dwdi_oidc_token_url`: OIDC token endpoint URL
+- `cscs_dwdi_oidc_scope`: OIDC scope (optional, defaults to "openid")
+
+## SOCKS Proxy Support
+
+Both backends support SOCKS proxy for network connectivity. This is useful when the DWDI API is only accessible
+through a proxy or jump host.
+
+### SOCKS Proxy Configuration
+
+Add the SOCKS proxy setting to your backend configuration:
+
+```yaml
+backend_settings:
+  # ... other settings ...
+  socks_proxy: "socks5://localhost:12345"  # SOCKS5 proxy URL
+```
+
+### Supported Proxy Types
+
+- **SOCKS5**: `socks5://hostname:port`
+- **SOCKS4**: `socks4://hostname:port`
+- **HTTP**: `http://hostname:port`
+
+### Usage Examples
+
+**SSH Tunnel with SOCKS5:**
+
+```bash
+# Create SSH tunnel to jump host
+ssh -D 12345 -N user@jumphost.cscs.ch
+
+# Configure backend to use tunnel
+backend_settings:
+  socks_proxy: "socks5://localhost:12345"
+```
+
+**HTTP Proxy:**
+
+```yaml
+backend_settings:
+  socks_proxy: "http://proxy.cscs.ch:8080"
+```
+
+## Resource Identification
+
+### Compute Resources
+
+For compute resources, the system uses account names as returned by the DWDI API. The Waldur resource
+`backend_id` should match the account name in the cluster accounting system.
+
+### Storage Resources
+
+For storage resources, there are two options:
+
+1. **Direct Path Usage**: Set the Waldur resource `backend_id` to the actual filesystem path
+2. **Path Mapping**: Use the `storage_path_mapping` setting to map resource IDs to paths
+
+## Usage Reporting
+
+Both backends are read-only and designed for usage reporting. They implement the `_get_usage_report()` method
+but do not support:
+
+- Account creation/deletion
+- Resource management
+- User management
+- Limit setting
+
+## Example Configurations
+
+See the `examples/` directory for complete configuration examples:
+
+- `cscs-dwdi-compute-config.yaml` - Compute backend only
+- `cscs-dwdi-storage-config.yaml` - Storage backend only
+- `cscs-dwdi-combined-config.yaml` - Both backends in one configuration
+
+## Installation
+
+The plugin is automatically discovered when the waldur-site-agent-cscs-dwdi package is installed alongside waldur-site-agent.
+
+```bash
+# Install all workspace packages including cscs-dwdi plugin
+uv sync --all-packages
+```
+
+## Testing
+
+Run the test suite:
+
+```bash
+uv run pytest plugins/cscs-dwdi/tests/
+```
+
+## API Compatibility
+
+This plugin is compatible with DWDI API version 1 (`/api/v1/`). It requires the following API endpoints to be available:
+
+**Compute API:**
+
+- `/api/v1/compute/usage-month/account`
+- `/api/v1/compute/usage-day/account`
+
+**Storage API:**
+
+- `/api/v1/storage/usage-month/filesystem_name/data_type`
+- `/api/v1/storage/usage-day/filesystem_name/data_type`
+
+## Troubleshooting
+
+### Authentication Issues
+
+- Verify OIDC client credentials are correct
+- Check that the token endpoint URL is accessible
+- Ensure the client has appropriate scopes
+
+### Storage Backend Issues
+
+- Verify `storage_filesystem` and `storage_data_type` match available values in DWDI
+- Check `storage_path_mapping` if using custom resource IDs
+- Ensure storage paths exist in the DWDI system
+
+### Connection Issues
+
+- Use the `ping()` method to test API connectivity
+- Check network connectivity to the DWDI API endpoint
+- Verify SSL/TLS configuration
+- If behind a firewall, configure SOCKS proxy (`socks_proxy` setting)
+
+### Proxy Issues
+
+- Verify proxy server is running and accessible
+- Check proxy authentication if required
+- Test proxy connectivity manually: `curl --proxy socks5://localhost:12345 https://dwdi.cscs.ch`
+- Ensure proxy supports the required protocol (SOCKS4/5, HTTP)
+
+## Development
+
+### Project Structure
+
+```text
+plugins/cscs-dwdi/
+├── pyproject.toml                           # Plugin configuration
+├── README.md                               # This documentation
+├── examples/                               # Configuration examples
+├── waldur_site_agent_cscs_dwdi/
+│   ├── __init__.py                         # Package init
+│   ├── backend.py                          # Backend implementations
+│   └── client.py                          # CSCS-DWDI API client
+└── tests/
+    └── test_cscs_dwdi.py                  # Plugin tests
+```
+
+### Key Classes
+
+- **`CSCSDWDIComputeBackend`**: Compute usage reporting backend
+- **`CSCSDWDIStorageBackend`**: Storage usage reporting backend
+- **`CSCSDWDIClient`**: HTTP client for CSCS-DWDI API communication
+
+### Extension Points
+
+To extend the plugin:
+
+1. **Additional Endpoints**: Modify `CSCSDWDIClient` to support more API endpoints
+2. **Authentication Methods**: Update authentication logic in `client.py`
+3. **Data Processing**: Enhance response processing methods for additional data formats

waldur_site_agent_cscs_dwdi-0.7.3.dist-info/RECORD

@@ -0,0 +1,7 @@
+waldur_site_agent_cscs_dwdi/__init__.py,sha256=OHO1yF5NTGt0otI-GollR_ppPXP--aUZRCdaT5-8IWw,56
+waldur_site_agent_cscs_dwdi/backend.py,sha256=2A53fAvjwnCaxsuA42bdqxHEp0DrwN1F4MYoB1ryCuQ,26846
+waldur_site_agent_cscs_dwdi/client.py,sha256=sqCBpioe5d6sJwp-iVGUL3AUIOxkXqTEA6u6mmd0F_Y,13782
+waldur_site_agent_cscs_dwdi-0.7.3.dist-info/METADATA,sha256=_8O14KeIMt7y4UH3-RboadKXx15txwC9R6VwbpGetVU,8131
+waldur_site_agent_cscs_dwdi-0.7.3.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+waldur_site_agent_cscs_dwdi-0.7.3.dist-info/entry_points.txt,sha256=gbp1thULdYQN4leLZeM8TBoruPSGQEKQxlQ0fg8u3Ug,187
+waldur_site_agent_cscs_dwdi-0.7.3.dist-info/RECORD,,

waldur_site_agent_cscs_dwdi-0.7.3.dist-info/entry_points.txt

@@ -0,0 +1,3 @@
+[waldur_site_agent.backends]
+cscs-dwdi-compute = waldur_site_agent_cscs_dwdi.backend:CSCSDWDIComputeBackend
+cscs-dwdi-storage = waldur_site_agent_cscs_dwdi.backend:CSCSDWDIStorageBackend

waldur_site_agent_cscs_dwdi-0.1.0.dist-info/METADATA

@@ -1,240 +0,0 @@
-Metadata-Version: 2.4
-Name: waldur-site-agent-cscs-dwdi
-Version: 0.1.0
-Summary: CSCS-DWDI reporting plugin for Waldur Site Agent
-Author-email: OpenNode Team <info@opennodecloud.com>
-Requires-Python: <4,>=3.9
-Requires-Dist: httpx>=0.25.0
-Requires-Dist: waldur-site-agent==0.1.0
-Description-Content-Type: text/markdown
-
-# CSCS-DWDI Plugin for Waldur Site Agent
-
-This plugin provides reporting functionality for Waldur Site Agent by integrating with the CSCS-DWDI
-(Data Warehouse and Data Intelligence) API.
-
-## Overview
-
-The CSCS-DWDI plugin is a **reporting-only backend** that fetches compute usage data from the CSCS-DWDI
-service and reports it to Waldur. It supports node-hour usage tracking for multiple accounts and users.
-
-## Features
-
-- **Monthly Usage Reporting**: Fetches usage data for the current month
-- **Multi-Account Support**: Reports usage for multiple accounts in a single API call
-- **Per-User Usage**: Breaks down usage by individual users within each account
-- **OIDC Authentication**: Uses OAuth2/OIDC for secure API access
-- **Automatic Aggregation**: Combines usage across different clusters and time periods
-
-## Configuration
-
-Add the following configuration to your Waldur Site Agent offering:
-
-```yaml
-offerings:
-  - name: "CSCS HPC Offering"
-    reporting_backend: "cscs-dwdi"
-    backend_settings:
-      cscs_dwdi_api_url: "https://dwdi-api.cscs.ch"
-      cscs_dwdi_client_id: "your-oidc-client-id"
-      cscs_dwdi_client_secret: "your-oidc-client-secret"
-      # Optional OIDC configuration (for production use)
-      cscs_dwdi_oidc_token_url: "https://identity.cscs.ch/realms/cscs/protocol/openid-connect/token"
-      cscs_dwdi_oidc_scope: "cscs-dwdi:read"
-
-    backend_components:
-      nodeHours:
-        measured_unit: "node-hours"
-        unit_factor: 1
-        accounting_type: "usage"
-        label: "Node Hours"
-      storage:
-        measured_unit: "TB"
-        unit_factor: 1
-        accounting_type: "usage"
-        label: "Storage Usage"
-```
-
-### Configuration Parameters
-
-#### Backend Settings
-
-| Parameter | Required | Description |
-|-----------|----------|-------------|
-| `cscs_dwdi_api_url` | Yes | Base URL for the CSCS-DWDI API service |
-| `cscs_dwdi_client_id` | Yes | OIDC client ID for authentication |
-| `cscs_dwdi_client_secret` | Yes | OIDC client secret for authentication |
-| `cscs_dwdi_oidc_token_url` | Yes | OIDC token endpoint URL (required for authentication) |
-| `cscs_dwdi_oidc_scope` | No | OIDC scope to request (defaults to "openid") |
-
-#### Backend Components
-
-Components must match the field names returned by the CSCS-DWDI API. For example:
-
-- `nodeHours` - Maps to the `nodeHours` field in API responses
-- `storage` - Maps to the `storage` field in API responses (if available)
-- `gpuHours` - Maps to the `gpuHours` field in API responses (if available)
-
-Each component supports:
-
-| Parameter | Description |
-|-----------|-------------|
-| `measured_unit` | Unit for display in Waldur (e.g., "node-hours", "TB") |
-| `unit_factor` | Conversion factor from API units to measured units |
-| `accounting_type` | Either "usage" for actual usage or "limit" for quotas |
-| `label` | Display label in Waldur interface |
-
-## Usage Data Format
-
-The plugin reports usage for all configured components:
-
-- **Component Types**: Configurable (e.g., `nodeHours`, `storage`, `gpuHours`)
-- **Units**: Based on API response and `unit_factor` configuration
-- **Granularity**: Monthly reporting with current month data
-- **User Attribution**: Individual user usage within each account
-- **Aggregation**: Automatically aggregates across clusters and time periods
-
-## API Integration
-
-The plugin uses the CSCS-DWDI API endpoints:
-
-- `GET /api/v1/compute/usage-month-multiaccount` - Primary endpoint for monthly usage data
-- Authentication via OIDC Bearer tokens
-
-### Authentication
-
-The plugin uses OAuth2/OIDC authentication with the following requirements:
-
-- Requires `cscs_dwdi_oidc_token_url` in backend settings
-- Uses OAuth2 `client_credentials` grant flow
-- Automatically handles token caching and renewal
-- Includes 5-minute safety margin for token expiry
-- Fails with proper error logging if OIDC configuration is missing
-
-### Data Processing
-
-1. **Account Filtering**: Only reports on accounts that match Waldur resource backend IDs
-2. **User Aggregation**: Combines usage for the same user across different dates and clusters
-3. **Time Range**: Automatically queries from the first day of the current month to today
-4. **Precision**: Rounds node-hours to 2 decimal places
-
-## Installation
-
-This plugin is part of the Waldur Site Agent workspace. To install:
-
-```bash
-# Install all workspace packages including cscs-dwdi plugin
-uv sync --all-packages
-
-# Install specific plugin for development
-uv sync --extra cscs-dwdi
-```
-
-## Testing
-
-Run the plugin tests:
-
-```bash
-# Run CSCS-DWDI plugin tests
-uv run pytest plugins/cscs-dwdi/tests/
-
-# Run with coverage
-uv run pytest plugins/cscs-dwdi/tests/ --cov=waldur_site_agent_cscs_dwdi
-```
-
-## Limitations
-
-This is a **reporting-only backend** that does not support:
-
-- Account creation or deletion
-- User management
-- Resource limit management
-- Order processing
-- Membership synchronization
-
-For these operations, use a different backend (e.g., SLURM) in combination with the CSCS-DWDI reporting backend:
-
-```yaml
-offerings:
-  - name: "Mixed Backend Offering"
-    order_processing_backend: "slurm"       # Use SLURM for orders
-    reporting_backend: "cscs-dwdi"          # Use CSCS-DWDI for reporting
-    membership_sync_backend: "slurm"        # Use SLURM for membership
-```
-
-## Error Handling
-
-The plugin includes comprehensive error handling:
-
-- **API Connectivity**: Ping checks verify API availability
-- **Authentication**: Token refresh and error handling
-- **Data Validation**: Validates API responses and filters invalid data
-- **Retry Logic**: Uses the framework's built-in retry mechanisms
-
-## Development
-
-### Project Structure
-
-```text
-plugins/cscs-dwdi/
-├── pyproject.toml                           # Plugin configuration
-├── README.md                               # This documentation
-├── waldur_site_agent_cscs_dwdi/
-│   ├── __init__.py                         # Package init
-│   ├── backend.py                          # Main backend implementation
-│   └── client.py                          # CSCS-DWDI API client
-└── tests/
-    └── test_cscs_dwdi.py                  # Plugin tests
-```
-
-### Key Classes
-
-- **`CSCSDWDIBackend`**: Main backend class implementing reporting functionality
-- **`CSCSDWDIClient`**: HTTP client for CSCS-DWDI API communication
-
-### Extension Points
-
-To extend the plugin:
-
-1. **Additional Endpoints**: Modify `CSCSDWDIClient` to support more API endpoints
-2. **Authentication Methods**: Update authentication logic in `client.py`
-3. **Data Processing**: Enhance `_process_api_response()` for additional data formats
-
-## Troubleshooting
-
-### Common Issues
-
-#### Authentication Failures
-
-- Verify OIDC client credentials
-- Check API URL configuration
-- Ensure proper token scopes
-
-#### Missing Usage Data
-
-- Verify account names match between Waldur and CSCS-DWDI
-- Check date ranges and API response format
-- Review API rate limits and quotas
-
-#### Network Connectivity
-
-- Test API connectivity with ping functionality
-- Verify network access from agent deployment environment
-- Check firewall and proxy settings
-
-### Debugging
-
-Enable debug logging for detailed API interactions:
-
-```python
-import logging
-logging.getLogger('waldur_site_agent_cscs_dwdi').setLevel(logging.DEBUG)
-```
-
-## Support
-
-For issues and questions:
-
-- Check the [Waldur Site Agent documentation](../../docs/)
-- Review plugin test cases for usage examples
-- Create issues in the project repository

waldur_site_agent_cscs_dwdi-0.1.0.dist-info/RECORD

@@ -1,7 +0,0 @@
-waldur_site_agent_cscs_dwdi/__init__.py,sha256=OHO1yF5NTGt0otI-GollR_ppPXP--aUZRCdaT5-8IWw,56
-waldur_site_agent_cscs_dwdi/backend.py,sha256=V5T9H5b9utxcD73TmACvrc4dZm-GEORLDI0C0tUD_7s,14251
-waldur_site_agent_cscs_dwdi/client.py,sha256=aAWNqrA0oBknWge_TQIU4u8PQi5YdIZtQTZ2XPREznQ,7555
-waldur_site_agent_cscs_dwdi-0.1.0.dist-info/METADATA,sha256=nTtebzVzy0mWYqZejNICgI5hlHFDxbsPAk0woaSABHM,7772
-waldur_site_agent_cscs_dwdi-0.1.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-waldur_site_agent_cscs_dwdi-0.1.0.dist-info/entry_points.txt,sha256=U3odcX7B4NmT9a98ov4uVxlng3JqiCRb5Ux3h4H_ZFE,93
-waldur_site_agent_cscs_dwdi-0.1.0.dist-info/RECORD,,

waldur_site_agent_cscs_dwdi-0.1.0.dist-info/entry_points.txt

@@ -1,2 +0,0 @@
-[waldur_site_agent.backends]
-cscs-dwdi = waldur_site_agent_cscs_dwdi.backend:CSCSDWDIBackend