sws-api-client 1.3.0__tar.gz → 1.3.2__tar.gz

{sws_api_client-1.3.0 → sws_api_client-1.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sws_api_client
-Version: 1.3.0
+Version: 1.3.2
 Summary: 
 Author: Matteo
 Author-email: matteo.terrinoni@fao.org

{sws_api_client-1.3.0 → sws_api_client-1.3.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "sws_api_client"
-version = "1.3.0"
+version = "1.3.2"
 description = ""
 authors = ["Matteo <matteo.terrinoni@fao.org>", "Daniele <daniele.mansillo@fao.org>"]
 readme = "README.md"

sws_api_client-1.3.2/sws_api_client/codelist.py

@@ -0,0 +1,69 @@
+"""Codelists Module for SWS API.
+
+This module provides functionality for managing codelists and retrieving
+codelist information through the SWS API client.
+"""
+
+import logging
+from time import sleep
+import os
+from pydantic import BaseModel, Extra
+from typing import List, Optional, Dict
+from sws_api_client.generic_models import Code, Multilanguage
+from sws_api_client.sws_api_client import SwsApiClient
+
+logger = logging.getLogger(__name__)
+
+class CodelistModel(BaseModel, extra="allow"):
+    """Model representing a codelist's metadata.
+
+    Attributes:
+        id (str): Unique identifier of the codelist
+        label (Multilanguage): Multilanguage labels for the codelist
+        type (str): Type of the codelist
+    """
+    id: str
+    label: Multilanguage
+    type: str
+
+class Codelist(BaseModel, extra="allow"):
+    """Model representing a complete codelist with its codes.
+
+    Attributes:
+        model (CodelistModel): Metadata of the codelist
+        codes (List[Code]): List of codes belonging to this codelist
+    """
+    model: CodelistModel
+    codes: List[Code]
+
+class Codelists:
+    """Class for managing codelist operations through the SWS API.
+
+    This class provides methods for retrieving codelist information
+    and managing codelist data.
+
+    Args:
+        sws_client (SwsApiClient): An instance of the SWS API client
+    """
+
+    def __init__(self, sws_client: SwsApiClient) -> None:
+        """Initialize the Codelists manager with SWS client."""
+        self.sws_client = sws_client
+
+    def get_codelist(self, codelist_id: str, nocache: bool = False) -> Codelist:
+        """Retrieve a codelist by its ID.
+
+        Args:
+            codelist_id (str): The identifier of the codelist
+            nocache (bool, optional): Whether to bypass cache. Defaults to False.
+
+        Returns:
+            Codelist: The requested codelist with its codes
+
+        Raises:
+            HTTPError: If the codelist cannot be retrieved
+        """
+        url = f"/admin/reference/codelist/{codelist_id}?nocache={'true' if nocache else 'false'}"
+
+        response = self.sws_client.discoverable.get('is_api', url)
+        return Codelist(**response)

{sws_api_client-1.3.0 → sws_api_client-1.3.2}/sws_api_client/datasets.py

@@ -1,3 +1,9 @@
+"""Datasets Module for SWS API.
+
+This module provides functionality for managing datasets, including creation,
+cloning, data import/export, and dimension management through the SWS API client.
+"""
+
 import logging
 from time import sleep
 import os
@@ -14,6 +20,16 @@ from concurrent.futures import ThreadPoolExecutor, as_completed
 logger = logging.getLogger(__name__)
 
 class Lifecycle(BaseModel):
+    """Model representing the lifecycle state of a dataset.
+
+    Attributes:
+        state (str): Current state of the dataset
+        type (str): Type of lifecycle
+        previousState (Optional[str]): Previous state if applicable
+        created (int): Creation timestamp
+        lastModified (int): Last modification timestamp
+        lastModifiedBy (str): User who last modified the dataset
+    """
     state: str
     type: str
     previousState: Optional[str]
@@ -22,14 +38,40 @@ class Lifecycle(BaseModel):
     lastModifiedBy: str
 
 class Domain(BaseModel):
+    """Model representing a domain.
+
+    Attributes:
+        id (str): Domain identifier
+        label (Multilanguage): Multilanguage labels
+        description (Dict): Domain description in multiple languages
+    """
     id: str
     label: Multilanguage
     description: Dict
 
 class Binding(BaseModel):
+    """Model representing column binding information.
+
+    Attributes:
+        joinColumn (str): Column used for joining
+    """
     joinColumn: str
 
 class Dimension(BaseModel):
+    """Model representing a dataset dimension.
+
+    Attributes:
+        id (str): Dimension identifier
+        label (Multilanguage): Multilanguage labels
+        description (Dict): Dimension description
+        sdmxName (str): SDMX name
+        codelist (str): Associated codelist
+        roots (List[str]): Root codes
+        binding (Binding): Column binding information
+        checkValidityPeriod (bool): Whether to check validity period
+        formulas (List): Associated formulas
+        type (str): Dimension type
+    """
     id: str
     label: Multilanguage
     description: Dict
@@ -98,13 +140,29 @@ class MappedCode(BaseModel):
     code: Code
     include: bool
 class Datasets:
+    """Class for managing dataset operations through the SWS API.
+
+    This class provides methods for creating, cloning, importing data,
+    and managing dataset dimensions.
+
+    Args:
+        sws_client (SwsApiClient): An instance of the SWS API client
+    """
 
     def __init__(self, sws_client: SwsApiClient) -> None:
+        """Initialize the Datasets manager with SWS client."""
         self.sws_client = sws_client
         self.codelists = Codelists(sws_client)
 
     def get_dataset_export_details(self, dataset_id: str) -> dict:
+        """Get export details for a dataset.
+
+        Args:
+            dataset_id (str): Dataset identifier
 
+        Returns:
+            dict: Dataset export information
+        """
         url = f"/dataset/{dataset_id}/info"
         params = {"extended": "true"}
 
@@ -113,14 +171,28 @@ class Datasets:
         return response
     
     def get_dataset_info(self, dataset_id: str) -> DataModel:
+        """Retrieve dataset information.
 
+        Args:
+            dataset_id (str): Dataset identifier
+
+        Returns:
+            DataModel: Complete dataset information model
+        """
         url = f"/admin/dataset/{dataset_id}"
 
         response = self.sws_client.discoverable.get('is_api', url)
         return DataModel(**response)
 
     def create_dataset(self, dataset: Dataset) -> DataModel:
+        """Create a new dataset.
+
+        Args:
+            dataset (Dataset): Dataset configuration
 
+        Returns:
+            DataModel: Created dataset model
+        """
         url = "/admin/dataset"
 
         response = self.sws_client.discoverable.post('is_api', url, data=dataset.model_dump())
@@ -128,14 +200,29 @@ class Datasets:
         return response
     
     def clone_dataset(self, dataset_id: str, new_id: str) -> DataModel:
+        """Clone an existing dataset with a new identifier.
+
+        Args:
+            dataset_id (str): Source dataset identifier
+            new_id (str): New dataset identifier
 
+        Returns:
+            DataModel: Cloned dataset model
+        """
         dataset = self.get_dataset_info(dataset_id)
         dataset.dataset.id = new_id
         new_dataset = self.create_dataset(dataset.dataset)
         return new_dataset
     
     def get_job_status(self, jobId: str) -> dict:
+        """Check status of a job.
 
+        Args:
+            jobId (str): Job identifier
+
+        Returns:
+            dict: Job status information with result and success flags
+        """
         url = f"/job/status/{jobId}"
 
         response = self.sws_client.discoverable.get('is_api', url)
@@ -145,10 +232,29 @@ class Datasets:
         return dict(result=result, success=success)
 
     def import_data(self, dataset_id: str, file_path, sessionId = None, zip=False ) -> bool:
+        """Import data into a dataset.
+
+        Args:
+            dataset_id (str): Target dataset identifier
+            file_path: Path to data file
+            sessionId (Optional[int]): Session identifier
+            zip (bool): Whether to zip the file
+
+        Returns:
+            bool: True if import successful
+        """
         return self.import_data_chunk(dataset_id, file_path, sessionId)
         
 
     def get_dataset_dimension_codes(self, dataset_id: str) -> Dict[str, List[Code]]:
+        """Get codes for all dimensions in a dataset.
+
+        Args:
+            dataset_id (str): Dataset identifier
+
+        Returns:
+            Dict[str, List[Code]]: Dictionary mapping dimension IDs to their codes
+        """
         dataset_info = self.get_dataset_info(dataset_id)
         dimensions = dataset_info.dataset.dimensions.dimensions
 
@@ -188,7 +294,19 @@ class Datasets:
         return dimensions_codes
 
     def import_data_chunk(self, dataset_id: str, file_path: str, sessionId: Optional[int] = None) -> dict:
-        """Helper function to import a single data chunk, zipping files over 10MB automatically."""
+        """Import a single chunk of data, automatically handling file compression.
+
+        Args:
+            dataset_id (str): Target dataset identifier
+            file_path (str): Path to data file
+            sessionId (Optional[int]): Session identifier
+
+        Returns:
+            dict: Import job result
+
+        Raises:
+            OSError: If file operations fail
+        """
         # Check file size and zip if greater than 10MB
         zip_file_path = None
         if os.path.getsize(file_path) > 10 * 1024 * 1024:  # 10MB in bytes
@@ -232,7 +350,14 @@ class Datasets:
         return self.get_job_result(job_id)
 
     def get_job_result(self, job_id: str) -> bool:
-        """Check job status until it's completed."""
+        """Poll for job completion and return result.
+
+        Args:
+            job_id (str): Job identifier
+
+        Returns:
+            bool: True if job completed successfully
+        """
         while True:
             logger.debug(f"Checking job status for job ID {job_id}")
             job_status = self.get_job_status(job_id)
@@ -241,7 +366,19 @@ class Datasets:
             sleep(5)
 
     def chunk_csv_file(self, file_path: str, chunk_size: int) -> List[str]:
-        """Splits the CSV file into smaller chunks while preserving leading zeros for all columns except 'value'."""
+        """Split CSV file into smaller chunks while preserving data format.
+
+        Args:
+            file_path (str): Path to CSV file
+            chunk_size (int): Number of rows per chunk
+
+        Returns:
+            List[str]: Paths to chunk files
+
+        Raises:
+            OSError: If file operations fail
+            pd.errors.EmptyDataError: If CSV file is empty
+        """
         temp_files = []
         
         # Load the data in chunks and process each chunk
@@ -257,8 +394,19 @@ class Datasets:
         return temp_files
 
     def import_data_concurrent(self, dataset_id: str, file_path: str, sessionId: Optional[int] = None, chunk_size: int = 10000, max_workers: int = 5) -> None:
-        """Splits the CSV and imports chunks concurrently with limited workers, showing progress in the logs."""
-        
+        """Import data in parallel using multiple threads.
+
+        Args:
+            dataset_id (str): Target dataset identifier
+            file_path (str): Path to data file
+            sessionId (Optional[int]): Session identifier
+            chunk_size (int): Rows per chunk
+            max_workers (int): Maximum number of concurrent imports
+
+        Raises:
+            OSError: If file operations fail
+            Exception: If import operations fail
+        """
         # Step 1: Split the file into chunks
         chunk_files = self.chunk_csv_file(file_path, chunk_size)
         total_chunks = len(chunk_files)  # Total number of chunks for progress tracking
@@ -297,4 +445,3 @@ class Datasets:
         logger.info("Importing chunks: completed")
 
 
-        

sws_api_client-1.3.2/sws_api_client/datatable.py

@@ -0,0 +1,162 @@
+"""Datatables Module for SWS API.
+
+This module provides functionality for managing datatables, including retrieving
+information, exporting data, and handling CSV paths through the SWS API client.
+"""
+
+import logging
+from pydantic import BaseModel, Field, HttpUrl
+from typing import List, Optional, Dict, Any
+from sws_api_client.sws_api_client import SwsApiClient
+logger = logging.getLogger(__name__)
+from datetime import datetime, time
+from time import sleep
+
+class LabelModel(BaseModel):
+    """Model for multilingual labels.
+
+    Attributes:
+        en (str): English label text
+    """
+    en: str
+
+class DescriptionModel(BaseModel):
+    """Model for multilingual descriptions.
+
+    Attributes:
+        en (str): English description text
+    """
+    en: str
+
+class ColumnModel(BaseModel):
+    """Model representing a datatable column.
+
+    Attributes:
+        id (str): Column identifier
+        label (LabelModel): Column labels in different languages
+        description (DescriptionModel): Column descriptions in different languages
+        type (str): Data type of the column
+        constraints (List[Any]): List of column constraints
+        defaultValue (Optional[Any]): Default value for the column
+        facets (List[Any]): List of column facets
+    """
+    id: str
+    label: LabelModel
+    description: DescriptionModel
+    type: str
+    constraints: List[Any]
+    defaultValue: Optional[Any]
+    facets: List[Any]
+
+class DataModel(BaseModel):
+    """Model representing datatable data status.
+
+    Attributes:
+        url (Optional[str]): URL to access the data
+        available (bool): Whether the data is available
+        uptodate (bool): Whether the data is up to date
+    """
+    url: Optional[str]
+    available: bool
+    uptodate: bool
+
+class DatatableModel(BaseModel):
+    """Model representing a complete datatable.
+
+    Attributes:
+        id (str): Datatable identifier
+        name (str): Name of the datatable
+        label (LabelModel): Labels in different languages
+        description (DescriptionModel): Descriptions in different languages
+        schema_name (str): Schema name
+        domains (List[str]): List of associated domains
+        plugins (List[Any]): List of associated plugins
+        columns (List[ColumnModel]): List of table columns
+        facets (List[Any]): List of table facets
+        last_update (datetime): Last update timestamp
+        data (DataModel): Data status information
+    """
+    id: str
+    name: str
+    label: LabelModel
+    description: DescriptionModel
+    schema_name: str = Field(..., alias="schema")
+    domains: List[str]
+    plugins: List[Any]
+    columns: List[ColumnModel]
+    facets: List[Any]
+    last_update: datetime
+    data: DataModel
+
+class Datatables:
+    """Class for managing datatable operations through the SWS API.
+
+    This class provides methods for retrieving datatable information
+    and managing data exports.
+
+    Args:
+        sws_client (SwsApiClient): An instance of the SWS API client
+    """
+
+    def __init__(self, sws_client: SwsApiClient) -> None:
+        """Initialize the Datatables manager with SWS client."""
+        self.sws_client = sws_client
+    
+    def get_datatable_info(self, datatable_id: str) -> DatatableModel:
+        """Retrieve information about a specific datatable.
+
+        Args:
+            datatable_id (str): The identifier of the datatable
+
+        Returns:
+            DatatableModel: Datatable information if found, None otherwise
+        """
+        url = f"/datatables/{datatable_id}"
+
+        response = self.sws_client.discoverable.get('datatable_api', url)
+        if(response.get('id') is not None):
+            return DatatableModel(**response)
+        else:
+            return None
+        
+    def invoke_export_datatable(self, datatable_id: str) -> Dict:
+        """Trigger an export operation for a datatable.
+
+        Args:
+            datatable_id (str): The identifier of the datatable
+
+        Returns:
+            Dict: Response containing export operation status
+        """
+        url = f"/datatables/{datatable_id}/invoke_export_2_s3"
+
+        response = self.sws_client.discoverable.post('datatable_api', url)
+
+        return response
+
+    def get_datatable_csv_path(self, datatable, timeout=60*15, interval=2) -> HttpUrl:
+        """Get the CSV file path for a datatable, waiting for export if necessary.
+
+        Args:
+            datatable: The datatable identifier
+            timeout (int): Maximum time to wait for export in seconds (default: 15 minutes)
+            interval (int): Time between checks in seconds (default: 2)
+
+        Returns:
+            HttpUrl: URL to the CSV file
+
+        Raises:
+            TimeoutError: If export doesn't complete within timeout period
+        """
+        info = self.get_datatable_info(datatable)
+        if info.data.available and info.data.uptodate:
+            return info.data.url
+        else:
+            self.invoke_export_datatable(datatable)
+            # get the updated info every interval seconds until the data are available to a maximum of timeout seconds
+            start = datetime.now()
+            while (datetime.now() - start).seconds < timeout:
+                info = self.get_datatable_info(datatable)
+                if info.data.available:
+                    return info.data.url
+                sleep(interval)