sibi-dst 0.3.31__py3-none-any.whl → 0.3.33__py3-none-any.whl

sibi_dst/df_helper/backends/django/_sql_model_builder.py

@@ -1,5 +1,4 @@
-#  Copyright (c) 2023. ISTMO Center S.A.  All Rights Reserved
-#
+
 import keyword
 import re
 from functools import lru_cache
@@ -49,13 +48,57 @@ apps_label = "datacubes"
 
 
 class DjangoSqlModelBuilder:
+    """
+    Handles the dynamic creation of Django ORM models based on database table structures.
+
+    This class takes input parameters such as database connection and table name,
+    and dynamically maps the table's schema to a Django ORM model. The resultant model
+    can be used for various ORM operations like querying, saving, and deleting records.
+    The class utilizes Django's introspection features and allows customization
+    through its fields and methods.
+
+    :ivar connection_name: The name of the database connection being used.
+    :type connection_name: str
+    :ivar table: The name of the database table for which the model is being built.
+    :type table: str
+    :ivar model: The dynamically generated Django model or None if not created yet.
+    :type model: type | None
+    """
     def __init__(self, **kwargs):
+        """
+        Represents an initialization method for a class that handles the
+        assignment of attributes and processes the given keyword arguments
+        through an internal utility function. This method sets up the
+        necessary attributes for later use.
+
+        :param kwargs: A collection of keyword arguments used by the internal
+            parsing method to populate the attributes of the class. Specific
+            expected keys and their usage should be detailed in the internal
+            implementation.
+        """
         self.connection_name = None
         self.table = None
         self.model = None
         self.__parse_builder(**kwargs)
 
     def __parse_builder(self, **kwargs):
+        """
+        Parses and initializes the builder properties based on provided keyword
+        arguments. Validates that the required 'connection_name' and 'table'
+        values are present and sets the corresponding attributes. If validation
+        fails, raises appropriate errors. Returns the updated builder object
+        after initialization. This method is primarily intended for internal
+        use to configure the builder.
+
+        :param kwargs: Keyword arguments containing configuration values for
+                       initializing the builder. Should include 'connection_name'
+                       and 'table' keys.
+        :type kwargs: dict
+        :return: Returns the instance of the builder object after initialization.
+        :rtype: self
+        :raises ValueError: If 'connection_name' or 'table' is not provided in
+                            the keyword arguments.
+        """
         self.connection_name = kwargs.get("connection_name", None)
         self.table = kwargs.get("table", None)
         self.model = None
@@ -67,6 +110,22 @@ class DjangoSqlModelBuilder:
 
     @lru_cache(maxsize=None)
     def build_model(self):
+        """
+        Builds and retrieves a model instance with dynamically defined fields.
+
+        This method attempts to retrieve a model instance by its name and, if it
+        does not exist, creates a new model with the specified table structure.
+        The model is either fetched or constructed using the provided data about
+        its fields. The result is cached for repeated calls to improve performance
+        and avoid redundant computations.
+
+        :raises LookupError: If the model cannot be fetched or created due to an
+                             invalid lookup.
+
+        :return: A model instance dynamically constructed or retrieved for the
+                 specified table and fields.
+        :rtype: Model
+        """
         model = None
         model_fields = self.get_model_fields()
         model_name = self.table2model(self.table)
@@ -78,6 +137,25 @@ class DjangoSqlModelBuilder:
         return model
 
     def create_model(self, name, fields) -> type:
+        """
+        Creates a Django model class dynamically.
+
+        This function takes in a model name and a dictionary of fields, dynamically
+        creates a Meta class where additional metadata for the model (like
+        `db_table`, `managed`, `app_label`) is defined, and then uses Python's
+        standard library `type()` function to generate and return the model class
+        on the fly.
+
+        :param name: The name of the model class to create.
+        :type name: str
+        :param fields: A dictionary mapping field names to their definitions in
+            Django's model field format. Each field definition should include
+            the field type and optional parameters.
+        :type fields: dict
+        :return: The dynamically created Django model class based on the provided
+            name and fields.
+        :rtype: type
+        """
         def parse_args(arg_string):
             arg_dict = {}
             # Match keyword arguments in the form key=value
@@ -118,9 +196,32 @@ class DjangoSqlModelBuilder:
 
     @staticmethod
     def table2model(table_name):
+        """
+        Converts a database table name to a corresponding model name by transforming
+        it from snake_case to CamelCase. This method takes a string representing
+        a table name, splits it by underscores, capitalizes the first letter of
+        each part, and then joins them into a single string.
+
+        :param table_name: The name of the database table in snake_case format
+        :type table_name: str
+        :return: A string representing the equivalent model name in CamelCase format
+        :rtype: str
+        """
         return "".join([x.title() for x in table_name.split("_")])
 
     def get_model_fields(self):
+        """
+        Generates the data structure for model fields from a database table using
+        introspection. The method extracts information about columns, primary keys,
+        unique constraints, and additional metadata to define the fields of the model.
+
+        :raises ValueError: If the specified connection or table is not found.
+        :raises Exception: For any database or introspection-related errors.
+
+        :returns: Dictionary containing the model field definitions based on the
+            table's structure and metadata.
+        :rtype: dict
+        """
         connection = connections[self.connection_name]
         if connection is None:
             raise ValueError("Connection %s not found" % self.connection_name)
@@ -265,7 +366,21 @@ class DjangoSqlModelBuilder:
     @staticmethod
     def normalize_col_name(col_name, used_column_names, is_relation):
         """
-        Modify the column name to make it Python-compatible as a field name
+        Normalizes a column name to conform to Python's variable naming conventions and addresses potential
+        name conflicts or issues with reserved words. Applies transformations to ensure the column name:
+        - Is lowercase.
+        - Replaces unsuitable characters with underscores.
+        - Avoids conflicts with Python keywords and digits at the start of the name.
+        - Resolves conflicts with previously used column names.
+
+        :param col_name: The original column name provided from the schema.
+        :param used_column_names: A list of previously used column names to avoid naming collisions.
+        :param is_relation: A boolean indicating if the column represents a relation (e.g., foreign key).
+        :return: A tuple containing:
+            - The normalized column name (str).
+            - A dictionary (`field_params`) with any relevant information for database configuration.
+              Includes the original column name if specific transformations were applied.
+            - A list (`field_notes`) containing strings explaining the applied transformations.
         """
         field_params = {}
         field_notes = []
@@ -326,9 +441,20 @@ class DjangoSqlModelBuilder:
     @staticmethod
     def get_field_type(connection, row):
         """
-        Given the database connection, the table name, and the cursor row
-        description, this routine will return the given field type name, as
-        well as any additional keyword parameters and notes for the field.
+        Determines the type of a database field based on its description and connection
+        introspection, and includes metadata such as parameters and additional notes.
+
+        This function extracts the field type from the database's introspection
+        interface and adds corresponding parameters (e.g., `max_length`, `decimal_places`)
+        and relevant notes if certain properties are inferred or guessed.
+
+        :param connection: The database connection object used for introspection.
+        :type connection: Any
+        :param row: An object containing field metadata, such as type code,
+            display size, collation, precision, and scale.
+        :type row: Any
+        :return: A tuple containing the field type, its parameters, and any notes.
+        :rtype: tuple[str, dict, list[str]]
         """
         field_params = {}
         field_notes = []

sibi_dst/df_helper/backends/http/_http_config.py

@@ -9,6 +9,26 @@ from sibi_dst.utils import Logger
 
 
 class HttpConfig(BaseModel):
+    """
+    Configuration for HTTP client operations, designed to manage and fetch data
+    from HTTP endpoints asynchronously. This class serves as a centralized configuration
+    and operation hub encapsulating settings such as base URL, query parameters, API keys,
+    and logger support. It employs `httpx` for HTTP interactions and leverages Dask for the
+    resulting data handling and transformation.
+
+    :ivar base_url: The base URL for HTTP communication.
+    :type base_url: HttpUrl
+    :ivar params: Optional dictionary containing query parameters to be used with GET requests.
+    :type params: Optional[Dict[str, Any]]
+    :ivar logger: The logger instance for logging operations. If not provided, a default logger
+        is initialized using the class name.
+    :type logger: Optional[Logger]
+    :ivar timeout: The timeout value in seconds for HTTP requests. Defaults to 300.
+    :type timeout: Optional[int]
+    :ivar api_key: The optional secret API key for authorization. If present, it will populate
+        the Authorization header in HTTP requests.
+    :type api_key: Optional[SecretStr]
+    """
     base_url: HttpUrl
     params: Optional[Dict[str, Any]] = Field(default_factory=dict)
     logger: Optional[Logger] = None
@@ -17,12 +37,43 @@ class HttpConfig(BaseModel):
     model_config = ConfigDict(arbitrary_types_allowed=True)
 
     def __init__(self, logger=None, **data):
+        """
+        Initializes the class with a logger and other data parameters.
+
+        This constructor allows the option to provide a custom logger. If no logger
+        is supplied during initialization, a default logger specific to the class
+        is created using the Logger utility. It also initializes the instance
+        with additional data passed as keyword arguments.
+
+        :param logger: Optional logger instance. If not provided, a default
+            logger is created using the class name as the logger name.
+        :type logger: logging.Logger, optional
+        :param data: Arbitrary keyword arguments containing data to initialize
+            the class.
+        :type data: dict
+        """
         super().__init__(**data)
         # Initialize the logger if not provided
         self.logger = logger or Logger.default_logger(logger_name=self.__class__.__name__)
 
     async def fetch_data(self, **options) -> dd.DataFrame:
-        """Asynchronously fetch JSON data from HTTP endpoint, substituting options into the URL path."""
+        """
+        Fetches data from a specified HTTP JSON source and returns it as a dask DataFrame.
+
+        This asynchronous method constructs a request URL based on the provided options
+        and sends an HTTP GET request. The fetched JSON data is normalized and
+        converted to a dask DataFrame for further use. It handles request errors and
+        JSON parsing errors effectively.
+
+        :param options: Arbitrary keyword arguments representing dynamic path segments
+            to be appended to the base URL.
+        :type options: dict
+        :return: A dask DataFrame containing the structured data retrieved
+            from the HTTP JSON source.
+        :rtype: dd.DataFrame
+        :raises httpx.RequestError: If there is an issue with the HTTP request.
+        :raises ValueError: If there is an error parsing JSON data.
+        """
         try:
             # Build URL with options as path segments
 

sibi_dst/df_helper/backends/parquet/_filter_handler.py

@@ -5,11 +5,39 @@ from sibi_dst.utils import Logger
 
 
 class ParquetFilterHandler(object):
+    """
+    Handles parquet filtering operations using dask dataframes.
+
+    This class is designed to apply complex filtering logic on dask dataframes
+    based on specified filter criteria. It includes support for operations such
+    as exact matches, ranges, string pattern matches, and null checks. Additionally,
+    it handles datetime-related field filtering including precise truncations and
+    specific date/time attributes.
+
+    :ivar logger: Logger object to handle logging within the class. Defaults to the class-level logger.
+    :type logger: Logger
+    """
     def __init__(self, logger=None):
         self.logger = logger or Logger.default_logger(logger_name=self.__class__.__name__)
 
     @staticmethod
     def apply_filters_dask(df, filters):
+        """
+        Applies a set of filters to a Dask DataFrame, enabling complex filtering operations
+        such as comparisons, ranges, string match operations, and more. Handles special
+        cases for datetime operations, including casting and extracting specific datetime
+        components for filtering.
+
+        :param df: Dask DataFrame to which the filters will be applied.
+        :type df: dask.dataframe.DataFrame
+        :param filters: Dictionary defining the filtering logic, where the keys specify
+            the column name and filter operation, and the values specify the corresponding
+            filter values to apply.
+        :type filters: dict
+        :return: A filtered Dask DataFrame based on the defined logic in the filters.
+        :rtype: dask.dataframe.DataFrame
+        :raises ValueError: If an unsupported operation is encountered in the filters.
+        """
         dt_operators = ['date', 'time']
         date_operators = ['year', 'month', 'day', 'hour', 'minute', 'second', 'week_day']
         comparison_operators = [

sibi_dst/df_helper/backends/parquet/_parquet_options.py

@@ -11,6 +11,45 @@ from sibi_dst.utils import Logger
 
 
 class ParquetConfig(BaseModel):
+    """
+    Represents configuration for managing and validating parquet file operations.
+
+    The `ParquetConfig` class provides attributes and methods necessary to handle operations
+    on parquet files in a file system. It includes functionalities for ensuring file paths
+    and extensions, validating storage paths and parameters, determining file recency,
+    and calculating the size of parquet files. This class is designed with flexibility to handle
+    different file systems through the integration with `fsspec` and allows storage path validations
+    with optional logging support.
+
+    :ivar load_parquet: Indicates whether parquet data should be loaded based on the
+        current configuration and validation.
+    :type load_parquet: bool
+    :ivar parquet_filename: The name of the parquet file, optional if folders are used.
+    :type parquet_filename: Optional[str]
+    :ivar parquet_storage_path: The base path for storing or retrieving parquet files.
+    :type parquet_storage_path: Optional[str]
+    :ivar parquet_full_path: The full path to a specific parquet file, derived from the
+        storage path and filename when applicable.
+    :type parquet_full_path: Optional[str]
+    :ivar parquet_folder_list: A list of folder paths to parquet data, derived from start
+        and end dates if specified.
+    :type parquet_folder_list: Optional[List[str]]
+    :ivar parquet_size_bytes: The total size of the parquet files, in bytes.
+    :type parquet_size_bytes: int
+    :ivar parquet_max_age_minutes: Maximum acceptable age of the most recent parquet file, in minutes.
+    :type parquet_max_age_minutes: int
+    :ivar parquet_is_recent: Indicates whether the parquet file is considered recent based
+        on the `parquet_max_age_minutes` condition.
+    :type parquet_is_recent: bool
+    :ivar parquet_start_date: The start date for parquet file validation or file path generation.
+    :type parquet_start_date: Optional[str]
+    :ivar parquet_end_date: The end date for parquet file validation or file path generation.
+    :type parquet_end_date: Optional[str]
+    :ivar fs: The file system object used for storage operations, compliant with `fsspec`.
+    :type fs: Optional[fsspec.spec.AbstractFileSystem]
+    :ivar logger: A logger for handling logging operations.
+    :type logger: Optional[Logger]
+    """
     load_parquet: bool = False
     parquet_filename: Optional[str] = None
     parquet_storage_path: Optional[str] = None
@@ -27,6 +66,20 @@ class ParquetConfig(BaseModel):
 
     @model_validator(mode='after')
     def check_parquet_params(self):
+        """
+        Validates and configures the parameters required for managing parquet files. This includes
+        configuring paths through `fsspec`, identifying file storage paths, checking the validity of
+        dates related to parquet files, ensuring proper parquet file extensions, and determining
+        whether existing parquet files are recent and loadable.
+
+        :return: The current instance with validated and migrated attributes configured for
+                 handling parquet files.
+
+        :raises ValueError: If certain conditions are not met, such as missing or invalid
+                           `parquet_storage_path`, providing only one of
+                           `parquet_start_date` or `parquet_end_date`, or if the
+                           `parquet_end_date` is earlier than the `parquet_start_date`.
+        """
         # Configure paths based on fsspec
         if self.logger is None:
             self.logger = Logger.default_logger(logger_name=self.__class__.__name__)
@@ -72,6 +125,23 @@ class ParquetConfig(BaseModel):
         return self
 
     def is_file_recent(self):
+        """
+        Determines whether the file at the specified parquet path is considered recent
+        based on its modification time and the maximum age limit defined.
+
+        The function first checks for the existence of the file at the specified
+        `parquet_full_path`. If the file does not exist, the function will return
+        False. If `parquet_max_age_minutes` is set to 0, it implies no maximum age
+        limit, and the function will return True. Otherwise, it retrieves the file's
+        last modified time and calculates the age of the file by comparing it with the
+        current time. The function returns True if the file's age does not exceed the
+        maximum age specified by `parquet_max_age_minutes`, otherwise it returns
+        False.
+
+        :return: Whether the file is considered recent based on its existence,
+                 modification time, and maximum age limit.
+        :rtype: bool
+        """
         if not self.fs.exists(self.parquet_full_path):
             return False
         if self.parquet_max_age_minutes == 0:
@@ -80,6 +150,24 @@ class ParquetConfig(BaseModel):
         return (datetime.datetime.now() - file_time) <= datetime.timedelta(minutes=self.parquet_max_age_minutes)
 
     def get_parquet_size_bytes(self):
+        """
+        Calculate the total size, in bytes, of all Parquet files within the defined
+        folders specified by `parquet_folder_list`. The function iteratively goes
+        through each folder in the provided list, applying a recursive wildcard
+        search to include all levels of nested directories, and calculates the
+        cumulative size of all found Parquet files using the file system's size
+        retrieval method.
+
+        :raises AttributeError: If `fs` or `parquet_folder_list` attributes are not set
+            or improperly configured when the method is called.
+        :raises NotImplementedError: If the `fs.size` or `fs.glob` methods are
+            unimplemented in the provided file system object or it otherwise lacks
+            necessary support for these operations.
+
+        :return: The cumulative size of all Parquet files located in the folders
+            defined by `parquet_folder_list`, measured in bytes.
+        :rtype: int
+        """
         total_size = 0
         for folder in self.parquet_folder_list:
             # Use a double wildcard ** to match any level of nested directories
@@ -88,7 +176,14 @@ class ParquetConfig(BaseModel):
         return total_size
 
     def load_files(self):
-
+        """
+        Loads parquet files into a Dask DataFrame based on the specified conditions. This
+        method checks if parquet file loading is enabled and loads either from a list of
+        parquet folder paths or a single specified parquet path.
+
+        :return: A Dask DataFrame containing loaded parquet file data.
+        :rtype: dask.dataframe.DataFrame
+        """
         if self.load_parquet:
             if self.parquet_folder_list:
                 return dd.read_parquet(self.parquet_folder_list, engine="pyarrow", filesystem=self.fs)
@@ -97,5 +192,14 @@ class ParquetConfig(BaseModel):
 
     @staticmethod
     def ensure_file_extension(filepath: str, extension: str) -> str:
+        """
+        Ensures that the specified file has the desired extension. If the file already has the
+        specified extension, it returns the filepath unchanged. Otherwise, it updates the file
+        extension to the given one and returns the modified filepath.
+
+        :param filepath: The path to the file as a string.
+        :param extension: The desired file extension, without the leading dot.
+        :return: The updated file path as a string, ensuring it has the specified extension.
+        """
         path = Path(filepath)
         return str(path.with_suffix(f".{extension}")) if path.suffix != f".{extension}" else filepath

sibi_dst/df_helper/backends/sqlalchemy/_db_connection.py

@@ -9,6 +9,23 @@ from ._sql_model_builder import SqlAlchemyModelBuilder
 
 
 class SqlAlchemyConnectionConfig(BaseModel):
+    """
+    Configuration class for managing an SQLAlchemy database connection.
+
+    This class provides configurations to establish a connection to a database,
+    validate the connection, and dynamically build a SQLAlchemy model for a specific
+    table if required. It initializes the database engine using the provided connection URL
+    and ensures that the connection and table information are properly validated.
+
+    :ivar connection_url: The URL used to connect to the database.
+    :type connection_url: str
+    :ivar table: The name of the database table for which a model will be constructed.
+    :type table: Optional[str]
+    :ivar model: The dynamically built SQLAlchemy model for the specified table.
+    :type model: Any
+    :ivar engine: The SQLAlchemy engine instance reused for database connections.
+    :type engine: Optional[Any]
+    """
     connection_url: str
     table: Optional[str] = None
     model: Any = None

sibi_dst/df_helper/backends/sqlalchemy/_load_from_db.py

@@ -8,6 +8,41 @@ from ._db_connection import SqlAlchemyConnectionConfig
 
 
 class SqlAlchemyLoadFromDb:
+    """
+    The SqlAlchemyLoadFromDb class provides functionality to load data from a
+    database using SQLAlchemy into a Dask DataFrame. It is capable of handling
+    large datasets efficiently by utilizing the Dask framework for parallel
+    computations.
+
+    This class is initialized with a database connection configuration, query
+    configuration, optional parameters, and a logger. It can execute a query
+    using the specified configurations and read the results into a Dask
+    DataFrame. This is useful for processing and analyzing large-scale data.
+
+    :ivar df: Dask DataFrame to store the loaded data.
+    :type df: dd.DataFrame
+    :ivar db_connection: Database connection configuration object, containing details
+        such as the table, model, and engine to be used for the query.
+    :type db_connection: SqlAlchemyConnectionConfig
+    :ivar table_name: Name of the database table being queried.
+    :type table_name: str
+    :ivar model: SQLAlchemy model associated with the database connection.
+    :type model: sqlalchemy.ext.declarative.api.DeclarativeMeta
+    :ivar engine: SQLAlchemy engine used for executing queries.
+    :type engine: sqlalchemy.engine.base.Engine
+    :ivar logger: Logger instance for logging debug and error information.
+    :type logger: Logger
+    :ivar query_config: Query configuration, including query-related details such
+        as the SQL query or query settings.
+    :type query_config: QueryConfig
+    :ivar params_config: Parameters configuration, including filter parameters for
+        the query.
+    :type params_config: ParamsConfig
+    :ivar debug: Debug flag indicating whether debug mode is enabled.
+    :type debug: bool
+    :ivar chunk_size: Size of data chunks to process at a time.
+    :type chunk_size: int
+    """
     df: dd.DataFrame = None
 
     def __init__(
@@ -19,7 +54,28 @@ class SqlAlchemyLoadFromDb:
             **kwargs,
     ):
         """
-        Initialize the loader with database connection, query, and parameters.
+        Initializes an instance of the class, setting up a database connection,
+        query configuration, parameter configuration, and other optional settings
+        like debugging and logging. The class aims to manage the integration and
+        interaction with SQLAlchemy-based database operations.
+
+        :param plugin_sqlalchemy:
+            The SQLAlchemy connection configuration object, which provides
+            the connection details like engine, table name, and model
+            associated with the database operations.
+        :param plugin_query:
+            The query configuration object, used to define specific query
+            options or rules. Defaults to None.
+        :param plugin_params:
+            The parameters configuration object, used for any additional
+            parameterized settings or configurations. Defaults to None.
+        :param logger:
+            Optional logger instance for logging purposes. If not provided,
+            a default logger is instantiated using the standard logging system.
+        :param kwargs:
+            Optional additional keyword arguments for customization. Can
+            include optional settings like `debug` mode or `chunk_size`
+            for batch operations.
         """
         self.db_connection = plugin_sqlalchemy
         self.table_name = self.db_connection.table
@@ -33,13 +89,35 @@ class SqlAlchemyLoadFromDb:
 
     def build_and_load(self) -> dd.DataFrame:
         """
-        Load data into a Dask DataFrame based on the query and parameters.
+        Builds and returns the resulting dataframe after calling the internal
+        build and load function. This method triggers the `_build_and_load`
+        function to process and prepare the data before returning it as
+        a dask dataframe.
+
+        :raises RuntimeError: If any error occurs during the build or load process.
+
+        :return: The processed data in a dask dataframe.
+        :rtype: dd.DataFrame
         """
         self._build_and_load()
         return self.df
 
     def _build_and_load(self) -> dd.DataFrame:
+        """
+        Builds and loads a Dask DataFrame from a SQLAlchemy-compatible source.
+
+        This method initializes a SQLAlchemyDask object with the provided model,
+        filters, engine URL, logger, chunk size, and debug configuration.
+        It attempts to load the data using the ``read_frame`` method of
+        SQLAlchemyDask. If the data cannot be loaded or the query returns
+        no rows, it creates and returns an empty Dask DataFrame.
 
+        :raises Exception: On failure to load data or to create a DataFrame.
+
+        :return: A Dask DataFrame object containing the queried data or an
+                 empty DataFrame if the query returns no results or fails.
+        :rtype: dask.dataframe.DataFrame
+        """
         try:
             self.df = SQLAlchemyDask(
                 model=self.model,