sibi-dst 0.3.32__tar.gz → 0.3.34__tar.gz

{sibi_dst-0.3.32 → sibi_dst-0.3.34}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sibi-dst
-Version: 0.3.32
+Version: 0.3.34
 Summary: Data Science Toolkit
 Author: Luis Valverde
 Author-email: lvalverdeb@gmail.com

{sibi_dst-0.3.32 → sibi_dst-0.3.34}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "sibi-dst"
-version = "0.3.32"
+version = "0.3.34"
 description = "Data Science Toolkit"
 authors = ["Luis Valverde <lvalverdeb@gmail.com>"]
 readme = "README.md"

{sibi_dst-0.3.32 → sibi_dst-0.3.34}/sibi_dst/df_helper/_df_helper.py

@@ -91,7 +91,7 @@ class DfHelper:
         self.filesystem_options = kwargs.pop('filesystem_options', {})
         kwargs.setdefault("live", True)
         kwargs.setdefault("logger", self.logger)
-        kwargs.setdefault("fs", fsspec.filesystem('file'))
+        self.fs =kwargs.setdefault("fs", fsspec.filesystem('file'))
         self.__post_init(**kwargs)
 
     def __str__(self):
@@ -208,6 +208,18 @@ class DfHelper:
                 return asyncio.run(self.__load_from_http(**options))
 
     def __load_from_sqlalchemy(self, **options):
+        """
+        Loads data from an SQLAlchemy database source into a dataframe. The method processes
+        the loaded data and applies post-processing to transform it into the desired structure.
+        If the operation fails, an empty pandas DataFrame is created as a fallback.
+
+        :param options: Additional keyword arguments to configure the data loading process.
+            These options can include configurations such as 'debug' and other parameters
+            required by the `SqlAlchemyLoadFromDb` class.
+        :type options: dict
+        :return: A dataframe containing the data loaded from the SQLAlchemy database.
+        :rtype: dask.dataframe.DataFrame
+        """
         try:
             options.setdefault("debug", self.debug)
             db_loader = SqlAlchemyLoadFromDb(
@@ -228,6 +240,17 @@ class DfHelper:
         return self.df
 
     def __load_from_db(self, **options) -> Union[pd.DataFrame, dd.DataFrame]:
+        """
+        Loads data from a Django database using a specific backend query mechanism. Processes the loaded data
+        and applies further post-processing before returning the dataframe. If the operation fails, an
+        empty dataframe with a single partition is returned instead.
+
+        :param options: Additional settings for the database loading process, which include optional configurations
+            like debug mode, among others.
+        :type options: dict
+        :return: A dataframe containing the loaded data either as a Pandas or Dask dataframe.
+        :rtype: Union[pd.DataFrame, dd.DataFrame]
+        """
         try:
             options.setdefault("debug", self.debug)
             db_loader = DjangoLoadFromDb(
@@ -248,7 +271,18 @@ class DfHelper:
         return self.df
 
     async def __load_from_http(self, **options) -> Union[pd.DataFrame, dd.DataFrame]:
-        """Delegate asynchronous HTTP data loading to HttpDatabackend plugin."""
+        """
+        Loads data asynchronously from an HTTP source using the configured HTTP plugin.
+        If the HTTP plugin is not properly configured, this method logs a debug message and
+        returns an empty Dask DataFrame. If an exception occurs during data fetching, the error
+        is logged and an empty Dask DataFrame with one partition is returned.
+
+        :param options: Additional keyword arguments that are passed to the HTTP plugin for
+                        fetching the data.
+        :returns: A DataFrame object that can either be a pandas or a Dask DataFrame. When the
+                  fetching operation fails, it defaults to returning an empty Dask DataFrame
+                  with a single partition.
+        """
         if not self.backend_http:
             self.logger.debug("HTTP plugin not configured properly.")
             return dd.from_pandas(pd.DataFrame(), npartitions=1)
@@ -339,12 +373,45 @@ class DfHelper:
 
             self.logger.debug("Processing of loaded data completed.")
 
-    def save_to_parquet(self, parquet_filename: Optional[str] = None):
-        ps = ParquetSaver(self.df, self.parquet_storage_path, self.logger)
+    def save_to_parquet(self, parquet_filename: Optional[str] = None, **kwargs):
+        """
+        Save the dataframe result to a Parquet file using specified configurations.
+
+        This method leverages the ParquetSaver class to store the dataframe result
+        into a Parquet file. It also provides functionality for overriding the default
+        filesystem (`fs`) and storage path (`parquet_storage_path`). The method logs
+        details about the saving operation for debugging purposes.
+
+        :param parquet_filename: The name of the Parquet file to save the dataframe to.
+                                  If not provided, a default name will be used.
+        :param kwargs: Additional arguments to customize the saving process. These may
+                       include:
+                       - `fs`: Filesystem to be used for saving Parquet files. If not
+                         provided, defaults to the instance's filesystem attribute.
+                       - `parquet_storage_path`: The root path in the filesystem where
+                         Parquet files should be saved. If not provided, defaults to
+                         the instance's attribute for storage path.
+        :return: None
+        """
+        fs = kwargs.pop('fs', self.fs)
+        parquet_storage_path = kwargs.pop('parquet_storage_path', self.parquet_storage_path)
+        ps = ParquetSaver(df_result=self.df, parquet_storage_path=parquet_storage_path, logger=self.logger, fs=fs)
         ps.save_to_parquet(parquet_filename)
-        self.logger.debug(f"Parquet saved to {parquet_filename} in parquet storage: {self.parquet_storage_path}.")
+        self.logger.debug(f"Parquet saved to {parquet_filename} in parquet storage: {parquet_storage_path}.")
 
     def save_to_clickhouse(self, **credentials):
+        """
+        Saves the current DataFrame to ClickHouse using the provided credentials. This
+        method first checks if the DataFrame is empty. If it is empty, the method logs
+        a debug message and does not proceed with saving. Otherwise, it initializes
+        a ClickHouseWriter instance and uses it to save the DataFrame to ClickHouse,
+        logging a debug message upon successful completion.
+
+        :param credentials: Credentials required to connect to ClickHouse as keyword
+            arguments.
+        :type credentials: dict
+        :return: None
+        """
         if self.df.map_partitions(len).compute().sum() == 0:
             self.logger.debug("Cannot write to clickhouse since Dataframe is empty")
             return
@@ -353,6 +420,21 @@ class DfHelper:
         self.logger.debug("Save to ClickHouse completed.")
 
     def __load_from_parquet(self, **options) -> Union[pd.DataFrame, dd.DataFrame]:
+        """
+        Loads data from parquet files into a DataFrame, applies provided filters, and handles exceptions.
+
+        This method leverages a backend-specific implementation to load data from parquet files into a
+        DataFrame. If additional options are provided and the data is successfully loaded, filters are
+        applied to the DataFrame using a filter handler. Errors during this process are handled gracefully
+        by logging the issue and returning an empty Dask DataFrame.
+
+        :param options: A dictionary of filter options to be applied to the DataFrame.
+        :type options: dict
+
+        :return: A DataFrame containing the loaded and filtered data. If the operation fails, an empty
+            Dask DataFrame is returned.
+        :rtype: Union[pd.DataFrame, dd.DataFrame]
+        """
         try:
             self.df = self.backend_parquet.load_files()
             if options and self.df is not None:
@@ -368,6 +450,27 @@ class DfHelper:
             return dd.from_pandas(pd.DataFrame(), npartitions=1)
 
     def load_period(self, **kwargs):
+        """
+        Loads a period with specified parameters.
+
+        This method acts as a wrapper around the private ``__load_period`` method. It
+        accepts arbitrary keyword arguments that are passed directly to the private
+        method for execution. The purpose of allowing keyword arguments is to permit
+        flexible configuration or parameterization for loading a specific period, based
+        on the internal implementation of the private ``__load_period`` method.
+
+        Note:
+        The arguments and return values are entirely determined by the private
+        method's behavior. This method is intentionally designed to mask details
+        of the internal logic behind the abstraction.
+
+        :param kwargs: Arbitrary keyword arguments to parameterize the internal logic
+            of loading a period. The specific keys and values expected by the
+            ``__load_period`` method depend on its own internal implementation.
+        :return: The result of calling the private ``__load_period`` method with the
+            provided keyword arguments. The return type is dependent on the internal
+            implementation of ``__load_period``.
+        """
         return self.__load_period(**kwargs)
 
     def __load_period(self, **kwargs):

{sibi_dst-0.3.32 → sibi_dst-0.3.34}/sibi_dst/df_helper/_parquet_artifact.py

@@ -9,11 +9,74 @@ from sibi_dst.utils import DateUtils
 
 
 class ParquetArtifact(DfHelper):
+    """
+    Class designed to manage Parquet data storage and retrieval using a specified
+    DataWrapper class for data processing. It provides functionality for loading,
+    updating, rebuilding, and generating Parquet files within a configurable
+    storage filesystem. The class ensures that all essential configurations and
+    filesystems are properly set up before operations.
+
+    Detailed functionality includes support for dynamically managing and generating
+    Parquet files based on time periods, with customizable options for paths,
+    filenames, date fields, and more. It is an abstraction for efficiently handling
+    storage tasks related to distributed or local file systems.
+
+    :ivar config: Configuration dictionary containing all configurable parameters
+                  for managing Parquet data storage, such as paths, filenames,
+                  and date ranges.
+    :type config: dict
+    :ivar df: Cached Dask DataFrame used to store and manipulate data loaded
+              from the Parquet file.
+    :type df: Optional[dask.dataframe.DataFrame]
+    :ivar data_wrapper_class: Class responsible for abstracting data processing
+                              operations required for Parquet file generation.
+    :type data_wrapper_class: type
+    :ivar date_field: Name of the field used to identify and process data by date.
+    :type date_field: Optional[str]
+    :ivar parquet_storage_path: Filesystem path to store Parquet files.
+    :type parquet_storage_path: Optional[str]
+    :ivar parquet_filename: Name of the Parquet file to be generated and managed.
+    :type parquet_filename: Optional[str]
+    :ivar parquet_start_date: Date string specifying the start date for data range
+                              processing.
+    :type parquet_start_date: Optional[str]
+    :ivar parquet_end_date: Date string specifying the end date for data range
+                            processing.
+    :type parquet_end_date: Optional[str]
+    :ivar filesystem_type: Type of the filesystem used for managing storage
+                           operations (e.g., `file`, `s3`, etc.).
+    :type filesystem_type: str
+    :ivar filesystem_options: Additional options for configuring the filesystem.
+    :type filesystem_options: dict
+    :ivar fs: Filesystem object used for storage operations.
+    :type fs: fsspec.AbstractFileSystem
+    """
     DEFAULT_CONFIG = {
         'backend': 'parquet'
     }
 
     def __init__(self, data_wrapper_class,  **kwargs):
+        """
+        Initializes an instance of the class with given configuration and validates
+        required parameters. Sets up the filesystem to handle storage, ensuring
+        necessary directories exist. The configuration supports a variety of options
+        to manage parquet storage requirements, including paths, filenames, and date
+        ranges.
+
+        :param data_wrapper_class: The class responsible for wrapping data to be managed
+                                   by this instance.
+        :type data_wrapper_class: type
+        :param kwargs: Arbitrary keyword arguments to override default configuration.
+                       Includes settings for `date_field`, `parquet_storage_path`,
+                       `parquet_filename`, `parquet_start_date`, `parquet_end_date`,
+                       `filesystem_type`, `filesystem_options`, and `fs`.
+        :type kwargs: dict
+
+        :raises ValueError: If any of the required configuration options
+                            (`date_field`, `parquet_storage_path`,
+                            `parquet_filename`, `parquet_start_date`,
+                            or `parquet_end_date`) are missing or not set properly.
+        """
         self.config = {
             **self.DEFAULT_CONFIG,
             **kwargs,

{sibi_dst-0.3.32 → sibi_dst-0.3.34}/sibi_dst/df_helper/_parquet_reader.py

@@ -7,6 +7,42 @@ from sibi_dst.df_helper import DfHelper
 
 
 class ParquetReader(DfHelper):
+    """
+    This class is a specialized helper for reading and managing Parquet files.
+
+    The `ParquetReader` class is designed to facilitate working with Parquet
+    datasets stored across different filesystems. It initializes the required
+    resources, ensures the existence of the specified Parquet directory,
+    and provides an abstraction to load the data into a Dask DataFrame.
+
+    The class requires configuration for the storage path and dates defining
+    a range of interest. It also supports various filesystem types through
+    `fsspec`.
+
+    :ivar config: Holds the final configuration for this instance, combining
+        `DEFAULT_CONFIG` with user-provided configuration.
+    :type config: dict
+    :ivar df: Stores the loaded Dask DataFrame after the `load()` method is
+        invoked. Initially set to None.
+    :type df: Optional[dd.DataFrame]
+    :ivar parquet_storage_path: The path to the Parquet storage directory.
+    :type parquet_storage_path: str
+    :ivar parquet_start_date: Start date for Parquet data selection. Must
+        be set in the configuration.
+    :type parquet_start_date: str
+    :ivar parquet_end_date: End date for Parquet data selection. Must be
+        set in the configuration.
+    :type parquet_end_date: str
+    :ivar filesystem_type: The type of filesystem the Parquet files are
+        stored on (e.g., "file", "s3").
+    :type filesystem_type: str
+    :ivar filesystem_options: Any additional options required for the
+        specified filesystem type.
+    :type filesystem_options: dict
+    :ivar fs: Instance of `fsspec` filesystem used to interact with the
+        Parquet storage.
+    :type fs: fsspec.AbstractFileSystem
+    """
     DEFAULT_CONFIG = {
         'backend': 'parquet'
     }

sibi_dst-0.3.34/sibi_dst/df_helper/backends/django/_db_connection.py

@@ -0,0 +1,88 @@
+from typing import Any
+
+from pydantic import BaseModel, model_validator
+
+from ._sql_model_builder import DjangoSqlModelBuilder
+
+
+class DjangoConnectionConfig(BaseModel):
+    """
+    Represents a configuration for establishing a Django database connection.
+
+    This class is used for defining the configurations necessary to establish a Django
+    database connection. It supports dynamic model generation if the model is not
+    provided explicitly. It also validates the connection configuration to ensure it
+    is properly set up before being used.
+
+    :ivar live: Indicates whether the connection is live. Automatically set to False if
+        a table is provided without a pre-built model.
+    :type live: bool
+    :ivar connection_name: The name of the database connection to use. This is a mandatory
+        parameter and must be provided.
+    :type connection_name: str
+    :ivar table: The name of the database table to use. Required for dynamic model
+        generation when no model is provided.
+    :type table: str
+    :ivar model: The Django model that represents the database table. If not provided,
+        this can be generated dynamically by using the table name.
+    :type model: Any
+    """
+    live: bool = False
+    connection_name: str = None
+    table: str = None
+    model: Any = None
+
+    @model_validator(mode="after")
+    def check_model(self):
+        """
+        Validates and modifies the instance based on the provided attributes and conditions.
+        This method ensures that all required parameters are populated and consistent, and it
+        dynamically builds a model if necessary. The method also ensures the connection is
+        validated after the model preparation process.
+
+        :raises ValueError: If `connection_name` is not provided.
+        :raises ValueError: If `table` name is not specified when building the model dynamically.
+        :raises ValueError: If there are errors during the dynamic model-building process.
+        :raises ValueError: If `validate_connection` fails due to invalid configuration.
+        :return: The validated and potentially mutated instance.
+        """
+        # connection_name is mandatory
+        if self.connection_name is None:
+            raise ValueError("Connection name must be specified")
+
+        # If table is provided, enforce live=False
+        if self.table:
+            self.live = False
+
+        # If model is not provided, build it dynamically
+        if not self.model:
+            if not self.table:
+                raise ValueError("Table name must be specified to build the model")
+            try:
+                self.model = DjangoSqlModelBuilder(
+                    connection_name=self.connection_name, table=self.table
+                ).build_model()
+            except Exception as e:
+                raise ValueError(f"Failed to build model: {e}")
+        else:
+            self.live = True
+        # Validate the connection after building the model
+        self.validate_connection()
+        return self
+
+    def validate_connection(self):
+        """
+        Ensures the database connection is valid by performing a simple
+        query. Raises a ValueError if the connection is broken or if any
+        other exception occurs during the query.
+
+        :raises ValueError: If the connection to the database cannot be
+            established or if the query fails.
+        """
+        try:
+            # Perform a simple query to test the connection
+            self.model.objects.using(self.connection_name).exists()
+        except Exception as e:
+            raise ValueError(
+                f"Failed to connect to the database '{self.connection_name}': {e}"
+            )