sibi-dst 0.3.31__tar.gz → 0.3.33__tar.gz

{sibi_dst-0.3.31 → sibi_dst-0.3.33}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sibi-dst
-Version: 0.3.31
+Version: 0.3.33
 Summary: Data Science Toolkit
 Author: Luis Valverde
 Author-email: lvalverdeb@gmail.com
@@ -20,6 +20,7 @@ Requires-Dist: django (>=5.1.4,<6.0.0)
 Requires-Dist: djangorestframework (>=3.15.2,<4.0.0)
 Requires-Dist: folium (>=0.19.4,<0.20.0)
 Requires-Dist: geopandas (>=1.0.1,<2.0.0)
+Requires-Dist: gunicorn (>=23.0.0,<24.0.0)
 Requires-Dist: httpx (>=0.27.2,<0.28.0)
 Requires-Dist: ipython (>=8.29.0,<9.0.0)
 Requires-Dist: jinja2 (>=3.1.4,<4.0.0)
@@ -42,6 +43,7 @@ Requires-Dist: sqlalchemy (>=2.0.36,<3.0.0)
 Requires-Dist: tornado (>=6.4.1,<7.0.0)
 Requires-Dist: tqdm (>=4.67.0,<5.0.0)
 Requires-Dist: uvicorn (>=0.34.0,<0.35.0)
+Requires-Dist: uvicorn-worker (>=0.3.0,<0.4.0)
 Description-Content-Type: text/markdown
 
 # sibi-dst

{sibi_dst-0.3.31 → sibi_dst-0.3.33}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "sibi-dst"
-version = "0.3.31"
+version = "0.3.33"
 description = "Data Science Toolkit"
 authors = ["Luis Valverde <lvalverdeb@gmail.com>"]
 readme = "README.md"
@@ -41,6 +41,8 @@ nltk = "^3.9.1"
 folium = "^0.19.4"
 geopandas = "^1.0.1"
 osmnx = "^2.0.1"
+gunicorn = "^23.0.0"
+uvicorn-worker = "^0.3.0"
 
 
 [build-system]

{sibi_dst-0.3.31 → sibi_dst-0.3.33}/sibi_dst/df_helper/_parquet_artifact.py

@@ -9,11 +9,74 @@ from sibi_dst.utils import DateUtils
 
 
 class ParquetArtifact(DfHelper):
+    """
+    Class designed to manage Parquet data storage and retrieval using a specified
+    DataWrapper class for data processing. It provides functionality for loading,
+    updating, rebuilding, and generating Parquet files within a configurable
+    storage filesystem. The class ensures that all essential configurations and
+    filesystems are properly set up before operations.
+
+    Detailed functionality includes support for dynamically managing and generating
+    Parquet files based on time periods, with customizable options for paths,
+    filenames, date fields, and more. It is an abstraction for efficiently handling
+    storage tasks related to distributed or local file systems.
+
+    :ivar config: Configuration dictionary containing all configurable parameters
+                  for managing Parquet data storage, such as paths, filenames,
+                  and date ranges.
+    :type config: dict
+    :ivar df: Cached Dask DataFrame used to store and manipulate data loaded
+              from the Parquet file.
+    :type df: Optional[dask.dataframe.DataFrame]
+    :ivar data_wrapper_class: Class responsible for abstracting data processing
+                              operations required for Parquet file generation.
+    :type data_wrapper_class: type
+    :ivar date_field: Name of the field used to identify and process data by date.
+    :type date_field: Optional[str]
+    :ivar parquet_storage_path: Filesystem path to store Parquet files.
+    :type parquet_storage_path: Optional[str]
+    :ivar parquet_filename: Name of the Parquet file to be generated and managed.
+    :type parquet_filename: Optional[str]
+    :ivar parquet_start_date: Date string specifying the start date for data range
+                              processing.
+    :type parquet_start_date: Optional[str]
+    :ivar parquet_end_date: Date string specifying the end date for data range
+                            processing.
+    :type parquet_end_date: Optional[str]
+    :ivar filesystem_type: Type of the filesystem used for managing storage
+                           operations (e.g., `file`, `s3`, etc.).
+    :type filesystem_type: str
+    :ivar filesystem_options: Additional options for configuring the filesystem.
+    :type filesystem_options: dict
+    :ivar fs: Filesystem object used for storage operations.
+    :type fs: fsspec.AbstractFileSystem
+    """
     DEFAULT_CONFIG = {
         'backend': 'parquet'
     }
 
     def __init__(self, data_wrapper_class,  **kwargs):
+        """
+        Initializes an instance of the class with given configuration and validates
+        required parameters. Sets up the filesystem to handle storage, ensuring
+        necessary directories exist. The configuration supports a variety of options
+        to manage parquet storage requirements, including paths, filenames, and date
+        ranges.
+
+        :param data_wrapper_class: The class responsible for wrapping data to be managed
+                                   by this instance.
+        :type data_wrapper_class: type
+        :param kwargs: Arbitrary keyword arguments to override default configuration.
+                       Includes settings for `date_field`, `parquet_storage_path`,
+                       `parquet_filename`, `parquet_start_date`, `parquet_end_date`,
+                       `filesystem_type`, `filesystem_options`, and `fs`.
+        :type kwargs: dict
+
+        :raises ValueError: If any of the required configuration options
+                            (`date_field`, `parquet_storage_path`,
+                            `parquet_filename`, `parquet_start_date`,
+                            or `parquet_end_date`) are missing or not set properly.
+        """
         self.config = {
             **self.DEFAULT_CONFIG,
             **kwargs,
@@ -61,6 +124,11 @@ class ParquetArtifact(DfHelper):
         dw = DataWrapper(self.data_wrapper_class, **params)
         dw.process()
 
+    def __exit__(self, exc_type, exc_value, traceback):
+        # Ensure resources are cleaned up
+        if self.fs:
+            self.fs.close()
+
     def update_parquet(self, period: str = 'today', **kwargs) -> None:
         """Update the Parquet file with data from a specific period."""
         kwargs.update(self.parse_parquet_period(period=period))

sibi_dst-0.3.33/sibi_dst/df_helper/_parquet_reader.py

@@ -0,0 +1,94 @@
+from typing import Optional
+
+import dask.dataframe as dd
+import fsspec
+
+from sibi_dst.df_helper import DfHelper
+
+
+class ParquetReader(DfHelper):
+    """
+    This class is a specialized helper for reading and managing Parquet files.
+
+    The `ParquetReader` class is designed to facilitate working with Parquet
+    datasets stored across different filesystems. It initializes the required
+    resources, ensures the existence of the specified Parquet directory,
+    and provides an abstraction to load the data into a Dask DataFrame.
+
+    The class requires configuration for the storage path and dates defining
+    a range of interest. It also supports various filesystem types through
+    `fsspec`.
+
+    :ivar config: Holds the final configuration for this instance, combining
+        `DEFAULT_CONFIG` with user-provided configuration.
+    :type config: dict
+    :ivar df: Stores the loaded Dask DataFrame after the `load()` method is
+        invoked. Initially set to None.
+    :type df: Optional[dd.DataFrame]
+    :ivar parquet_storage_path: The path to the Parquet storage directory.
+    :type parquet_storage_path: str
+    :ivar parquet_start_date: Start date for Parquet data selection. Must
+        be set in the configuration.
+    :type parquet_start_date: str
+    :ivar parquet_end_date: End date for Parquet data selection. Must be
+        set in the configuration.
+    :type parquet_end_date: str
+    :ivar filesystem_type: The type of filesystem the Parquet files are
+        stored on (e.g., "file", "s3").
+    :type filesystem_type: str
+    :ivar filesystem_options: Any additional options required for the
+        specified filesystem type.
+    :type filesystem_options: dict
+    :ivar fs: Instance of `fsspec` filesystem used to interact with the
+        Parquet storage.
+    :type fs: fsspec.AbstractFileSystem
+    """
+    DEFAULT_CONFIG = {
+        'backend': 'parquet'
+    }
+
+    def __init__(self, filesystem_type="file", filesystem_options=None, **kwargs):
+        self.config = {
+            **self.DEFAULT_CONFIG,
+            **kwargs,
+        }
+        self.df: Optional[dd.DataFrame] = None
+        self.parquet_storage_path = self.config.setdefault('parquet_storage_path', None)
+        if self.parquet_storage_path is None:
+            raise ValueError('parquet_storage_path must be set')
+        self.parquet_start_date = self.config.setdefault('parquet_start_date', None)
+        if self.parquet_start_date is None:
+            raise ValueError('parquet_start_date must be set')
+
+        self.parquet_end_date = self.config.setdefault('parquet_end_date', None)
+        if self.parquet_end_date is None:
+            raise ValueError('parquet_end_date must be set')
+
+        # Filesystem setup
+        self.filesystem_type = filesystem_type
+        self.filesystem_options = filesystem_options or {}
+        self.fs = self.config.setdefault('fs', None)
+        if self.fs is None:
+            self.fs = fsspec.filesystem(self.filesystem_type, **self.filesystem_options)
+        self.config.setdefault('fs', self.fs)
+
+        if not self.directory_exists():
+            raise ValueError(f"{self.parquet_storage_path} does not exist")
+
+        super().__init__(**self.config)
+
+    def load(self, **kwargs):
+        self.df = super().load(**kwargs)
+        return self.df
+
+    def directory_exists(self):
+        try:
+            info = self.fs.info(self.parquet_storage_path)
+            return info['type'] == 'directory'
+        except FileNotFoundError:
+            return False
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        # Ensure resources are cleaned up
+        if self.fs:
+            self.fs.close()

sibi_dst-0.3.33/sibi_dst/df_helper/backends/django/_db_connection.py

@@ -0,0 +1,88 @@
+from typing import Any
+
+from pydantic import BaseModel, model_validator
+
+from ._sql_model_builder import DjangoSqlModelBuilder
+
+
+class DjangoConnectionConfig(BaseModel):
+    """
+    Represents a configuration for establishing a Django database connection.
+
+    This class is used for defining the configurations necessary to establish a Django
+    database connection. It supports dynamic model generation if the model is not
+    provided explicitly. It also validates the connection configuration to ensure it
+    is properly set up before being used.
+
+    :ivar live: Indicates whether the connection is live. Automatically set to False if
+        a table is provided without a pre-built model.
+    :type live: bool
+    :ivar connection_name: The name of the database connection to use. This is a mandatory
+        parameter and must be provided.
+    :type connection_name: str
+    :ivar table: The name of the database table to use. Required for dynamic model
+        generation when no model is provided.
+    :type table: str
+    :ivar model: The Django model that represents the database table. If not provided,
+        this can be generated dynamically by using the table name.
+    :type model: Any
+    """
+    live: bool = False
+    connection_name: str = None
+    table: str = None
+    model: Any = None
+
+    @model_validator(mode="after")
+    def check_model(self):
+        """
+        Validates and modifies the instance based on the provided attributes and conditions.
+        This method ensures that all required parameters are populated and consistent, and it
+        dynamically builds a model if necessary. The method also ensures the connection is
+        validated after the model preparation process.
+
+        :raises ValueError: If `connection_name` is not provided.
+        :raises ValueError: If `table` name is not specified when building the model dynamically.
+        :raises ValueError: If there are errors during the dynamic model-building process.
+        :raises ValueError: If `validate_connection` fails due to invalid configuration.
+        :return: The validated and potentially mutated instance.
+        """
+        # connection_name is mandatory
+        if self.connection_name is None:
+            raise ValueError("Connection name must be specified")
+
+        # If table is provided, enforce live=False
+        if self.table:
+            self.live = False
+
+        # If model is not provided, build it dynamically
+        if not self.model:
+            if not self.table:
+                raise ValueError("Table name must be specified to build the model")
+            try:
+                self.model = DjangoSqlModelBuilder(
+                    connection_name=self.connection_name, table=self.table
+                ).build_model()
+            except Exception as e:
+                raise ValueError(f"Failed to build model: {e}")
+        else:
+            self.live = True
+        # Validate the connection after building the model
+        self.validate_connection()
+        return self
+
+    def validate_connection(self):
+        """
+        Ensures the database connection is valid by performing a simple
+        query. Raises a ValueError if the connection is broken or if any
+        other exception occurs during the query.
+
+        :raises ValueError: If the connection to the database cannot be
+            established or if the query fails.
+        """
+        try:
+            # Perform a simple query to test the connection
+            self.model.objects.using(self.connection_name).exists()
+        except Exception as e:
+            raise ValueError(
+                f"Failed to connect to the database '{self.connection_name}': {e}"
+            )