sibi-dst 0.3.32__py3-none-any.whl → 0.3.34__py3-none-any.whl

sibi_dst/df_helper/backends/sqlalchemy/_load_from_db.py

@@ -8,6 +8,41 @@ from ._db_connection import SqlAlchemyConnectionConfig
 
 
 class SqlAlchemyLoadFromDb:
+    """
+    The SqlAlchemyLoadFromDb class provides functionality to load data from a
+    database using SQLAlchemy into a Dask DataFrame. It is capable of handling
+    large datasets efficiently by utilizing the Dask framework for parallel
+    computations.
+
+    This class is initialized with a database connection configuration, query
+    configuration, optional parameters, and a logger. It can execute a query
+    using the specified configurations and read the results into a Dask
+    DataFrame. This is useful for processing and analyzing large-scale data.
+
+    :ivar df: Dask DataFrame to store the loaded data.
+    :type df: dd.DataFrame
+    :ivar db_connection: Database connection configuration object, containing details
+        such as the table, model, and engine to be used for the query.
+    :type db_connection: SqlAlchemyConnectionConfig
+    :ivar table_name: Name of the database table being queried.
+    :type table_name: str
+    :ivar model: SQLAlchemy model associated with the database connection.
+    :type model: sqlalchemy.ext.declarative.api.DeclarativeMeta
+    :ivar engine: SQLAlchemy engine used for executing queries.
+    :type engine: sqlalchemy.engine.base.Engine
+    :ivar logger: Logger instance for logging debug and error information.
+    :type logger: Logger
+    :ivar query_config: Query configuration, including query-related details such
+        as the SQL query or query settings.
+    :type query_config: QueryConfig
+    :ivar params_config: Parameters configuration, including filter parameters for
+        the query.
+    :type params_config: ParamsConfig
+    :ivar debug: Debug flag indicating whether debug mode is enabled.
+    :type debug: bool
+    :ivar chunk_size: Size of data chunks to process at a time.
+    :type chunk_size: int
+    """
     df: dd.DataFrame = None
 
     def __init__(
@@ -19,7 +54,28 @@ class SqlAlchemyLoadFromDb:
             **kwargs,
     ):
         """
-        Initialize the loader with database connection, query, and parameters.
+        Initializes an instance of the class, setting up a database connection,
+        query configuration, parameter configuration, and other optional settings
+        like debugging and logging. The class aims to manage the integration and
+        interaction with SQLAlchemy-based database operations.
+
+        :param plugin_sqlalchemy:
+            The SQLAlchemy connection configuration object, which provides
+            the connection details like engine, table name, and model
+            associated with the database operations.
+        :param plugin_query:
+            The query configuration object, used to define specific query
+            options or rules. Defaults to None.
+        :param plugin_params:
+            The parameters configuration object, used for any additional
+            parameterized settings or configurations. Defaults to None.
+        :param logger:
+            Optional logger instance for logging purposes. If not provided,
+            a default logger is instantiated using the standard logging system.
+        :param kwargs:
+            Optional additional keyword arguments for customization. Can
+            include optional settings like `debug` mode or `chunk_size`
+            for batch operations.
         """
         self.db_connection = plugin_sqlalchemy
         self.table_name = self.db_connection.table
@@ -33,13 +89,35 @@ class SqlAlchemyLoadFromDb:
 
     def build_and_load(self) -> dd.DataFrame:
         """
-        Load data into a Dask DataFrame based on the query and parameters.
+        Builds and returns the resulting dataframe after calling the internal
+        build and load function. This method triggers the `_build_and_load`
+        function to process and prepare the data before returning it as
+        a dask dataframe.
+
+        :raises RuntimeError: If any error occurs during the build or load process.
+
+        :return: The processed data in a dask dataframe.
+        :rtype: dd.DataFrame
         """
         self._build_and_load()
         return self.df
 
     def _build_and_load(self) -> dd.DataFrame:
+        """
+        Builds and loads a Dask DataFrame from a SQLAlchemy-compatible source.
+
+        This method initializes a SQLAlchemyDask object with the provided model,
+        filters, engine URL, logger, chunk size, and debug configuration.
+        It attempts to load the data using the ``read_frame`` method of
+        SQLAlchemyDask. If the data cannot be loaded or the query returns
+        no rows, it creates and returns an empty Dask DataFrame.
 
+        :raises Exception: On failure to load data or to create a DataFrame.
+
+        :return: A Dask DataFrame object containing the queried data or an
+                 empty DataFrame if the query returns no results or fails.
+        :rtype: dask.dataframe.DataFrame
+        """
         try:
             self.df = SQLAlchemyDask(
                 model=self.model,

sibi_dst/df_helper/backends/sqlalchemy/_sql_model_builder.py

@@ -10,6 +10,28 @@ apps_label = "datacubes"
 
 
 class SqlAlchemyModelBuilder:
+    """
+    Provides functionality for building SQLAlchemy ORM models dynamically from
+    reflected database tables. This class is intended for use with a SQLAlchemy
+    engine and metadata to automatically generate ORM models for specified
+    database tables.
+
+    The primary purpose of this class is to simplify the process of creating
+    SQLAlchemy ORM models by reflecting tables from a connected database,
+    dynamically generating model classes, and handling relationships between
+    tables.
+
+    :ivar engine: SQLAlchemy engine connected to the database.
+    :type engine: Engine
+    :ivar table_name: Name of the table for which the model is generated.
+    :type table_name: str
+    :ivar metadata: SQLAlchemy MetaData instance for reflecting tables.
+    :type metadata: MetaData
+    :ivar table: Reflected SQLAlchemy Table object for the specified table name.
+    :type table: Optional[Table]
+    :ivar class_name: Dynamically normalized class name derived from table_name.
+    :type class_name: str
+    """
     _model_cache = {}  # Local cache for model classes
 
     def __init__(self, engine, table_name):
@@ -27,6 +49,16 @@ class SqlAlchemyModelBuilder:
         self.class_name = self.normalize_class_name(self.table_name)
 
     def build_model(self) -> type:
+        """
+        Builds and returns a database model class corresponding to the specified table name.
+        The method checks if the model is already registered in the ORM's registry. If not,
+        it reflects the database schema of the specified table and dynamically creates the
+        model class.
+
+        :raises ValueError: If the specified table does not exist in the database.
+        :return: A database model class corresponding to the specified table name.
+        :rtype: type
+        """
         # Check if the model is already registered
         model = Base.registry._class_registry.get(self.class_name)
         if model:
@@ -42,10 +74,17 @@ class SqlAlchemyModelBuilder:
 
     def create_model(self) -> type:
         """
-        Create a SQLAlchemy ORM model for the reflected table.
+        Generates a SQLAlchemy model class dynamically based on the specified table and
+        its columns. The method extracts column information, defines the necessary
+        attributes, and creates the model class if it doesn't already exist in the
+        SQLAlchemy base registry.
 
-        Returns:
-            type: Dynamically generated SQLAlchemy ORM model class.
+        :raises KeyError: If the table or table name does not exist in the provided
+            schema.
+        :raises Exception: If the model creation fails for any reason.
+
+        :return: The dynamically created or fetched model class.
+        :rtype: type
         """
         # Normalize the class name from the table name
         columns = self.get_columns(self.table)
@@ -70,13 +109,17 @@ class SqlAlchemyModelBuilder:
 
     def get_columns(self, table: Table):
         """
-        Extract columns from the table and create corresponding SQLAlchemy fields.
-
-        Args:
-            table (Table): SQLAlchemy Table object.
-
-        Returns:
-            dict: Dictionary of column attributes.
+        Extracts and returns a dictionary of column names and their corresponding column
+        objects from a given table, excluding reserved names. Reserved names are used
+        internally and should not overlap with column names in the provided table. The
+        method ensures sanitized column names through normalization and filters out any
+        column matching reserved keywords.
+
+        :param table: The table object from which columns are to be extracted.
+        :type table: Table
+        :return: A dictionary containing the sanitized column names as keys and their
+            corresponding column objects as values, excluding reserved names.
+        :rtype: dict
         """
         columns = {}
         reserved_names = ["metadata", "class_", "table"]
@@ -89,11 +132,18 @@ class SqlAlchemyModelBuilder:
 
     def add_relationships(self, attrs, table: Table):
         """
-        Add relationships to the model for foreign keys.
-
-        Args:
-            attrs (dict): Attributes of the dynamically created model.
-            table (Table): SQLAlchemy Table object.
+        Adds relationships to the provided attributes dictionary for a given database table.
+
+        This method iterates through the foreign keys of the provided table, constructs
+        relationship attributes, and updates the attributes dictionary with relationships
+        that connect the current table to related tables.
+
+        :param attrs: Dictionary of attributes to which relationships will be added.
+                      The dictionary will be updated with new relationship mappings.
+        :type attrs: dict
+        :param table: A database table object containing foreign key relationships.
+                      The method will use this table to establish relationships.
+        :return: None
         """
         for fk in table.foreign_keys:
             related_table_name = fk.column.table.name
@@ -104,26 +154,37 @@ class SqlAlchemyModelBuilder:
     @staticmethod
     def normalize_class_name(table_name: str) -> str:
         """
-        Normalize a table name into a valid Python class name.
-
-        Args:
-            table_name (str): Name of the table.
-
-        Returns:
-            str: Normalized class name.
+        Generate a normalized class name from a given table name by capitalizing
+        each word separated by underscores and concatenating them.
+
+        This static method takes a string representation of a table name, where
+        words are separated by underscores, and converts it into a camel case
+        class name. It processes the string by capitalizing the first letter of
+        each word and removing the underscores. The normalized class name
+        returned can be used programmatically for various purposes, such as
+        class generation or naming conventions.
+
+        :param table_name: The table name to normalize, with words separated by
+            underscores. E.g., 'sample_table' becomes 'SampleTable'.
+        :type table_name: str
+        :return: A normalized class name in camel case format.
+        :rtype: str
         """
         return "".join(word.capitalize() for word in table_name.split("_"))
 
     @staticmethod
     def normalize_column_name(column_name: str) -> str:
         """
-        Normalize a column name into a valid Python identifier.
-
-        Args:
-            column_name (str): Name of the column.
-
-        Returns:
-            str: Normalized column name.
+        Normalize a column name by replacing any non-word characters or leading numbers
+        with underscores, while ensuring it does not conflict with reserved keywords
+        such as 'class', 'def', 'return', etc. If the normalized name conflicts with
+        a Python reserved keyword, "_field" is appended to it.
+
+        :param column_name: The original name of the column to be normalized.
+        :type column_name: str
+        :return: A normalized column name that is safe and compatible for usage
+            in various contexts such as database columns or Python code.
+        :rtype: str
         """
         column_name = re.sub(r"\W|^(?=\d)", "_", column_name)
         if column_name in {"class", "def", "return", "yield", "global"}:

sibi_dst/df_helper/core/_params_config.py

@@ -27,6 +27,36 @@ LOOKUP_SEP = "__"
 
 
 class ParamsConfig(BaseModel):
+    """
+    Defines a configuration model for parameters with functionality for parsing,
+    validation, and conversion of legacy filters.
+
+    This class extends BaseModel from Pydantic and is designed to handle multiple
+    sets of configurations, including field mappings, filters, dataframe parameters,
+    and dataframe options. It allows for flexible parsing of parameters across a
+    variety of supported structures and ensures that legacy filters can be
+    appropriately converted for compatibility.
+
+    :ivar field_map: Maps field names to their equivalent legacy field names.
+    :type field_map: Optional[Dict]
+    :ivar legacy_filters: Indicates whether legacy filters should be processed.
+    :type legacy_filters: bool
+    :ivar sticky_filters: Stores additional filters as key-value pairs that persist
+        across parameter parsing.
+    :type sticky_filters: Dict[str, Union[str, bool, int, float, list, tuple]]
+    :ivar filters: Holds all the current filters including sticky and dynamically
+        parsed filters.
+    :type filters: Dict[str, Union[str, Dict, bool, int, float, list, tuple]]
+    :ivar df_params: Contains parameters related to dataframe configurations in a
+        structured format.
+    :type df_params: Dict[str, Union[tuple, str, bool, None]]
+    :ivar df_options: Stores optional configurations for a dataframe, allowing for
+        additional behavior customization.
+    :type df_options: Dict[str, Union[bool, str, None]]
+    :ivar params: Dictionary of parameters provided for configuration, supporting
+        both basic and nested structures.
+    :type params: Dict[str, Union[str, bool, int, float, List[Union[str, int, bool, float]]]]
+    """
     field_map: Optional[Dict] = Field(default_factory=dict)
     legacy_filters: bool = False
     sticky_filters: Dict[str, Union[str, bool, int, float, list, tuple]] = Field(default_factory=dict)
@@ -42,6 +72,17 @@ class ParamsConfig(BaseModel):
         return self
 
     def parse_params(self, params):
+        """
+        Parses and separates the given parameters into specific categories such as dataframe parameters,
+        dataframe options, and filters. Updates existing class attributes with the parsed values,
+        retaining any sticky filters. Also handles the legacy filters if provided.
+
+        :param params: Dictionary containing parameters to process. These parameters can include specific
+            keys relevant for dataframe configuration (e.g., dataframe parameters, dataframe options)
+            as well as arbitrary filter settings.
+        :type params: dict
+        :return: None
+        """
         self.legacy_filters = params.pop('legacy_filters', self.legacy_filters)
         self.field_map = params.pop('field_map', self.field_map)
         self.sticky_filters = params.pop('params', self.sticky_filters)
@@ -60,6 +101,24 @@ class ParamsConfig(BaseModel):
             self.convert_legacy_filters()
 
     def convert_legacy_filters(self):
+        """
+            Converts legacy filter fields in the `self.filters` dictionary to their
+            modern equivalents using the mappings provided in `self.field_map`.
+            This method ensures backward compatibility for filters by automatically
+            translating the old field names into the current system.
+
+            The function first verifies that the required dictionaries (`legacy_filters`,
+            `field_map`, `filters`) are valid. It creates a reverse map of `field_map` for
+            efficient lookup, processes the key names within `self.filters`, and updates
+            them to reflect the legacy mapping.
+
+            :raises KeyError: If any required dictionary key is missing during processing.
+
+            :param self.legacy_filters: A boolean flag indicating whether legacy filters
+                are being used.
+            :type self.legacy_filters: bool
+
+        """
         if not self.legacy_filters or not self.field_map or not self.filters:
             return
         # create a reverse map of the field_map

sibi_dst/geopy_helper/geo_location_service.py

@@ -8,6 +8,20 @@ app_geo_locator_test_place = os.environ.get('GEO_LOCATOR_TEST_PLACE', "San Jose,
 
 
 class GeolocationService:
+    """
+    Provides geolocation services, such as forward and reverse geocoding.
+
+    This class is intended to interface with a geocoding service (e.g., Nominatim)
+    for performing geocoding operations. It initializes the geolocation service
+    based on a provided or default configuration and provides methods for geocoding
+    addresses or retrieving addresses from coordinates.
+
+    :ivar geolocator: Instance of the geocoding service used for geolocation tasks.
+        Will be `None` if initialization fails or is incomplete.
+    :type geolocator: Optional[Nominatim]
+    :ivar debug: Indicates whether debug messages are enabled.
+    :type debug: bool
+    """
     debug: bool = False
 
     def __init__(self, debug=False):

sibi_dst/geopy_helper/utils.py

@@ -5,6 +5,16 @@ geolocator = None
 
 
 def get_geolocator():
+    """
+    Provides a function to instantiate or retrieve a global geolocator instance
+    using the GeolocationService class. If the geolocator has already been
+    created, it will return the original global instance. Otherwise, it initializes
+    a new instance of the GeolocationService with debugging enabled and stores it
+    globally.
+
+    :return: The global instance of the GeolocationService
+    :rtype: GeolocationService
+    """
     global geolocator
     if geolocator is None:
         geolocator = GeolocationService(debug=True)
@@ -15,6 +25,23 @@ def get_geolocator():
 
 
 def get_address_by_coordinates(latitude, longitude, exactly_one=True):
+    """
+    Retrieves the address based on the provided geographic coordinates (latitude and
+    longitude). Utilizes the geopy library's geolocator to find and reverse-geocode
+    the location associated with the given coordinates. Returns a human-readable
+    address if available or an error message for specific conditions.
+
+    :param latitude: The latitude of the location to find the address for.
+    :type latitude: float
+    :param longitude: The longitude of the location to find the address for.
+    :type longitude: float
+    :param exactly_one: If true, ensures exactly one result is returned. If false,
+        returns a list of possible matches. Defaults to True.
+    :type exactly_one: bool, optional
+    :return: A string containing the human-readable address of the location or an
+        error message in case of failure.
+    :rtype: str
+    """
     geolocator = get_geolocator()
     try:
         location = geolocator.reverse((latitude, longitude), exactly_one=exactly_one)
@@ -28,10 +55,17 @@ def get_address_by_coordinates(latitude, longitude, exactly_one=True):
 
 def get_coordinates_for_address(address):
     """
-    Geocode an address using a custom Nominatim server.
+    Gets geographical coordinates (latitude and longitude) along with the full formatted
+    address for a given address string. Makes use of a geolocation service to retrieve
+    the data and handles possible exceptions during the process.
+
+    :param address: The address as a string for which coordinates need to be determined.
+    :type address: str
 
-    :param address: The address to geocode.
-    :return: A dictionary with the location's latitude, longitude, and full address, or a message if an error occurs.
+    :return: A dictionary containing the full formatted address, latitude, and longitude
+             if the location is found. Otherwise, returns a string describing an error
+             or that no location was found.
+    :rtype: dict or str
     """
     geolocator = get_geolocator()
     try: