sibi-dst 0.3.31__py3-none-any.whl → 0.3.33__py3-none-any.whl

sibi_dst/osmnx_helper/utils.py

@@ -23,6 +23,37 @@ from geopy.distance import geodesic
 
 
 class PBFHandler:
+    """
+    Handles the creation, management, and visualization of graph data derived
+    from .pbf (Protocolbuffer Binary Format) files. This class enables the
+    loading, processing, saving, and reutilization of graph, node, and edge
+    data for geographical regions, supporting verbose mode for detailed outputs.
+
+    :ivar graph: The generated graph object representing the spatial network; can be None if not yet loaded or processed.
+    :type graph: Optional[NetworkX.Graph]
+    :ivar nodes: GeoDataFrame representing the nodes of the graph; can be None if not yet loaded or processed.
+    :type nodes: Optional[geopandas.GeoDataFrame]
+    :ivar edges: GeoDataFrame representing the edges of the graph; can be None if not yet loaded or processed.
+    :type edges: Optional[geopandas.GeoDataFrame]
+    :ivar rebuild: Indicates whether to rebuild the graph data, ignoring any existing cached files. Default is ``False``.
+    :type rebuild: bool
+    :ivar verbose: Enables verbose mode to provide detailed status messages during operations. Default is ``False``.
+    :type verbose: bool
+    :ivar place: The name of the geographical region to process with OpenStreetMap. Default is ``Costa Rica``.
+    :type place: str
+    :ivar filepath: The path to the directory where the graph, nodes, and edges pickle files are saved. Default is ``gis_data/``.
+    :type filepath: str
+    :ivar file_prefix: The prefix for the filenames of the saved graph, node, and edge pickle files. Default is ``costa-rica-``.
+    :type file_prefix: str
+    :ivar network_type: The type of network to extract from OpenStreetMap, such as "all" or other specific network types. Default is ``all``.
+    :type network_type: str
+    :ivar graph_file: Full path of the file to save or load the graph data as a pickle file.
+    :type graph_file: str
+    :ivar node_file: Full path of the file to save or load the graph's node data as a pickle file.
+    :type node_file: str
+    :ivar edge_file: Full path of the file to save or load the graph's edge data as a pickle file.
+    :type edge_file: str
+    """
     def __init__(self, **kwargs):
         self.graph = None
         self.nodes = None
@@ -38,6 +69,23 @@ class PBFHandler:
         self.edge_file = f"{self.filepath}{self.file_prefix}edges.pkl"
 
     def load(self):
+        """
+        Loads the required data files for processing. If the files do not exist or
+        if the `rebuild` flag is set to True, it will process and recreate the
+        necessary data from the source. Otherwise, it will load the data from
+        existing pickle files. This function ensures the target directory exists,
+        and processes files conditionally based on their presence.
+
+        :param verbose: Flag to control the verbosity of the function's output.
+        :param rebuild: Indicates whether the data should be rebuilt from the raw
+            source files.
+        :param graph_file: Path to the graph file to be loaded or rebuilt.
+        :param node_file: Path to the node file to be loaded or rebuilt.
+        :param edge_file: Path to the edge file to be loaded or rebuilt.
+        :param filepath: Path to the directory where files are processed and saved.
+
+        :return: None
+        """
         if self.verbose:
             print("Loading data...")
 
@@ -62,7 +110,31 @@ class PBFHandler:
 
     def process_pbf(self):
         """
-        Load a PBF file and create a graph.
+        Processes the Protocolbuffer Binary Format (PBF) data specified for a given place by
+        utilizing the OSMnx library to create a graph representation and extracts nodes and
+        edges into GeoDataFrames. The function provides verbose output if enabled.
+
+        :param self: Refers to the current instance of the class containing this method.
+
+        :param self.verbose: bool
+            A flag to control verbose output. If True, detailed processing status messages are
+            logged to the console.
+
+        :param self.place: str
+            The name or description of the geographic place for which PBF data is processed. It
+            is used to construct a graph representation of the place.
+
+        :param self.network_type: str
+            The type of network graph to be created, typically one of 'all', 'walk', 'drive',
+            etc., reflecting the type of paths or streets included in the graph.
+
+        :return: None
+            This function does not return a value, but updates class attributes ``graph``,
+            ``nodes``, and ``edges``.
+
+        :raises Exception:
+            Raises a general exception when there is an error in processing the PBF data. Error
+            details are printed when verbose output is enabled.
         """
         try:
             if self.verbose:
@@ -79,7 +151,20 @@ class PBFHandler:
 
     def save_to_pickle(self):
         """
-        Save the graph, nodes, and edges to pickle files.
+        Saves data, including graph, nodes, and edges, to pickle files. Each data object is
+        saved to its corresponding file if available. If verbose mode is enabled, prints
+        messages indicating the saving progress and success.
+
+        :param self:
+            Represents the instance of the class that contains attributes `graph_file`,
+            `graph`, `node_file`, `nodes`, `edge_file`, `edges`, and `verbose`. These
+            attributes determine the files to save to and the data to save.
+
+        :raises Exception:
+            Raises an exception if an error occurs during the saving process.
+
+        :return:
+            None
         """
         try:
             if self.verbose:
@@ -104,7 +189,13 @@ class PBFHandler:
 
     def load_from_pickle(self):
         """
-        Load the graph, nodes, and edges from pickle files.
+        Loads data from pickle files specified by the attributes `graph_file`, `node_file`,
+        and `edge_file` and assigns them to the corresponding attributes `graph`,
+        `nodes`, and `edges`, respectively. Displays verbose messages during the load
+        process if the `verbose` attribute is set to True.
+
+        :raises Exception: If an error occurs during reading or deserialization of the
+                           pickle files.
         """
         try:
             if self.verbose:
@@ -128,7 +219,13 @@ class PBFHandler:
 
     def plot_graph(self):
         """
-        Plot the graph.
+        Plots the loaded graph using the OSMnx library.
+
+        This method checks if a graph is loaded and, if available, plots it. Outputs
+        verbose messages during the process if verbosity is enabled.
+
+        :raises Exception: Raises if an error occurs during the plotting process.
+        :return: None
         """
         try:
             if self.graph is not None:
@@ -145,6 +242,23 @@ class PBFHandler:
 
 
 def get_bounding_box_from_points(gps_points, margin=0.001):
+    """
+    Calculates a bounding box from a list of GPS points, with an optional margin added
+    to expand the bounding box in all directions. The function iterates over the GPS
+    points to determine the maximum and minimum latitude and longitude values, then
+    applies the specified margin to calculate the bounding box's boundaries.
+
+    :param gps_points: A list of GPS points, where each point is represented as a tuple
+        containing a latitude and a longitude (latitude, longitude).
+    :type gps_points: list[tuple[float, float]]
+    :param margin: An optional margin value to expand the bounding box in all directions.
+        Default value is 0.001.
+    :type margin: float
+    :return: A tuple containing the bounding box boundaries in the following order:
+        north (maximum latitude), south (minimum latitude), east (maximum longitude),
+        and west (minimum longitude), each adjusted with the margin.
+    :rtype: tuple[float, float, float, float]
+    """
     latitudes = [point[0] for point in gps_points]
     longitudes = [point[1] for point in gps_points]
 
@@ -157,6 +271,28 @@ def get_bounding_box_from_points(gps_points, margin=0.001):
 
 
 def add_arrows(map_object, locations, color, n_arrows):
+    """
+    Adds directional arrows to a map object to indicate paths or flows along a polyline
+    defined by the given locations.
+
+    The function computes directional arrows based on the locations list, places them
+    along the defined path at intervals determined by the number of arrows, and adds
+    these arrows to the specified `map_object`.
+
+    .. note::
+        The function works optimally when the number of locations is greater than two.
+
+    :param map_object: The folium map object to which the directional arrows will be added.
+    :param locations: A list containing tuples of latitude and longitude values that define
+        the polyline. Each tuple represents a geographic point.
+    :type locations: list[tuple[float, float]]
+    :param color: The color to be used for the directional arrows.
+    :type color: str
+    :param n_arrows: The number of arrows to be drawn along the path.
+    :type n_arrows: int
+    :return: The modified folium map object containing the added arrows.
+    :rtype: folium.Map
+    """
     # Get the number of locations
     n = len(locations)
 
@@ -179,6 +315,26 @@ def add_arrows(map_object, locations, color, n_arrows):
 
 
 def extract_subgraph(G, north, south, east, west):
+    """
+    Extracts a subgraph from the input graph `G` within a specified bounding box. The bounding
+    box is defined by its north, south, east, and west coordinates. The function identifies
+    nodes from the graph that lie within this bounding box and creates a subgraph containing
+    only these nodes and their corresponding edges.
+
+    :param G: The input graph representing the original main graph.
+    :type G: networkx.Graph
+    :param north: The northern latitude that defines the upper boundary of the bounding box.
+    :type north: float
+    :param south: The southern latitude that defines the lower boundary of the bounding box.
+    :type south: float
+    :param east: The eastern longitude that defines the right boundary of the bounding box.
+    :type east: float
+    :param west: The western longitude that defines the left boundary of the bounding box.
+    :type west: float
+    :return: A subgraph extracted from the input graph `G` containing nodes and edges within
+             the specified bounding box.
+    :rtype: networkx.Graph
+    """
     # Create a bounding box polygon
     # from osmnx v2 this is how it is done
     if ox.__version__ >= '2.0':
@@ -199,6 +355,26 @@ def extract_subgraph(G, north, south, east, west):
 
 
 def get_distance_between_points(point_a, point_b, unit='km'):
+    """
+    Calculate the geographical distance between two points on Earth.
+
+    This function computes the distance between two points on the Earth's surface
+    specified in their geographical coordinates (latitude, longitude). The calculation
+    employs the geodesic distance, which represents the shortest distance between
+    two points on the Earth's surface. The distance can be returned in different units of
+    measurement depending on the provided parameter.
+
+    :param point_a: A tuple representing the latitude and longitude of the first
+        point in decimal degrees (e.g., (latitude, longitude)). Must be a tuple of
+        two float values.
+    :param point_b: A tuple representing the latitude and longitude of the second
+        point in decimal degrees (e.g., (latitude, longitude)). Must be a tuple of
+        two float values.
+    :param unit: A string value representing the unit of the calculated distance. Can be
+        'km' for kilometers (default), 'm' for meters, or 'mi' for miles.
+    :return: A float value of the distance between the two points in the specified unit.
+        Returns 0 if the input validation fails or the specified unit is invalid.
+    """
     if not isinstance(point_a, tuple) or len(point_a) != 2:
         return 0
     if not all(isinstance(x, float) and not math.isnan(x) for x in point_a):
@@ -226,6 +402,20 @@ tile_options = {
 
 
 def attach_supported_tiles(map_object, default_tile="OpenStreetMap"):
+    """
+    Attaches supported tile layers to a given folium map object, excluding the
+    default tile layer, to provide layer selection functionality in the map.
+
+    This function allows dynamic addition of multiple tile layers to the map
+    object while avoiding duplication of the default tile. By filtering out the
+    default tile, it prevents redundancy and ensures a cleaner map interface.
+
+    :param map_object: The folium map object to which the tile layers will be added.
+        It must be an instance of Folium's Map class or a compatible map object.
+    :param default_tile: The name of the default tile layer to exclude from the
+        list of tiles added to the map. If not specified, defaults to 'OpenStreetMap'.
+    :return: None. The function modifies the provided map object in place.
+    """
     # Normalize the default tile name to lowercase for comparison
     normalized_default_tile = default_tile.lower()
 
@@ -237,12 +427,44 @@ def attach_supported_tiles(map_object, default_tile="OpenStreetMap"):
 
 
 def get_graph(**options):
+    """
+    Generates and returns a graph along with its nodes and edges based on the
+    provided options. The function initializes a PBFHandler instance with the
+    given options, processes any data required, and retrieves the resulting
+    graph structure.
+
+    :param options: Variable-length keyword arguments passed to initialize the
+                    PBFHandler instance. These parameters play a role in
+                    determining how the graph data is processed and structured.
+    :return: Returns a tuple containing three elements:
+             - The generated graph object
+             - The list or collection of nodes within the graph
+             - The list or collection of edges that describe relationships
+               between nodes in the graph
+    """
     handler = PBFHandler(**options)
     handler.load()
     return handler.graph, handler.nodes, handler.edges
 
 
 def add_query_params(url, params):
+    """
+    Update the query parameters of a given URL with new parameters.
+
+    This function takes a URL and a dictionary of parameters, merges these
+    parameters with the existing parameters in the URL, and returns a new URL
+    with updated query parameters.
+
+    :param url: The original URL whose query parameters are to be updated,
+        including the scheme, netloc, path, and optional query string and fragment.
+    :type url: str
+    :param params: A dictionary containing the new parameters to be added or updated
+        in the query string of the given URL.
+    :type params: dict
+    :return: A new URL with updated query parameters after merging the original
+        and new parameters.
+    :rtype: str
+    """
     # Parse the original URL
     url_components = urlsplit(url)
 

sibi_dst/utils/clickhouse_writer.py

@@ -9,6 +9,33 @@ from sibi_dst.utils import Logger
 
 
 class ClickHouseWriter:
+    """
+    Provides functionality to write a Dask DataFrame to a ClickHouse database using
+    a specified schema. This class handles the creation of tables, schema generation,
+    data transformation, and data insertion. It ensures compatibility between Dask
+    data types and ClickHouse types.
+
+    :ivar clickhouse_host: Host address of the ClickHouse database.
+    :type clickhouse_host: str
+    :ivar clickhouse_port: Port of the ClickHouse database.
+    :type clickhouse_port: int
+    :ivar clickhouse_dbname: Name of the database to connect to in ClickHouse.
+    :type clickhouse_dbname: str
+    :ivar clickhouse_user: Username for database authentication.
+    :type clickhouse_user: str
+    :ivar clickhouse_password: Password for database authentication.
+    :type clickhouse_password: str
+    :ivar clickhouse_table: Name of the table to store the data in.
+    :type clickhouse_table: str
+    :ivar logger: Logger instance for logging messages.
+    :type logger: logging.Logger
+    :ivar client: Instance of the ClickHouse database client.
+    :type client: clickhouse_connect.Client or None
+    :ivar df: Dask DataFrame to be written into ClickHouse.
+    :type df: dask.dataframe.DataFrame
+    :ivar order_by: Field or column name to use for table ordering.
+    :type order_by: str
+    """
     dtype_to_clickhouse = {
         'int64': 'Int64',
         'int32': 'Int32',

sibi_dst/utils/data_utils.py

@@ -5,12 +5,43 @@ from sibi_dst.utils import Logger
 
 
 class DataUtils:
-
+    """
+    Utility class for data transformation, manipulation, and merging.
+
+    This class provides functionalities for transforming numeric and boolean columns, merging
+    lookup data, checking DataFrame emptiness, and converting columns to datetime format in
+    Pandas or Dask DataFrames. It is designed to handle data preprocessing steps efficiently
+    for both small-scale and large-scale datasets. Logging and debug options are available
+    to trace execution and monitor operations.
+
+    :ivar logger: Logger instance for logging messages.
+    :type logger: logging.Logger
+    :ivar debug: Flag to enable or disable debug mode.
+    :type debug: bool
+    """
     def __init__(self, logger=None, **kwargs):
         self.logger = logger or Logger.default_logger(logger_name=self.__class__.__name__)
         self.debug = kwargs.get('debug', False)
 
     def transform_numeric_cols(self, df, columns, fill_value=0, dtype=int):
+        """
+        This function transforms the specified numeric columns in the given dataframe by converting
+        their data types to the specified dtype, with an optional parameter for replacing missing
+        values. It first checks if the provided columns exist in the dataframe, processes each column
+        to replace non-numeric values with NaN, fills NaN values with the given fill_value, and finally
+        converts the column to the specified dtype.
+
+        :param df: DataFrame to be transformed.
+        :type df: dask.dataframe.DataFrame
+        :param columns: List of column names to be transformed.
+        :type columns: list[str]
+        :param fill_value: Value used to replace missing or invalid data. Default is 0.
+        :type fill_value: int or float
+        :param dtype: Target data type for the columns after transformation. Default is int.
+        :type dtype: type
+        :return: Transformed dataframe with the specified numeric columns converted and modified.
+        :rtype: dask.dataframe.DataFrame
+        """
         if not columns:
             self.logger.warning('No columns specified')
         self.logger.debug(f'Dataframe type:{type(df)}')

sibi_dst/utils/data_wrapper.py

@@ -12,6 +12,62 @@ from sibi_dst.utils import ParquetSaver
 
 
 class DataWrapper:
+    """
+    Utility class for handling file-based operations, including processing and saving data
+    in Parquet format, while managing a hierarchy of conditions such as overwrite, history
+    threshold, and missing file detection.
+
+    This class aims to simplify the process of managing large datasets stored in a filesystem.
+    It allows for controlled updates to data files based on parameters set by the user, with
+    support for different filesystem types and options.
+
+    It also provides features like logging actions, managing processing threads, generating
+    update plans, checking file age, and dynamically creating date ranges for data operations.
+
+    The design supports flexible integration with user-defined classes (dataclasses) to define
+    custom loading and processing behavior.
+
+    :ivar dataclass: The user-defined class for data processing.
+    :type dataclass: Type
+    :ivar date_field: The name of the date field in the user-defined class.
+    :type date_field: str
+    :ivar data_path: Base path for the dataset storage.
+    :type data_path: str
+    :ivar parquet_filename: File name for the Parquet file.
+    :type parquet_filename: str
+    :ivar start_date: Start date for processing.
+    :type start_date: datetime.date
+    :ivar end_date: End date for processing.
+    :type end_date: datetime.date
+    :ivar fs: File system object for managing files.
+    :type fs: Optional[fsspec.AbstractFileSystem]
+    :ivar filesystem_type: Type of the filesystem (e.g., "file", "s3").
+    :type filesystem_type: str
+    :ivar filesystem_options: Additional options for initializing the filesystem.
+    :type filesystem_options: Optional[Dict]
+    :ivar verbose: Flag to enable verbose logging.
+    :type verbose: bool
+    :ivar class_params: Parameters to initialize the dataclass.
+    :type class_params: Optional[Dict]
+    :ivar load_params: Additional parameters for loading functions.
+    :type load_params: Optional[Dict]
+    :ivar reverse_order: Flag to reverse the order of date range generation.
+    :type reverse_order: bool
+    :ivar overwrite: Whether to overwrite all files during processing.
+    :type overwrite: bool
+    :ivar ignore_missing: Whether to ignore missing files.
+    :type ignore_missing: bool
+    :ivar logger: Logger instance for logging information.
+    :type logger: Optional[Logger]
+    :ivar max_age_minutes: Maximum file age threshold in minutes.
+    :type max_age_minutes: int
+    :ivar history_days_threshold: Number of days for the history threshold.
+    :type history_days_threshold: int
+    :ivar show_progress: Flag to enable progress display.
+    :type show_progress: bool
+    :ivar timeout: Timeout in seconds for processing tasks with threads.
+    :type timeout: Optional[int]
+    """
     DEFAULT_MAX_AGE_MINUTES = 1440
     DEFAULT_HISTORY_DAYS_THRESHOLD = 30
 
@@ -80,7 +136,19 @@ class DataWrapper:
             yield date.date()
 
     def process(self):
-        """Execute the update plan using 'update_priority' to determine processing order."""
+        """
+        Processes update tasks by generating an update plan, filtering required updates, and distributing
+        the workload across threads based on priority levels.
+
+        This method operates by assessing required updates through generated conditions,
+        grouping them by priority levels, and processing them in parallel threads.
+        Each thread handles the updates for a specific priority level, ensuring a streamlined approach
+        to handling the updates efficiently.
+
+        :raises TimeoutError: If a thread processing a priority level exceeds the allowed timeout duration.
+
+        :return: None
+        """
         update_plan_table = self.generate_update_plan_with_conditions()
 
         # Display the update plan table to the user if requested
@@ -171,7 +239,20 @@ class DataWrapper:
             return True  #
 
     def process_date(self, date: datetime.date):
-        """Process a specific date by regenerating data as necessary."""
+        """
+        Processes data for a given date and saves it as a Parquet file.
+
+        This method processes data for the specified date by loading the data
+        corresponding to that day, saving it into a structured storage format
+        (Parquet), and logging relevant information such as processing time
+        and errors that may occur during the process. It uses provided
+        dataclass and parameters to operate and ensures the data is stored
+        in a structured folder hierarchy.
+
+        :param date: The specific date for which data processing and saving should occur
+        :type date: datetime.date
+        :return: None
+        """
         folder = f'{self.data_path}{date.year}/{date.month:02d}/{date.day:02d}/'
         full_parquet_filename = f"{folder}{self.parquet_filename}"
 
@@ -196,10 +277,17 @@ class DataWrapper:
 
     def generate_update_plan_with_conditions(self):
         """
-        Generate an update plan that evaluates files based on the specified hierarchy:
-        1. Overwrite (all files regenerated).
-        2. History threshold: Files within `history_days_threshold` are evaluated for `max_age_minutes`.
-        3. Missing files: Detect missing files, ignoring future dates.
+        Generates an update plan for data files based on specific conditions. The function evaluates the need for updating or
+        overwriting data files for a given date range. Conditions include file existence, whether the file falls within a
+        specified historical threshold, and the necessity to overwrite or handle missing files. A priority map is utilized to
+        assign priority levels to update categories.
+
+        :raises FileNotFoundError: If any file is referenced that does not exist and the ``ignore_missing`` property is set to False.
+        :raises AttributeError: If any required attribute like ``fs``, ``dataclass``, or others are not properly set or initialized.
+
+        :return: A Pandas DataFrame representing the update plan, where each row contains information about a date, the conditions
+            evaluated for that date, and the determined update priority.
+        :rtype: pandas.DataFrame
         """
         rows = []
 

sibi_dst/utils/date_utils.py

@@ -8,6 +8,24 @@ from sibi_dst.utils import Logger
 
 
 class DateUtils:
+    """
+    Utility class for date-related operations.
+
+    The DateUtils class provides a variety of operations to manipulate and retrieve
+    information about dates, such as calculating week ranges, determining start or
+    end dates for specific periods (quarters, months, years), and dynamically
+    registering custom time period functions. It also supports parsing specific
+    periods for date range computations and ensuring the input date is correctly
+    converted to the desired format.
+
+    :ivar logger: Logger instance used for logging messages. Defaults to the logger
+                  for the current class if not provided.
+    :type logger: Logger
+
+    :ivar _PERIOD_FUNCTIONS: Stores dynamically registered period functions that
+                             return start and end dates.
+    :type _PERIOD_FUNCTIONS: Dict[str, Callable[[], Tuple[datetime.date, datetime.date]]]
+    """
     _PERIOD_FUNCTIONS: Dict[str, Callable[[], Tuple[datetime.date, datetime.date]]] = {}
 
     def __init__(self, logger=None):
@@ -127,6 +145,23 @@ class DateUtils:
 
 
 class BusinessDays:
+    """
+    Provides functionality for handling business days calculations with a custom
+    holiday list. The class includes methods for calculating the number of
+    business days, modifying dates by adding business days, and applying these
+    operations to Dask DataFrames.
+
+    :ivar logger: Logger instance for logging error, warning, and debug messages.
+    :type logger: logging.Logger
+    :ivar HOLIDAY_LIST: Dictionary mapping years to lists of holiday dates.
+    :type HOLIDAY_LIST: dict
+    :ivar bd_cal: Numpy busdaycalendar object containing holidays and week mask.
+    :type bd_cal: numpy.busdaycalendar
+    :ivar holidays: Array of holiday dates used by the business day calendar.
+    :type holidays: numpy.ndarray
+    :ivar week_mask: Boolean array indicating working days within a week.
+    :type week_mask: numpy.ndarray
+    """
     def __init__(self, holiday_list, logger):
         """
         Initialize a BusinessDays object with a given holiday list.

sibi_dst/utils/log_utils.py

@@ -1,11 +1,28 @@
-#  Copyright (c) 2023. ISTMO Center S.A.  All Rights Reserved
-#
 import logging
 import os
 import sys
 
 
 class Logger:
+    """
+    Handles the creation, setup, and management of logging functionalities.
+
+    This class facilitates logging by creating and managing a logger instance with
+    customizable logging directory, name, and file. It ensures logs from a script
+    are stored in a well-defined directory and file, and provides various logging
+    methods for different log levels. The logger automatically formats and handles
+    log messages. Additionally, this class provides a class method to initialize a
+    logger with default behaviors.
+
+    :ivar log_dir: Path to the directory where log files are stored.
+    :type log_dir: str
+    :ivar logger_name: Name of the logger instance.
+    :type logger_name: str
+    :ivar log_file: Base name of the log file.
+    :type log_file: str
+    :ivar logger: The initialized logger instance used for logging messages.
+    :type logger: logging.Logger
+    """
     def __init__(self, log_dir, logger_name, log_file):
         self.log_dir = log_dir
         self.logger_name = logger_name

sibi_dst/utils/parquet_saver.py

@@ -38,6 +38,7 @@ class ParquetSaver:
         schema = self._define_schema()
         self._convert_dtypes(schema)
         self._save_dataframe_to_parquet(full_path, schema)
+        self.fs.close()
 
     def _define_schema(self) -> pa.Schema:
         """Define a PyArrow schema dynamically based on df_result column types."""

sibi_dst/utils/storage_manager.py

@@ -90,4 +90,7 @@ class StorageManager:
         """
         print("Rebuilding depot structure...")
         self.rebuild_depot_paths(depots, clear_existing=clear_existing)
-        print("Rebuild complete.")
+        print("Rebuild complete.")
+
+    def get_fs_instance(self):
+        return fsspec.filesystem(self.fs_type, **self.fs_options)

{sibi_dst-0.3.31.dist-info → sibi_dst-0.3.33.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sibi-dst
-Version: 0.3.31
+Version: 0.3.33
 Summary: Data Science Toolkit
 Author: Luis Valverde
 Author-email: lvalverdeb@gmail.com
@@ -20,6 +20,7 @@ Requires-Dist: django (>=5.1.4,<6.0.0)
 Requires-Dist: djangorestframework (>=3.15.2,<4.0.0)
 Requires-Dist: folium (>=0.19.4,<0.20.0)
 Requires-Dist: geopandas (>=1.0.1,<2.0.0)
+Requires-Dist: gunicorn (>=23.0.0,<24.0.0)
 Requires-Dist: httpx (>=0.27.2,<0.28.0)
 Requires-Dist: ipython (>=8.29.0,<9.0.0)
 Requires-Dist: jinja2 (>=3.1.4,<4.0.0)
@@ -42,6 +43,7 @@ Requires-Dist: sqlalchemy (>=2.0.36,<3.0.0)
 Requires-Dist: tornado (>=6.4.1,<7.0.0)
 Requires-Dist: tqdm (>=4.67.0,<5.0.0)
 Requires-Dist: uvicorn (>=0.34.0,<0.35.0)
+Requires-Dist: uvicorn-worker (>=0.3.0,<0.4.0)
 Description-Content-Type: text/markdown
 
 # sibi-dst