sibi-dst 0.3.40__py3-none-any.whl → 0.3.43__py3-none-any.whl

sibi_dst/osmnx_helper/v2/base_osm_map.py

@@ -0,0 +1,153 @@
+from __future__ import annotations
+
+import html
+from abc import ABC, abstractmethod
+from typing import Optional
+
+import folium
+import geopandas as gpd
+import numpy as np
+import osmnx as ox
+import pandas as pd
+from folium.plugins import Fullscreen
+from networkx import MultiDiGraph
+
+
+class BaseOsmMap(ABC):
+    # Define available tile options for the map
+    tile_options = {
+        "OpenStreetMap": "OpenStreetMap",
+        "CartoDB": "cartodbpositron",
+        "CartoDB Voyager": "cartodbvoyager"
+    }
+    # Default geographical bounds (Costa Rica)
+    bounds = [[8.0340, -85.9417], [11.2192, -82.5566]]
+
+    def __init__(
+            self,
+            osmnx_graph: MultiDiGraph,
+            df: pd.DataFrame,
+            lat_col: str = "latitude",
+            lon_col: str = "longitude",
+            map_html_title: str = "OSM Basemap",
+            zoom_start: int = 13,
+            fullscreen: bool = True,
+            fullscreen_position: str = "topright",
+            tiles: str = "OpenStreetMap",
+            verbose: bool = False,
+            sort_keys: Optional[list[str]] = None,
+            dt_field: Optional[str] = None,
+            calc_nearest_nodes: bool = False,
+            max_bounds: bool = False,
+    ):
+        if df.empty:
+            raise ValueError("df must not be empty")
+
+        # Store attributes
+        self.df = df.copy()
+        self.osmnx_graph = osmnx_graph
+        self.lat_col = lat_col
+        self.lon_col = lon_col
+        self.map_html_title = self._sanitize_html(map_html_title)
+        self.zoom_start = zoom_start
+        self.fullscreen = fullscreen
+        self.fullscreen_position = fullscreen_position
+        self.tiles = tiles
+        self.verbose = verbose
+        self.sort_keys = sort_keys
+        self.dt_field = dt_field
+        self.calc_nearest_nodes = calc_nearest_nodes
+        self.max_bounds = max_bounds
+        self.dt = self.df[self.dt_field].to_list() if self.dt_field else None
+        self.nearest_nodes = None
+        self.G = None
+        self.osm_map = None
+
+        self._prepare_df()
+        self._initialize_map()
+
+    def _prepare_df(self):
+        """Sort and preprocess the DataFrame."""
+        if self.sort_keys:
+            self.df.sort_values(by=self.sort_keys, inplace=True, ignore_index=True)
+        self.gps_points = self.df[[self.lat_col, self.lon_col]].to_numpy()
+
+        # Compute nearest nodes if required
+        if self.calc_nearest_nodes and not self.df.empty:
+            self.nearest_nodes = ox.distance.nearest_nodes(
+                self.osmnx_graph, X=self.df[self.lon_col], Y=self.df[self.lat_col]
+            )
+
+    def _initialize_map(self):
+        """Initialize the folium map centered around the dataset."""
+        if self.gps_points.size == 0:
+            raise ValueError("No valid GPS points available for map initialization")
+
+        center = self.gps_points.mean(axis=0).tolist()
+        if self.osm_map is None:
+            self.osm_map = folium.Map(
+                location=center, zoom_start=self.zoom_start, tiles=self.tiles, max_bounds=self.max_bounds
+            )
+        self.G = self._extract_subgraph(*self._get_bounding_box_from_points())
+
+    def _get_bounding_box_from_points(self, margin: float = 0.001) -> tuple[float, float, float, float]:
+        """Compute bounding box for the dataset with margin."""
+        latitudes, longitudes = self.gps_points[:, 0], self.gps_points[:, 1]
+        return max(latitudes) + margin, min(latitudes) - margin, max(longitudes) + margin, min(longitudes) - margin
+
+    def _extract_subgraph(self, north: float, south: float, east: float, west: float) -> MultiDiGraph:
+        """Extract a subgraph from OSM data within the bounding box."""
+        bbox_poly = gpd.GeoSeries([ox.utils_geo.bbox_to_poly((west, south, east, north))])
+        nodes_gdf = ox.graph_to_gdfs(self.osmnx_graph, nodes=True, edges=False)
+        nodes_within_bbox = gpd.sjoin(nodes_gdf, gpd.GeoDataFrame(geometry=bbox_poly), predicate="within")
+        return self.osmnx_graph.subgraph(nodes_within_bbox.index)
+
+    def _post_process_map(self):
+        """Perform final adjustments to the map."""
+        self._attach_supported_tiles()
+        self.add_tile_layer()
+        self._add_fullscreen()
+        self._add_map_title()
+        if self.max_bounds and self.bounds:
+            self.osm_map.fit_bounds(self.bounds)
+
+    def _attach_supported_tiles(self):
+        """Attach additional tile layers to the map."""
+        for name, tile in self.tile_options.items():
+            if tile.lower() != self.tiles.lower():
+                folium.TileLayer(name=name, tiles=tile, show=False).add_to(self.osm_map)
+
+    def _add_fullscreen(self):
+        """Enable fullscreen control if required."""
+        if self.fullscreen:
+            Fullscreen(position=self.fullscreen_position).add_to(self.osm_map)
+
+    def _add_map_title(self):
+        """Add a title to the map if provided."""
+        if self.map_html_title:
+            self.osm_map.get_root().html.add_child(folium.Element(self.map_html_title))
+
+    @staticmethod
+    def _sanitize_html(input_html: str) -> str:
+        """Sanitize HTML input to prevent script injection."""
+        return html.escape(input_html)
+
+    @abstractmethod
+    def process_map(self):
+        """Abstract method to define map processing logic in subclasses."""
+        pass
+
+    def pre_process_map(self):
+        """Optional preprocessing step before main processing."""
+        pass
+
+    def add_tile_layer(self):
+        """Add a layer control to the map."""
+        folium.LayerControl().add_to(self.osm_map)
+
+    def generate_map(self) -> folium.Map:
+        """Generate and return the processed map."""
+        self.pre_process_map()
+        self.process_map()
+        self._post_process_map()
+        return self.osm_map

sibi_dst/utils/__init__.py

@@ -4,6 +4,7 @@ from .log_utils import Logger
 from .date_utils import *
 from .data_utils import DataUtils
 from .file_utils import FileUtils
+from .phone_formatter import PhoneNumberFormatter
 from .filepath_generator import FilePathGenerator
 from .df_utils import DfUtils
 from .storage_manager import StorageManager
@@ -18,8 +19,10 @@ __all__ = [
     "ConfigManager",
     "ConfigLoader",
     "DateUtils",
+    "FileAgeChecker",
     "BusinessDays",
     "FileUtils",
+    "PhoneNumberFormatter",
     "DataWrapper",
     "DataUtils",
     "FilePathGenerator",

sibi_dst/utils/data_utils.py

@@ -1,3 +1,5 @@
+from typing import Union, List
+
 import dask.dataframe as dd
 import pandas as pd
 
@@ -23,6 +25,58 @@ class DataUtils:
         self.logger = logger or Logger.default_logger(logger_name=self.__class__.__name__)
         self.debug = kwargs.get('debug', False)
 
+    @staticmethod
+    def _transform_column(series, fill_value, dtype):
+        """
+        Helper method to transform a column by converting it to numeric, filling missing values,
+        and casting to the specified dtype.
+
+        :param series: The column to transform.
+        :type series: pd.Series or dd.Series
+        :param fill_value: Value to replace missing or invalid data.
+        :type fill_value: int or float
+        :param dtype: Target data type for the column.
+        :type dtype: type
+        :return: Transformed column.
+        :rtype: pd.Series or dd.Series
+        """
+        return (
+            pd.to_numeric(series, errors="coerce")  # Convert to numeric, invalid to NaN
+            .fillna(fill_value)  # Replace NaN with fill_value
+            .astype(dtype)  # Convert to target dtype
+        )
+
+    def transform_numeric_columns(self, df: Union[pd.DataFrame, dd.DataFrame], columns: List[str], fill_value=0,
+                                  dtype=int):
+        """
+        Transform specified numeric columns in the DataFrame by converting their data types
+        to the specified dtype and replacing missing values with the given fill_value.
+
+        :param df: DataFrame to be transformed.
+        :type df: pd.DataFrame or dd.DataFrame
+        :param columns: List of column names to transform.
+        :type columns: list[str]
+        :param fill_value: Value to replace missing or invalid data. Default is 0.
+        :type fill_value: int or float
+        :param dtype: Target data type for the columns. Default is int.
+        :type dtype: type
+        :return: Transformed DataFrame.
+        :rtype: pd.DataFrame or dd.DataFrame
+        """
+        if not columns:
+            self.logger.warning("No columns specified.")
+            return df
+
+        self.logger.debug(f"DataFrame type: {type(df)}")
+        columns = [col for col in columns if col in df.columns]
+
+        for col in columns:
+            df[col] = df[col].map_partitions(
+                self._transform_column, fill_value, dtype, meta=(col, dtype)
+            )
+
+        return df
+
     def transform_numeric_cols(self, df, columns, fill_value=0, dtype=int):
         """
         This function transforms the specified numeric columns in the given dataframe by converting
@@ -57,34 +111,21 @@ class DataUtils:
 
         return df
 
-    def transform_boolean_columns(self, df, columns=None):
+    def transform_boolean_columns(self, df: Union[pd.DataFrame, dd.DataFrame], columns: List[str], fill_value=0):
         """
-        Detect if the provided columns in a DataFrame (Pandas or Dask) contain only 0 and 1
-        and convert them to boolean. Detection is performed using a sample.
-
-        Parameters:
-        - df (pandas.DataFrame or dask.dataframe.DataFrame): The DataFrame.
-        - columns (list of str): List of columns to check and transform.
-        - sample_size (int): Number of rows to sample for detection. Ignored for Pandas DataFrames.
+        Convert specified columns in the DataFrame to boolean, replacing missing values with
+        the given fill_value.
 
-        Returns:
-        - pandas.DataFrame or dask.dataframe.DataFrame: Updated DataFrame with transformed boolean columns.
+        :param df: DataFrame to be transformed.
+        :type df: pd.DataFrame or dd.DataFrame
+        :param columns: List of column names to transform.
+        :type columns: list[str]
+        :param fill_value: Value to replace missing or invalid data. Default is 0.
+        :type fill_value: int or float
+        :return: Transformed DataFrame.
+        :rtype: pd.DataFrame or dd.DataFrame
         """
-
-        # Apply transformation to each specified column
-        for col in columns:
-            if col in df.columns:
-                # Replace NaN with 0, then convert to boolean
-                df[col] = df[col].map_partitions(
-                    lambda s: pd.to_numeric(s, errors='coerce')  # Convert to numeric, invalid to NaN
-                    .fillna(0)  # Replace NaN with 0
-                    .astype(int)  # Ensure integer type
-                    .astype(bool),  # Convert to boolean
-                    meta=(col, 'bool')
-                )
-        if self.debug:
-            self.logger.debug(f'Dataframe type:{type(df)}, boolean applied to columns: {columns}')
-        return df
+        return self.transform_numeric_columns(df, columns, fill_value=fill_value, dtype=bool)
 
     def merge_lookup_data(self, classname, df, **kwargs):
         """