sibi-dst 0.3.27__py3-none-any.whl → 0.3.29__py3-none-any.whl

sibi_dst/osmnx_helper/utils.py

@@ -0,0 +1,267 @@
+import math
+import os
+import pickle
+from urllib.parse import urlencode, urlsplit, urlunsplit
+
+import folium
+import geopandas as gpd
+import numpy as np
+import osmnx as ox
+from geopy.distance import geodesic
+
+
+#
+# options = {
+#    'ox_files_save_path': ox_files_save_path,
+#    'network_type': 'drive',
+#    'place': 'Costa Rica',
+#    'files_prefix': 'costa-rica-',
+# }
+# Usage example
+# handler = PBFHandler(**options)
+# handler.load()
+
+
+class PBFHandler:
+    def __init__(self, **kwargs):
+        self.graph = None
+        self.nodes = None
+        self.edges = None
+        self.rebuild = kwargs.setdefault("rebuild", False)
+        self.verbose = kwargs.setdefault("verbose", False)
+        self.place = kwargs.setdefault('place', 'Costa Rica')
+        self.filepath = kwargs.setdefault('ox_files_save_path', "gis_data/")
+        self.file_prefix = kwargs.setdefault('file_prefix', 'costa-rica-')
+        self.network_type = kwargs.setdefault('network_type', 'all')
+        self.graph_file = f"{self.filepath}{self.file_prefix}graph.pkl"
+        self.node_file = f"{self.filepath}{self.file_prefix}nodes.pkl"
+        self.edge_file = f"{self.filepath}{self.file_prefix}edges.pkl"
+
+    def load(self):
+        if self.verbose:
+            print("Loading data...")
+
+        files_to_check = [self.graph_file, self.node_file, self.edge_file]
+
+        if self.rebuild:
+            for file in files_to_check:
+                if os.path.exists(file):
+                    os.remove(file)
+        if not os.path.exists(self.filepath):
+            os.makedirs(self.filepath, exist_ok=True)
+            # self.process_pbf()
+            # self.save_to_pickle()
+        if not all(os.path.exists(f) for f in files_to_check):
+            self.process_pbf()
+            self.save_to_pickle()
+        else:
+            self.load_from_pickle()
+
+        if self.verbose:
+            print("Data loaded successfully.")
+
+    def process_pbf(self):
+        """
+        Load a PBF file and create a graph.
+        """
+        try:
+            if self.verbose:
+                print(f"Processing PBF for {self.place}...")
+
+            self.graph = ox.graph_from_place(self.place, network_type=self.network_type)
+            self.nodes, self.edges = ox.graph_to_gdfs(self.graph)
+
+            if self.verbose:
+                print("PBF processed successfully.")
+        except Exception as e:
+            print(f"Error processing PBF: {e}")
+            raise
+
+    def save_to_pickle(self):
+        """
+        Save the graph, nodes, and edges to pickle files.
+        """
+        try:
+            if self.verbose:
+                print("Saving data to pickle files...")
+
+            data_to_save = {
+                self.graph_file: self.graph,
+                self.node_file: self.nodes,
+                self.edge_file: self.edges
+            }
+
+            for file, data in data_to_save.items():
+                if data is not None:
+                    with open(file, 'wb') as f:
+                        pickle.dump(data, f, pickle.HIGHEST_PROTOCOL)
+
+            if self.verbose:
+                print("Data saved to pickle files successfully.")
+        except Exception as e:
+            print(f"Error saving to pickle: {e}")
+            raise
+
+    def load_from_pickle(self):
+        """
+        Load the graph, nodes, and edges from pickle files.
+        """
+        try:
+            if self.verbose:
+                print("Loading data from pickle files...")
+
+            files_to_load = {
+                self.graph_file: 'graph',
+                self.node_file: 'nodes',
+                self.edge_file: 'edges'
+            }
+
+            for file, attr in files_to_load.items():
+                with open(file, 'rb') as f:
+                    setattr(self, attr, pickle.load(f))
+
+            if self.verbose:
+                print("Data loaded from pickle files successfully.")
+        except Exception as e:
+            print(f"Error loading from pickle: {e}")
+            raise
+
+    def plot_graph(self):
+        """
+        Plot the graph.
+        """
+        try:
+            if self.graph is not None:
+                if self.verbose:
+                    print("Plotting the graph...")
+                ox.plot_graph(self.graph)
+                if self.verbose:
+                    print("Graph plotted successfully.")
+            else:
+                print("Graph is not loaded. Please load a PBF file first.")
+        except Exception as e:
+            print(f"Error plotting the graph: {e}")
+            raise
+
+
+def get_bounding_box_from_points(gps_points, margin=0.001):
+    latitudes = [point[0] for point in gps_points]
+    longitudes = [point[1] for point in gps_points]
+
+    north = max(latitudes) + margin
+    south = min(latitudes) - margin
+    east = max(longitudes) + margin
+    west = min(longitudes) - margin
+
+    return north, south, east, west
+
+
+def add_arrows(map_object, locations, color, n_arrows):
+    # Get the number of locations
+    n = len(locations)
+
+    # If there are more than two points...
+    if n > 2:
+        # Add arrows along the path
+        for i in range(0, n - 1, n // n_arrows):
+            # Get the start and end point for this segment
+            start, end = locations[i], locations[i + 1]
+
+            # Calculate the direction in which to place the arrow
+            rotation = -np.arctan2((end[1] - start[1]), (end[0] - start[0])) * 180 / np.pi
+
+            folium.RegularPolygonMarker(location=end,
+                                        fill_color=color,
+                                        number_of_sides=2,
+                                        radius=6,
+                                        rotation=rotation).add_to(map_object)
+    return map_object
+
+
+def extract_subgraph(G, north, south, east, west):
+    # Create a bounding box polygon
+    # from osmnx v2 this is how it is done
+    if ox.__version__ >= '2.0':
+        bbox_poly = gpd.GeoSeries([ox.utils_geo.bbox_to_poly(bbox=(west, south, east, north))])
+    else:
+        bbox_poly = gpd.GeoSeries([ox.utils_geo.bbox_to_poly(north, south, east, west)])
+
+    # Get nodes GeoDataFrame
+    nodes_gdf = ox.graph_to_gdfs(G, nodes=True, edges=False)
+
+    # Find nodes within the bounding box
+    nodes_within_bbox = nodes_gdf[nodes_gdf.geometry.within(bbox_poly.geometry.unary_union)]
+
+    # Create subgraph
+    subgraph = G.subgraph(nodes_within_bbox.index)
+
+    return subgraph
+
+
+def get_distance_between_points(point_a, point_b, unit='km'):
+    if not isinstance(point_a, tuple) or len(point_a) != 2:
+        return 0
+    if not all(isinstance(x, float) and not math.isnan(x) for x in point_a):
+        return 0
+    if not isinstance(point_b, tuple) or len(point_b) != 2:
+        return 0
+    if not all(isinstance(x, float) and not math.isnan(x) for x in point_b):
+        return 0
+    distance = geodesic(point_a, point_b)
+    if unit == 'km':
+        return distance.kilometers
+    elif unit == 'm':
+        return distance.meters
+    elif unit == 'mi':
+        return distance.miles
+    else:
+        return 0
+
+
+tile_options = {
+    "OpenStreetMap": "OpenStreetMap",
+    "CartoDB": "cartodbpositron",
+    "CartoDB Voyager": "cartodbvoyager"
+}
+
+
+def attach_supported_tiles(map_object, default_tile="OpenStreetMap"):
+    # Normalize the default tile name to lowercase for comparison
+    normalized_default_tile = default_tile.lower()
+
+    # Filter out the default tile layer from the options to avoid duplication
+    tile_options_filtered = {k: v for k, v in tile_options.items() if v.lower() != normalized_default_tile}
+
+    for tile, description in tile_options_filtered.items():
+        folium.TileLayer(name=tile, tiles=description, show=False).add_to(map_object)
+
+
+def get_graph(**options):
+    handler = PBFHandler(**options)
+    handler.load()
+    return handler.graph, handler.nodes, handler.edges
+
+
+def add_query_params(url, params):
+    # Parse the original URL
+    url_components = urlsplit(url)
+
+    # Parse original query parameters and update with new params
+    original_params = dict([tuple(pair.split('=')) for pair in url_components.query.split('&') if pair])
+    original_params.update(params)
+
+    # Construct the new query string
+    new_query_string = urlencode(original_params)
+
+    # Construct the new URL
+    new_url = urlunsplit((
+        url_components.scheme,
+        url_components.netloc,
+        url_components.path,
+        new_query_string,
+        url_components.fragment
+    ))
+
+    return new_url
+
+

sibi_dst/tests/test_data_wrapper_class.py

@@ -0,0 +1,78 @@
+import unittest
+from unittest.mock import patch, MagicMock
+import datetime
+import pandas as pd
+from sibi_dst.utils import Logger, ParquetSaver
+from sibi_dst.utils.data_wrapper import DataWrapper
+
+
+class TestDataWrapper(unittest.TestCase):
+
+    def setUp(self):
+        self.dataclass = MagicMock()
+        self.date_field = "created_at"
+        self.data_path = "/path/to/data"
+        #self.data_path = "s3://your-bucket-name/path/to/data"
+        self.parquet_filename = "data.parquet"
+        self.start_date = "2022-01-01"
+        self.end_date = "2022-12-31"
+        self.filesystem_type = "file"
+        self.filesystem_options = {
+            #"key": "your_aws_access_key",
+            #"secret": "your_aws_secret_key",
+            #"client_kwargs": {"endpoint_url": "https://s3.amazonaws.com"}
+        }
+        self.logger = Logger.default_logger(logger_name="TestLogger")
+
+    def test_initialization(self):
+        wrapper = DataWrapper(
+            dataclass=self.dataclass,
+            date_field=self.date_field,
+            data_path=self.data_path,
+            parquet_filename=self.parquet_filename,
+            start_date=self.start_date,
+            end_date=self.end_date,
+            filesystem_type=self.filesystem_type,
+            filesystem_options=self.filesystem_options,
+            logger=self.logger
+        )
+        self.assertEqual(wrapper.dataclass, self.dataclass)
+        self.assertEqual(wrapper.date_field, self.date_field)
+        self.assertEqual(wrapper.data_path, "/path/to/data/")
+        self.assertEqual(wrapper.parquet_filename, self.parquet_filename)
+        self.assertEqual(wrapper.start_date, datetime.date(2022, 1, 1))
+        self.assertEqual(wrapper.end_date, datetime.date(2022, 12, 31))
+        self.assertEqual(wrapper.filesystem_type, self.filesystem_type)
+        self.assertEqual(wrapper.filesystem_options, self.filesystem_options)
+        self.assertEqual(wrapper.logger, self.logger)
+
+    def test_convert_to_date(self):
+        self.assertEqual(DataWrapper.convert_to_date("2022-01-01"), datetime.date(2022, 1, 1))
+        self.assertEqual(DataWrapper.convert_to_date(datetime.date(2022, 1, 1)), datetime.date(2022, 1, 1))
+        with self.assertRaises(ValueError):
+            DataWrapper.convert_to_date("invalid-date")
+
+    @patch('fsspec.filesystem')
+    def test_is_file_older_than(self, mock_filesystem):
+        mock_fs = mock_filesystem.return_value
+        mock_fs.info.return_value = {'mtime': (datetime.datetime.now() - datetime.timedelta(minutes=1500)).timestamp()}
+
+        wrapper = DataWrapper(
+            dataclass=self.dataclass,
+            date_field=self.date_field,
+            data_path=self.data_path,
+            parquet_filename=self.parquet_filename,
+            start_date=self.start_date,
+            end_date=self.end_date,
+            filesystem_type=self.filesystem_type,
+            filesystem_options=self.filesystem_options,
+            logger=self.logger
+        )
+
+        self.assertTrue(wrapper.is_file_older_than("some/file/path"))
+        mock_fs.info.return_value = {'mtime': (datetime.datetime.now() - datetime.timedelta(minutes=1000)).timestamp()}
+        self.assertFalse(wrapper.is_file_older_than("some/file/path"))
+
+
+if __name__ == '__main__':
+    unittest.main()

{sibi_dst-0.3.27.dist-info → sibi_dst-0.3.29.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sibi-dst
-Version: 0.3.27
+Version: 0.3.29
 Summary: Data Science Toolkit
 Author: Luis Valverde
 Author-email: lvalverdeb@gmail.com
@@ -8,6 +8,7 @@ Requires-Python: >=3.11,<4.0
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Requires-Dist: apache-airflow-client (>=2.10.0,<3.0.0)
 Requires-Dist: chardet (>=5.2.0,<6.0.0)
 Requires-Dist: charset-normalizer (>=3.4.0,<4.0.0)
@@ -17,10 +18,13 @@ Requires-Dist: dask-expr (>=1.1.20,<2.0.0)
 Requires-Dist: dask[complete] (>=2024.11.1,<2025.0.0)
 Requires-Dist: django (>=5.1.4,<6.0.0)
 Requires-Dist: djangorestframework (>=3.15.2,<4.0.0)
+Requires-Dist: folium (>=0.19.4,<0.20.0)
+Requires-Dist: geopandas (>=1.0.1,<2.0.0)
 Requires-Dist: httpx (>=0.27.2,<0.28.0)
 Requires-Dist: ipython (>=8.29.0,<9.0.0)
 Requires-Dist: jinja2 (>=3.1.4,<4.0.0)
 Requires-Dist: mysqlclient (>=2.2.6,<3.0.0)
+Requires-Dist: nltk (>=3.9.1,<4.0.0)
 Requires-Dist: openpyxl (>=3.1.5,<4.0.0)
 Requires-Dist: pandas (>=2.2.3,<3.0.0)
 Requires-Dist: paramiko (>=3.5.0,<4.0.0)

{sibi_dst-0.3.27.dist-info → sibi_dst-0.3.29.dist-info}/RECORD

@@ -1,6 +1,6 @@
 sibi_dst/__init__.py,sha256=CLHfzrFNqklNx5uMKAPtbZfkbBbVYR5qsiMro0RTfmA,252
-sibi_dst/df_helper/__init__.py,sha256=aiAu7j1SWDiw3RVI4UJmvLcADP34OfrJTCYpdupPGII,234
-sibi_dst/df_helper/_df_helper.py,sha256=vG-Lb9lj8s5cACTvfYp7JhXt1PajttHVhKYzBWR-9Vc,13953
+sibi_dst/df_helper/__init__.py,sha256=5yzslP6zYYOHsTtAzHnNDXHYjf_T6yW7baxwgtduWqQ,292
+sibi_dst/df_helper/_df_helper.py,sha256=MttqHot8dlHzo4G522JL-z6LOFWYVXqqz06k-4YcvRM,23447
 sibi_dst/df_helper/_parquet_artifact.py,sha256=nx1wTEyrjARpCCPNwBxYiBROee3CSb6c-u7Cpme_tdk,4978
 sibi_dst/df_helper/_parquet_reader.py,sha256=sbe8DsScNT2h6huNsz8mUxVnUGpJeRzbaONZ3u2sQeQ,1685
 sibi_dst/df_helper/backends/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -22,9 +22,21 @@ sibi_dst/df_helper/backends/sqlalchemy/_load_from_db.py,sha256=ML-m_WeTR1_UMgiDR
 sibi_dst/df_helper/backends/sqlalchemy/_sql_model_builder.py,sha256=Bmhh6VvmBfNfBA2JpuEdsYD_193yJ768Si2TvkY9HmU,4405
 sibi_dst/df_helper/core/__init__.py,sha256=o4zDwgVmaijde3oix0ezb6KLxI5QFy-SGUhFTDVFLT4,569
 sibi_dst/df_helper/core/_defaults.py,sha256=eNpHD2sZxir-2xO0b3_V16ryw8YP_5FfpIKK0HNuiN4,7011
-sibi_dst/df_helper/core/_filter_handler.py,sha256=g9FMcB_koT724ggcWt98jow2XgUnmupK_fNhF95W5bQ,10217
+sibi_dst/df_helper/core/_filter_handler.py,sha256=t3uLLJX5hWO_dWKCCz8Dwpc9RZ5PMHBIWkHSELCpFXI,11131
 sibi_dst/df_helper/core/_params_config.py,sha256=Og3GYth0GVWpcOYWZWRy7CZ5PDsg63Nmqo-W7TUrA_0,3503
 sibi_dst/df_helper/core/_query_config.py,sha256=Y8LVSyaKuVkrPluRDkQoOwuXHQxner1pFWG3HPfnDHM,441
+sibi_dst/df_helper/data_cleaner.py,sha256=lkxQoXLvGzXCicFUimnA5nen5qkrO1oxgl_p2Be2o8w,5183
+sibi_dst/geopy_helper/__init__.py,sha256=Q1RJiUZIOlV0QNNLjxZ_2IZS5LqIe5jRbeQkfD1Vm60,112
+sibi_dst/geopy_helper/geo_location_service.py,sha256=l0dV0XuEk-tcWdaOymgN9WulR6xp3k7yJUgqYvnqCKo,2288
+sibi_dst/geopy_helper/utils.py,sha256=R9X6ew0L_SuCpsA_AQK1wd3BspRGtV83q3mhBkcKr4A,1664
+sibi_dst/osmnx_helper/__init__.py,sha256=QeAKEeVXZk_qn8o0d3BOoGgv2lzatcI2yBqY3ZqviKI,153
+sibi_dst/osmnx_helper/base_osm_map.py,sha256=s2OY_XfwjZA3ImJNtCgevGBCbwRVe3dY3QVkTHEulB0,5794
+sibi_dst/osmnx_helper/basemaps/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+sibi_dst/osmnx_helper/basemaps/calendar_html.py,sha256=UArt6FDgoCgoRte45Xo3IHqd-RNzW0YgitgZYfOFasY,4031
+sibi_dst/osmnx_helper/basemaps/router_plotter.py,sha256=QznnBGsUwhl8ZITcVNBrQDm-MXAd0jpJGPuyozKyQg0,8537
+sibi_dst/osmnx_helper/utils.py,sha256=8sF-wNSL38WzhWS3DceZ1cP8BM11i7D0bI-E4XYD8K4,8449
+sibi_dst/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+sibi_dst/tests/test_data_wrapper_class.py,sha256=Nkup5OFH5Cos2fxPaU7g9IEyINJM0uJ5-rOZ-eNtd20,3275
 sibi_dst/utils/__init__.py,sha256=z51o5sjIo_gTjnDXk5SBniCxWJIrDBMS7df0dTs8VMk,775
 sibi_dst/utils/airflow_manager.py,sha256=-d44EKUZNYJyp4wuNwRvilRQktunArPOB5fZuWdQv10,7526
 sibi_dst/utils/clickhouse_writer.py,sha256=xUhFDOuZt0eZDpVJNuLb7pfTHUV06NCYrNUx_a7qrSM,8580
@@ -38,6 +50,6 @@ sibi_dst/utils/filepath_generator.py,sha256=hjI7gQwfwRToPeuzoUQDayHKQrr4Ivhi4Chl
 sibi_dst/utils/log_utils.py,sha256=4eLmoV8VC7wDwPr1mRfDKP24_-laGO6ogE4U0u3DUuA,2315
 sibi_dst/utils/parquet_saver.py,sha256=hLrWr1G132y94eLopDPPGQGDsAiR1lQ8id4QQtGYPE4,4349
 sibi_dst/utils/storage_manager.py,sha256=7nkfeBW_2xlF59pGj7V2aY5TLwpJnPQuPVclqjavJOA,3856
-sibi_dst-0.3.27.dist-info/METADATA,sha256=YFb0ZGbz2m0-aczvItyKK4Iqf1wn6pSVGE41ZUQ6YI8,2265
-sibi_dst-0.3.27.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-sibi_dst-0.3.27.dist-info/RECORD,,
+sibi_dst-0.3.29.dist-info/METADATA,sha256=KsqbOJSoa7LOBN5ibbG-MLCgCSGSI_Ps7LfdLHDPErQ,2436
+sibi_dst-0.3.29.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+sibi_dst-0.3.29.dist-info/RECORD,,

{sibi_dst-0.3.27.dist-info → sibi_dst-0.3.29.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 1.9.0
+Generator: poetry-core 1.9.1
 Root-Is-Purelib: true
 Tag: py3-none-any