sibi-dst 0.3.26__py3-none-any.whl → 0.3.28__py3-none-any.whl

sibi_dst/osmnx_helper/basemaps/calendar_html.py

@@ -0,0 +1,122 @@
+
+# HTML and CSS for the calendar button and popup
+calendar_html = """
+<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/flatpickr/dist/flatpickr.min.css">
+<script src="https://cdn.jsdelivr.net/npm/flatpickr"></script>
+
+<style>
+  /* Style for the calendar button */
+  .calendar-btn {
+    background-color: white;
+    border: 1px solid gray;
+    border-radius: 3px;
+    padding: 5px;
+    font-size: 16px;
+    cursor: pointer;
+    position: fixed;  /* Changed from absolute to fixed */
+    bottom: 50px;     /* Adjust position relative to the viewport */
+    left: 10px;       /* Adjust position relative to the viewport */
+    z-index: 10000;   /* Ensure it stays on top of other elements */
+  }
+
+  /* Calendar popup with sufficient size */
+  .calendar-popup {
+    display: none;
+    position: fixed;  /* Keep the popup fixed so it stays in view */
+    bottom: 100px;
+    left: 10px;
+    background-color: white;
+    padding: 10px;
+    border: 1px solid gray;
+    border-radius: 3px;
+    z-index: 10000;   /* Ensure it stays on top of other elements */
+    width: 250px;
+    height: 300px;
+  }
+
+  /* Ensure the calendar fits properly */
+  #calendar {
+    width: 100%;
+    height: auto;
+  }
+</style>
+
+<!-- Calendar Button -->
+<div class="calendar-btn">📅 Select Date</div>
+
+<!-- Calendar Popup -->
+<div class="calendar-popup" id="calendar-popup">
+  <div id="calendar"></div>
+</div>
+
+<script>
+  // Initialize Flatpickr calendar
+  const today = new Date().toISOString().split('T')[0];
+  // Function to show the "Please wait" message
+  function showLoadingMessage() {
+    let loadingMessage = document.createElement("div");
+    loadingMessage.id = "loading-message";
+    loadingMessage.style.position = "fixed";
+    loadingMessage.style.top = "50%";
+    loadingMessage.style.left = "50%";
+    loadingMessage.style.transform = "translate(-50%, -50%)";
+    loadingMessage.style.backgroundColor = "rgba(0, 0, 0, 0.8)";
+    loadingMessage.style.color = "white";
+    loadingMessage.style.padding = "20px";
+    loadingMessage.style.borderRadius = "5px";
+    loadingMessage.style.zIndex = "9999";
+    loadingMessage.innerText = "Please wait...";
+    document.body.appendChild(loadingMessage);
+  }
+
+  // Function to remove the "Please wait" message
+  function removeLoadingMessage() {
+    let loadingMessage = document.getElementById("loading-message");
+    if (loadingMessage) {
+      loadingMessage.remove();
+    }
+  }
+
+
+  flatpickr("#calendar", {
+    inline: true,  // Render the calendar inline within the container
+    maxDate: today,  // Disable future dates
+    onChange: function(selectedDates, dateStr, instance) {
+      console.log("Selected date: " + dateStr);  // Debugging: Log the selected date
+      // Get the current URL and create a URL object to manipulate the query parameters
+      // Get the current URL from the parent window
+      showLoadingMessage();
+      let currentUrl = window.parent.location.href;
+
+      // If the URL contains "srcdoc", remove it and use the correct base path
+      if (currentUrl.includes("srcdoc")) {
+        currentUrl = currentUrl.replace("srcdoc", "");
+      }
+
+      const url = new URL(currentUrl);
+
+      // Set or update the 'date' parameter while preserving existing parameters
+      url.searchParams.set('date', dateStr);
+
+      console.log("Updated URL: " + url.toString());  // Debugging: Log the updated URL
+
+      // Update the parent window's location with the new URL
+      window.parent.location.href = url.toString();
+    }
+  });
+  // Remove the "Please wait" message once the page has finished loading
+  window.addEventListener("load", function() {
+    removeLoadingMessage();
+  });
+
+  // Toggle the calendar popup when the button is clicked
+  document.querySelector(".calendar-btn").addEventListener("click", function() {
+    var popup = document.getElementById("calendar-popup");
+    if (popup.style.display === "none" || popup.style.display === "") {
+      popup.style.display = "block";
+    } else {
+      popup.style.display = "none";
+    }
+  });
+</script>
+"""

sibi_dst/osmnx_helper/basemaps/router_plotter.py

@@ -0,0 +1,186 @@
+from __future__ import annotations
+from sibi_dst.osmnx_helper.utils import get_distance_between_points, add_arrows
+from collections import defaultdict
+import folium
+from folium.plugins import AntPath
+import networkx as nx
+
+from sibi_dst.osmnx_helper import BaseOsmMap
+from sibi_dst.osmnx_helper.basemaps.calendar_html import calendar_html
+
+class RoutePlotter(BaseOsmMap):
+    def __init__(self, osmnx_graph, df, **kwargs):
+        self.action_field = kwargs.pop('action_field', '')
+        self.action_groups = kwargs.pop('action_groups', {})
+        self.action_styles = kwargs.pop('action_styles', {})
+        self.use_ant_path = kwargs.pop('use_ant_path', True)
+        self.show_calendar = kwargs.pop('show_calendar', True)
+        self.show_map_title = kwargs.pop('show_map_title', True)
+        self.sort_keys = kwargs.pop('sort_keys', None)
+        self.main_route_layer = folium.FeatureGroup(name="Main Route")
+        self.feature_groups = {}
+        self.feature_group_counts = {}
+        self.total_distance = 0.0
+        self.actions = []
+        self.action_group_counts = {action_group: 0 for action_group in self.action_groups}
+        self.marker_count = 1
+        kwargs.update({'calc_nearest_nodes': True})
+        kwargs['dt_field'] = 'date_time'
+        super().__init__(osmnx_graph, df, **kwargs)
+
+    def pre_process_map(self):
+        super().pre_process_map()
+        self.actions = self.df[self.action_field].tolist()
+
+    def process_map(self):
+        self._calculate_routes()
+        self._plot_routes()
+        self._add_markers()
+        self.main_route_layer.add_to(self.osm_map)
+        if self.show_calendar:
+            self._add_calendar()
+
+    def _calculate_routes(self):
+        if self.verbose:
+            print("Calculating routes and markers...")
+        distances = [
+            get_distance_between_points(tuple(self.gps_points[0]), tuple(coord), 'm')
+            for coord in self.gps_points
+        ]
+        self.max_distance_index = distances.index(max(distances))
+        self.max_time_index = self.dt.index(max(self.dt))
+        self.route_polylines = []
+        self.markers = defaultdict(list)  # Store markers for action groups
+        for i in range(len(self.gps_points) - 1):
+            polyline, color, markers = self._calculate_route(i)
+            if polyline:
+                self.route_polylines.append((polyline, color))
+                for action_group, action_markers in markers.items():
+                    self.markers[action_group].extend(action_markers)
+                    self.action_group_counts[action_group] += 1
+                    self.marker_count += 1
+        if self.verbose:
+            print("Route and marker calculation complete.")
+
+        for action_group in self.action_groups:
+            count = self.action_group_counts[action_group]
+            self.feature_groups[action_group] = folium.FeatureGroup(name=f"{action_group} ({count})").add_to(
+                self.osm_map)
+            self.osm_map.add_child(self.feature_groups[action_group])
+
+    def _calculate_route(self, i):
+        if self.verbose:
+            print(f"Calculating for item:{i}")
+        orig = self.nearest_nodes[i]
+        dest = self.nearest_nodes[i + 1]
+        try:
+            route = nx.shortest_path(self.G, orig, dest, weight='length')
+            route_length = sum(self.G[u][v][0]['length'] for u, v in zip(route[:-1], route[1:]))
+            self.total_distance += route_length
+            offset = 0 if i < self.max_distance_index else 0.0005
+            lats, lons = zip(*[(self.G.nodes[node]['y'] + offset, self.G.nodes[node]['x']) for node in route])
+            color = 'blue' if i < self.max_distance_index else 'red'
+            polyline = list(zip(lats, lons))
+            markers = self._calculate_markers(i)
+            return polyline, color, markers
+        except nx.NetworkXNoPath:
+            if self.verbose:
+                print(f"Item:{i}-No path found for {orig} to {dest}")
+            return None, None, {}
+        except nx.NodeNotFound:
+            if self.verbose:
+                print(f"Item:{i}-No path found for {orig} to {dest}")
+            return None, None, {}
+
+    def _calculate_markers(self, i):
+        # Calculate markers for action groups
+        markers = defaultdict(list)
+        for action_group in self.action_groups:
+            action_indices = [idx for idx, action in enumerate(self.actions) if action == action_group]
+            for idx in action_indices:
+                if idx == i:
+                    location = self.gps_points[i]
+                    tooltip = f"Result {self.marker_count}: {action_group}<br>Date/time:{self.dt[i]}"
+                    popup_data = self._get_data(i)
+                    action_style = self.action_styles.get(action_group,
+                                                          {'color': 'blue', 'icon': 'marker', 'prefix': 'fa'})
+                    markers[action_group].append((location, tooltip, popup_data, action_style))
+        return markers
+
+    def _plot_routes(self):
+        if self.verbose:
+            print("Plotting routes and markers...")
+        # self.action_group_counts = {action_group: 0 for action_group in self.feature_groups.keys()}
+        for polyline, color in self.route_polylines:
+            if self.use_ant_path:
+                AntPath(
+                    locations=polyline,
+                    color=color,
+                    weight=3,  # Increase line thickness
+                    opacity=10,  # Increase opacity
+                    # pulse_color=color,
+                    delay=1000,  # Slower animation to reduce flickering
+                    # dash_array=[20, 30]  # Adjust dash pattern if needed
+                ).add_to(self.main_route_layer)
+            else:
+                folium.PolyLine(locations=polyline, color=color).add_to(self.main_route_layer)
+                self.osm_map = add_arrows(self.osm_map, polyline, color, n_arrows=3)
+        # Plot markers for action groups
+        for action_group, action_markers in self.markers.items():
+            for location, tooltip, popup_data, action_style in action_markers:
+                folium.Marker(
+                    location=location,
+                    popup=folium.Popup(popup_data, max_width=600),
+                    tooltip=tooltip,
+                    icon=folium.Icon(
+                        icon=action_style.get("icon"),
+                        color=action_style.get("color"),
+                        prefix=action_style.get("prefix")
+                    )
+                ).add_to(self.feature_groups[action_group])
+
+        if self.verbose:
+            print("Route and marker plotting complete.")
+
+    def _add_markers(self):
+        if self.verbose:
+            print("Adding markers...")
+        # Add start marker
+        start_popup = folium.Popup(f"Start of route at {self.dt[0]}", max_width=300)
+        folium.Marker(location=self.gps_points[0], popup=start_popup,
+                      icon=folium.Icon(icon='flag-checkered', prefix='fa')).add_to(self.osm_map)
+        # Add total distance marker at the end
+        folium.Marker(
+            self.gps_points[-1],
+            popup=f"End of Route at {self.dt[self.max_time_index]}. Total Distance Travelled: {self.total_distance / 1000:.2f} km",
+            icon=folium.Icon(color="red", icon="flag-checkered", prefix="fa")
+        ).add_to(self.osm_map)
+        if self.verbose:
+            print("Marker addition complete.")
+
+    def _add_calendar(self):
+        calendar_element = folium.Element(calendar_html)
+        self.osm_map.get_root().html.add_child(calendar_element)
+
+    def _add_map_title(self):
+        if self.map_html_title and self.show_map_title:
+            title_html = f'''
+                 <div style="position: fixed; 
+                             top: 10px; 
+                             left: 50%; 
+                             transform: translate(-50%, 0%);
+                             z-index: 9999; 
+                             font-size: 24px; 
+                             font-weight: bold; 
+                             background-color: white; 
+                             padding: 10px; 
+                             border: 2px solid black;
+                             border-radius: 5px;">
+                    {self.map_html_title}
+                 </div>
+                 '''
+            self.osm_map.get_root().html.add_child(folium.Element(title_html))
+
+    def _get_data(self, index):
+        # implement in subclass to populate popups
+        ...

sibi_dst/osmnx_helper/utils.py

@@ -0,0 +1,267 @@
+import math
+import os
+import pickle
+from urllib.parse import urlencode, urlsplit, urlunsplit
+
+import folium
+import geopandas as gpd
+import numpy as np
+import osmnx as ox
+from geopy.distance import geodesic
+
+
+#
+# options = {
+#    'ox_files_save_path': ox_files_save_path,
+#    'network_type': 'drive',
+#    'place': 'Costa Rica',
+#    'files_prefix': 'costa-rica-',
+# }
+# Usage example
+# handler = PBFHandler(**options)
+# handler.load()
+
+
+class PBFHandler:
+    def __init__(self, **kwargs):
+        self.graph = None
+        self.nodes = None
+        self.edges = None
+        self.rebuild = kwargs.setdefault("rebuild", False)
+        self.verbose = kwargs.setdefault("verbose", False)
+        self.place = kwargs.setdefault('place', 'Costa Rica')
+        self.filepath = kwargs.setdefault('ox_files_save_path', "gis_data/")
+        self.file_prefix = kwargs.setdefault('file_prefix', 'costa-rica-')
+        self.network_type = kwargs.setdefault('network_type', 'all')
+        self.graph_file = f"{self.filepath}{self.file_prefix}graph.pkl"
+        self.node_file = f"{self.filepath}{self.file_prefix}nodes.pkl"
+        self.edge_file = f"{self.filepath}{self.file_prefix}edges.pkl"
+
+    def load(self):
+        if self.verbose:
+            print("Loading data...")
+
+        files_to_check = [self.graph_file, self.node_file, self.edge_file]
+
+        if self.rebuild:
+            for file in files_to_check:
+                if os.path.exists(file):
+                    os.remove(file)
+        if not os.path.exists(self.filepath):
+            os.makedirs(self.filepath, exist_ok=True)
+            # self.process_pbf()
+            # self.save_to_pickle()
+        if not all(os.path.exists(f) for f in files_to_check):
+            self.process_pbf()
+            self.save_to_pickle()
+        else:
+            self.load_from_pickle()
+
+        if self.verbose:
+            print("Data loaded successfully.")
+
+    def process_pbf(self):
+        """
+        Load a PBF file and create a graph.
+        """
+        try:
+            if self.verbose:
+                print(f"Processing PBF for {self.place}...")
+
+            self.graph = ox.graph_from_place(self.place, network_type=self.network_type)
+            self.nodes, self.edges = ox.graph_to_gdfs(self.graph)
+
+            if self.verbose:
+                print("PBF processed successfully.")
+        except Exception as e:
+            print(f"Error processing PBF: {e}")
+            raise
+
+    def save_to_pickle(self):
+        """
+        Save the graph, nodes, and edges to pickle files.
+        """
+        try:
+            if self.verbose:
+                print("Saving data to pickle files...")
+
+            data_to_save = {
+                self.graph_file: self.graph,
+                self.node_file: self.nodes,
+                self.edge_file: self.edges
+            }
+
+            for file, data in data_to_save.items():
+                if data is not None:
+                    with open(file, 'wb') as f:
+                        pickle.dump(data, f, pickle.HIGHEST_PROTOCOL)
+
+            if self.verbose:
+                print("Data saved to pickle files successfully.")
+        except Exception as e:
+            print(f"Error saving to pickle: {e}")
+            raise
+
+    def load_from_pickle(self):
+        """
+        Load the graph, nodes, and edges from pickle files.
+        """
+        try:
+            if self.verbose:
+                print("Loading data from pickle files...")
+
+            files_to_load = {
+                self.graph_file: 'graph',
+                self.node_file: 'nodes',
+                self.edge_file: 'edges'
+            }
+
+            for file, attr in files_to_load.items():
+                with open(file, 'rb') as f:
+                    setattr(self, attr, pickle.load(f))
+
+            if self.verbose:
+                print("Data loaded from pickle files successfully.")
+        except Exception as e:
+            print(f"Error loading from pickle: {e}")
+            raise
+
+    def plot_graph(self):
+        """
+        Plot the graph.
+        """
+        try:
+            if self.graph is not None:
+                if self.verbose:
+                    print("Plotting the graph...")
+                ox.plot_graph(self.graph)
+                if self.verbose:
+                    print("Graph plotted successfully.")
+            else:
+                print("Graph is not loaded. Please load a PBF file first.")
+        except Exception as e:
+            print(f"Error plotting the graph: {e}")
+            raise
+
+
+def get_bounding_box_from_points(gps_points, margin=0.001):
+    latitudes = [point[0] for point in gps_points]
+    longitudes = [point[1] for point in gps_points]
+
+    north = max(latitudes) + margin
+    south = min(latitudes) - margin
+    east = max(longitudes) + margin
+    west = min(longitudes) - margin
+
+    return north, south, east, west
+
+
+def add_arrows(map_object, locations, color, n_arrows):
+    # Get the number of locations
+    n = len(locations)
+
+    # If there are more than two points...
+    if n > 2:
+        # Add arrows along the path
+        for i in range(0, n - 1, n // n_arrows):
+            # Get the start and end point for this segment
+            start, end = locations[i], locations[i + 1]
+
+            # Calculate the direction in which to place the arrow
+            rotation = -np.arctan2((end[1] - start[1]), (end[0] - start[0])) * 180 / np.pi
+
+            folium.RegularPolygonMarker(location=end,
+                                        fill_color=color,
+                                        number_of_sides=2,
+                                        radius=6,
+                                        rotation=rotation).add_to(map_object)
+    return map_object
+
+
+def extract_subgraph(G, north, south, east, west):
+    # Create a bounding box polygon
+    # from osmnx v2 this is how it is done
+    if ox.__version__ >= '2.0':
+        bbox_poly = gpd.GeoSeries([ox.utils_geo.bbox_to_poly(bbox=(west, south, east, north))])
+    else:
+        bbox_poly = gpd.GeoSeries([ox.utils_geo.bbox_to_poly(north, south, east, west)])
+
+    # Get nodes GeoDataFrame
+    nodes_gdf = ox.graph_to_gdfs(G, nodes=True, edges=False)
+
+    # Find nodes within the bounding box
+    nodes_within_bbox = nodes_gdf[nodes_gdf.geometry.within(bbox_poly.geometry.unary_union)]
+
+    # Create subgraph
+    subgraph = G.subgraph(nodes_within_bbox.index)
+
+    return subgraph
+
+
+def get_distance_between_points(point_a, point_b, unit='km'):
+    if not isinstance(point_a, tuple) or len(point_a) != 2:
+        return 0
+    if not all(isinstance(x, float) and not math.isnan(x) for x in point_a):
+        return 0
+    if not isinstance(point_b, tuple) or len(point_b) != 2:
+        return 0
+    if not all(isinstance(x, float) and not math.isnan(x) for x in point_b):
+        return 0
+    distance = geodesic(point_a, point_b)
+    if unit == 'km':
+        return distance.kilometers
+    elif unit == 'm':
+        return distance.meters
+    elif unit == 'mi':
+        return distance.miles
+    else:
+        return 0
+
+
+tile_options = {
+    "OpenStreetMap": "OpenStreetMap",
+    "CartoDB": "cartodbpositron",
+    "CartoDB Voyager": "cartodbvoyager"
+}
+
+
+def attach_supported_tiles(map_object, default_tile="OpenStreetMap"):
+    # Normalize the default tile name to lowercase for comparison
+    normalized_default_tile = default_tile.lower()
+
+    # Filter out the default tile layer from the options to avoid duplication
+    tile_options_filtered = {k: v for k, v in tile_options.items() if v.lower() != normalized_default_tile}
+
+    for tile, description in tile_options_filtered.items():
+        folium.TileLayer(name=tile, tiles=description, show=False).add_to(map_object)
+
+
+def get_graph(**options):
+    handler = PBFHandler(**options)
+    handler.load()
+    return handler.graph, handler.nodes, handler.edges
+
+
+def add_query_params(url, params):
+    # Parse the original URL
+    url_components = urlsplit(url)
+
+    # Parse original query parameters and update with new params
+    original_params = dict([tuple(pair.split('=')) for pair in url_components.query.split('&') if pair])
+    original_params.update(params)
+
+    # Construct the new query string
+    new_query_string = urlencode(original_params)
+
+    # Construct the new URL
+    new_url = urlunsplit((
+        url_components.scheme,
+        url_components.netloc,
+        url_components.path,
+        new_query_string,
+        url_components.fragment
+    ))
+
+    return new_url
+
+

sibi_dst/tests/test_data_wrapper_class.py

@@ -0,0 +1,78 @@
+import unittest
+from unittest.mock import patch, MagicMock
+import datetime
+import pandas as pd
+from sibi_dst.utils import Logger, ParquetSaver
+from sibi_dst.utils.data_wrapper import DataWrapper
+
+
+class TestDataWrapper(unittest.TestCase):
+
+    def setUp(self):
+        self.dataclass = MagicMock()
+        self.date_field = "created_at"
+        self.data_path = "/path/to/data"
+        #self.data_path = "s3://your-bucket-name/path/to/data"
+        self.parquet_filename = "data.parquet"
+        self.start_date = "2022-01-01"
+        self.end_date = "2022-12-31"
+        self.filesystem_type = "file"
+        self.filesystem_options = {
+            #"key": "your_aws_access_key",
+            #"secret": "your_aws_secret_key",
+            #"client_kwargs": {"endpoint_url": "https://s3.amazonaws.com"}
+        }
+        self.logger = Logger.default_logger(logger_name="TestLogger")
+
+    def test_initialization(self):
+        wrapper = DataWrapper(
+            dataclass=self.dataclass,
+            date_field=self.date_field,
+            data_path=self.data_path,
+            parquet_filename=self.parquet_filename,
+            start_date=self.start_date,
+            end_date=self.end_date,
+            filesystem_type=self.filesystem_type,
+            filesystem_options=self.filesystem_options,
+            logger=self.logger
+        )
+        self.assertEqual(wrapper.dataclass, self.dataclass)
+        self.assertEqual(wrapper.date_field, self.date_field)
+        self.assertEqual(wrapper.data_path, "/path/to/data/")
+        self.assertEqual(wrapper.parquet_filename, self.parquet_filename)
+        self.assertEqual(wrapper.start_date, datetime.date(2022, 1, 1))
+        self.assertEqual(wrapper.end_date, datetime.date(2022, 12, 31))
+        self.assertEqual(wrapper.filesystem_type, self.filesystem_type)
+        self.assertEqual(wrapper.filesystem_options, self.filesystem_options)
+        self.assertEqual(wrapper.logger, self.logger)
+
+    def test_convert_to_date(self):
+        self.assertEqual(DataWrapper.convert_to_date("2022-01-01"), datetime.date(2022, 1, 1))
+        self.assertEqual(DataWrapper.convert_to_date(datetime.date(2022, 1, 1)), datetime.date(2022, 1, 1))
+        with self.assertRaises(ValueError):
+            DataWrapper.convert_to_date("invalid-date")
+
+    @patch('fsspec.filesystem')
+    def test_is_file_older_than(self, mock_filesystem):
+        mock_fs = mock_filesystem.return_value
+        mock_fs.info.return_value = {'mtime': (datetime.datetime.now() - datetime.timedelta(minutes=1500)).timestamp()}
+
+        wrapper = DataWrapper(
+            dataclass=self.dataclass,
+            date_field=self.date_field,
+            data_path=self.data_path,
+            parquet_filename=self.parquet_filename,
+            start_date=self.start_date,
+            end_date=self.end_date,
+            filesystem_type=self.filesystem_type,
+            filesystem_options=self.filesystem_options,
+            logger=self.logger
+        )
+
+        self.assertTrue(wrapper.is_file_older_than("some/file/path"))
+        mock_fs.info.return_value = {'mtime': (datetime.datetime.now() - datetime.timedelta(minutes=1000)).timestamp()}
+        self.assertFalse(wrapper.is_file_older_than("some/file/path"))
+
+
+if __name__ == '__main__':
+    unittest.main()

sibi_dst/utils/clickhouse_writer.py

@@ -125,7 +125,7 @@ class ClickHouseWriter:
         """
         Writes the Dask DataFrame to a ClickHouse table partition by partition.
         """
-        if len(self.df.head().index) == 0:
+        if len(self.df.index) == 0:
             self.logger.debug("No data found. Nothing written.")
             return
 

sibi_dst/utils/data_utils.py

@@ -95,7 +95,7 @@ class DataUtils:
         # Get unique IDs from source column
         ids = df[source_col].dropna().unique()
         # Compute if it's a Dask Series
-        if isinstance(ids, dd.core.Series):
+        if isinstance(ids, dd.Series):
             ids = ids.compute()
 
         # Check if any IDs are found