geo-activity-playground 0.11.0__tar.gz → 0.12.0__tar.gz

geo_activity_playground-0.12.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Martin Ueding
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

{geo_activity_playground-0.11.0 → geo_activity_playground-0.12.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: geo-activity-playground
-Version: 0.11.0
+Version: 0.12.0
 Summary: Analysis of geo data activities like rides, runs or hikes.
 License: MIT
 Author: Martin Ueding

{geo_activity_playground-0.11.0 → geo_activity_playground-0.12.0}/geo_activity_playground/core/activity_parsers.py

@@ -159,6 +159,12 @@ def read_activity(path: pathlib.Path) -> pd.DataFrame:
         except AttributeError as e:
             print(df)
             print(df.dtypes)
+            types = {}
+            for elem in df["time"]:
+                t = str(type(elem))
+                if t not in types:
+                    types[t] = elem
+            print(types)
             raise ActivityParseError(
                 "It looks like the date parsing has gone wrong."
             ) from e

{geo_activity_playground-0.11.0 → geo_activity_playground-0.12.0}/geo_activity_playground/explorer/clusters.py

@@ -24,11 +24,12 @@ def adjacent_to(tile: tuple[int, int]) -> Iterator[tuple[int, int]]:
 class ExplorerClusterState:
     def __init__(self, zoom: int) -> None:
         self.num_neighbors: dict[tuple[int, int], int] = {}
-        self.cluster_tiles: dict[tuple[int, int], list[tuple[int, int]]] = {}
+        self.memberships: dict[tuple[int, int], tuple[int, int]] = {}
+        self.clusters: dict[tuple[int, int], list[tuple[int, int]]] = {}
         self.cluster_evolution = pd.DataFrame()
         self.start = 0
 
-        self._state_path = pathlib.Path(f"Cache/explorer_cluster_{zoom}_state.json")
+        self._state_path = pathlib.Path(f"Cache/explorer_cluster_{zoom}_state_v2.json")
         self._cluster_evolution_path = pathlib.Path(
             f"Cache/explorer_cluster_{zoom}_evolution.parquet"
         )
@@ -42,12 +43,14 @@ class ExplorerClusterState:
                 tuple(map(int, key.split("/"))): value
                 for key, value in data["num_neighbors"].items()
             }
-            self.cluster_tiles = {
-                tuple(map(int, key.split("/"))): [
-                    tuple(t) for t in data["clusters"][str(value)]
-                ]
+            self.memberships = {
+                tuple(map(int, key.split("/"))): tuple(value)
                 for key, value in data["memberships"].items()
             }
+            self.clusters = {
+                tuple(map(int, key.split("/"))): [tuple(t) for t in value]
+                for key, value in data["clusters"].items()
+            }
             self.start = data["start"]
 
         if self._cluster_evolution_path.exists():
@@ -60,11 +63,11 @@ class ExplorerClusterState:
                 f"{x}/{y}": count for (x, y), count in self.num_neighbors.items()
             },
             "memberships": {
-                f"{x}/{y}": id(members)
-                for (x, y), members in self.cluster_tiles.items()
+                f"{x}/{y}": value for (x, y), value in self.memberships.items()
             },
             "clusters": {
-                id(members): members for members in self.cluster_tiles.values()
+                f"{x}/{y}": [tuple(member) for member in members]
+                for (x, y), members in self.clusters.items()
             },
             "start": self.start,
         }
@@ -107,34 +110,43 @@ def get_explorer_cluster_evolution(zoom: int) -> ExplorerClusterState:
 
         # If the current tile has all neighbors, make it it's own cluster.
         if s.num_neighbors[tile] == 4:
-            s.cluster_tiles[tile] = [tile]
+            s.clusters[tile] = [tile]
+            s.memberships[tile] = tile
 
         # Also make the adjacent tiles their own clusters, if they are full.
         this_and_neighbors = [tile] + list(adjacent_to(tile))
         for other in this_and_neighbors:
             if s.num_neighbors.get(other, 0) == 4:
-                s.cluster_tiles[other] = [other]
+                s.clusters[other] = [other]
+                s.memberships[other] = other
 
         for candidate in this_and_neighbors:
-            if candidate not in s.cluster_tiles:
+            # If the the candidate is not a cluster tile, skip.
+            if candidate not in s.memberships:
                 continue
             # The candidate is a cluster tile. Let's see whether any of the neighbors are also cluster tiles but with a different cluster. Then we need to join them.
             for other in adjacent_to(candidate):
-                if other not in s.cluster_tiles:
+                if other not in s.memberships:
                     continue
                 # The other tile is also a cluster tile.
-                if s.cluster_tiles[candidate] is s.cluster_tiles[other]:
+                if s.memberships[candidate] == s.memberships[other]:
                     continue
                 # The two clusters are not the same. We add the other's cluster tile to this tile.
-                s.cluster_tiles[candidate].extend(s.cluster_tiles[other])
+                this_cluster = s.clusters[s.memberships[candidate]]
+                assert isinstance(other, tuple), other
+                assert isinstance(s.memberships[other], tuple), s.memberships[other]
+                other_cluster = s.clusters[s.memberships[other]]
+                other_cluster_name = s.memberships[other]
+                this_cluster.extend(other_cluster)
                 # Update the other cluster tiles that they now point to the new cluster. This also updates the other tile.
-                for member in s.cluster_tiles[other]:
-                    s.cluster_tiles[member] = s.cluster_tiles[candidate]
+                for member in other_cluster:
+                    s.memberships[member] = s.memberships[candidate]
+                del s.clusters[other_cluster_name]
                 new_clusters = True
 
         if new_clusters:
             max_cluster_size = max(
-                (len(members) for members in s.cluster_tiles.values()),
+                (len(members) for members in s.clusters.values()),
                 default=0,
             )
             if max_cluster_size > max_cluster_so_far:

{geo_activity_playground-0.11.0 → geo_activity_playground-0.12.0}/geo_activity_playground/explorer/converters.py

@@ -1,6 +1,7 @@
 import functools
 import logging
 import pathlib
+import pickle
 
 import pandas as pd
 
@@ -75,14 +76,16 @@ def reduce_tile_group(group: pd.DataFrame) -> pd.DataFrame:
 def get_tile_history(repository: ActivityRepository, zoom: int) -> pd.DataFrame:
     logger.info("Building explorer tile history from all activities …")
 
-    cache_file = pathlib.Path(f"Cache/first_time_per_tile_{zoom}.parquet")
+    cache_file = pathlib.Path(f"Cache/first_time_per_tile_{zoom}.pickle")
+
     if cache_file.exists():
-        tiles = pd.read_parquet(cache_file)
+        with open(cache_file, "rb") as f:
+            tile_visits = pickle.load(f)
     else:
-        tiles = pd.DataFrame()
+        tile_visits = {}
 
     with work_tracker(
-        pathlib.Path(f"Cache/task_first_time_per_tile_{zoom}.json")
+        pathlib.Path(f"Cache/task_first_time_per_tile_{zoom}_v2.json")
     ) as parsed_activities:
         for activity in repository.iter_activities(new_to_old=False):
             if activity.id in parsed_activities:
@@ -93,22 +96,33 @@ def get_tile_history(repository: ActivityRepository, zoom: int) -> pd.DataFrame:
             shard = get_first_tiles(activity.id, repository, zoom)
             if not len(shard):
                 continue
-            shard2 = pd.DataFrame(
-                {
-                    "tile_x": shard["tile_x"],
-                    "tile_y": shard["tile_y"],
-                    "first_id": activity.id,
-                    "last_id": activity.id,
-                    "first_time": shard["time"],
-                    "last_time": shard["time"],
-                    "count": 1,
-                }
-            )
-            tiles = pd.concat([tiles, shard2])
-    logger.info("Consolidating explorer tile history …")
-    tiles = tiles.groupby(["tile_x", "tile_y"]).apply(reduce_tile_group).reset_index()
+            for _, row in shard.iterrows():
+                tile = (row["tile_x"], row["tile_y"])
+                if tile in tile_visits:
+                    d = tile_visits[tile]
+                    d["count"] += 1
+                    if d["first_time"] > row["time"]:
+                        d["first_time"] = row["time"]
+                        d["first_id"] = activity.id
+                    if d["last_time"] < row["time"]:
+                        d["last_time"] = row["time"]
+                        d["last_id"] = activity.id
+                else:
+                    tile_visits[tile] = {
+                        "count": 1,
+                        "first_time": row["time"],
+                        "first_id": activity.id,
+                        "last_time": row["time"],
+                        "last_id": activity.id,
+                    }
 
     logger.info("Store explorer tile history to cache file …")
-    tiles.to_parquet(cache_file)
+    with open(cache_file, "wb") as f:
+        pickle.dump(tile_visits, f)
 
+    tiles = pd.DataFrame(
+        [{"tile_x": x, "tile_y": y, **meta} for (x, y), meta in tile_visits.items()]
+    )
+    parquet_output = pathlib.Path(f"Cache/first_time_per_tile_{zoom}.parquet")
+    tiles.to_parquet(parquet_output)
     return tiles

{geo_activity_playground-0.11.0 → geo_activity_playground-0.12.0}/geo_activity_playground/explorer/grid_file.py

@@ -1,7 +1,9 @@
 import datetime
+import itertools
 import logging
 import pathlib
 from typing import Iterator
+from typing import Optional
 
 import geojson
 import gpxpy
@@ -45,7 +47,7 @@ def get_three_color_tiles(
                 cmap_last(max(1 - last_age_days / (2 * 365), 0.0))
             ),
             "cluster": False,
-            "color": None,
+            "color": "#303030",
             "first_visit": row["first_time"].date().isoformat(),
             "last_visit": row["last_time"].date().isoformat(),
             "num_visits": row["count"],
@@ -74,26 +76,32 @@ def get_three_color_tiles(
         for x in range(square_x, square_x + square_size):
             for y in range(square_y, square_y + square_size):
                 tile_dict[(x, y)]["square"] = True
-                tile_dict[(x, y)]["color"] = "blue"
 
     # Add cluster information.
-    for xy, members in cluster_state.cluster_tiles.items():
-        tile_dict[xy]["this_cluster_size"] = len(members)
-        tile_dict[xy]["cluster"] = True
+    for members in cluster_state.clusters.values():
+        for member in members:
+            tile_dict[member]["this_cluster_size"] = len(members)
+            tile_dict[member]["cluster"] = True
     if len(cluster_state.cluster_evolution) > 0:
         max_cluster_size = cluster_state.cluster_evolution["max_cluster_size"].iloc[-1]
     else:
         max_cluster_size = 0
-    num_cluster_tiles = len(cluster_state.cluster_tiles)
+    num_cluster_tiles = len(cluster_state.memberships)
 
     # Apply cluster colors.
-    for xy, xy_dict in tile_dict.items():
-        if xy_dict["square"]:
-            xy_dict["color"] = "blue"
-        elif xy_dict["cluster"]:
-            xy_dict["color"] = "green"
-        else:
-            xy_dict["color"] = "red"
+    cluster_cmap = matplotlib.colormaps["tab10"]
+    for color, members in zip(
+        itertools.cycle(map(cluster_cmap, [0, 1, 2, 3, 4, 5, 6, 8, 9])),
+        sorted(
+            cluster_state.clusters.values(),
+            key=lambda members: len(members),
+            reverse=True,
+        ),
+    ):
+        hex_color = matplotlib.colors.to_hex(color)
+        print(f"Color cluster with {len(members)} tiles in {hex_color}.")
+        for member in members:
+            tile_dict[member]["color"] = hex_color
 
     result = {
         "explored_geojson": geojson.dumps(
@@ -113,6 +121,19 @@ def get_three_color_tiles(
         "num_cluster_tiles": num_cluster_tiles,
         "num_tiles": len(tile_dict),
         "square_size": square_size,
+        "square_geojson": geojson.dumps(
+            geojson.FeatureCollection(
+                features=[
+                    make_explorer_rectangle(
+                        square_x,
+                        square_y,
+                        square_x + square_size,
+                        square_y + square_size,
+                        zoom,
+                    )
+                ]
+            )
+        ),
     }
     return result
 
@@ -134,15 +155,23 @@ def get_explored_tiles(tiles: pd.DataFrame, zoom: int) -> list[list[list[float]]
 
 def make_explorer_tile(
     tile_x: int, tile_y: int, properties: dict, zoom: int
+) -> geojson.Feature:
+    return make_explorer_rectangle(
+        tile_x, tile_y, tile_x + 1, tile_y + 1, zoom, properties
+    )
+
+
+def make_explorer_rectangle(
+    x1: int, y1: int, x2: int, y2: int, zoom: int, properties: Optional[dict] = None
 ) -> geojson.Feature:
     corners = [
         get_tile_upper_left_lat_lon(*args)
         for args in [
-            (tile_x, tile_y, zoom),
-            (tile_x + 1, tile_y, zoom),
-            (tile_x + 1, tile_y + 1, zoom),
-            (tile_x, tile_y + 1, zoom),
-            (tile_x, tile_y, zoom),
+            (x1, y1, zoom),
+            (x2, y1, zoom),
+            (x2, y2, zoom),
+            (x1, y2, zoom),
+            (x1, y1, zoom),
         ]
     ]
     return geojson.Feature(

{geo_activity_playground-0.11.0 → geo_activity_playground-0.12.0}/geo_activity_playground/webui/explorer_controller.py

@@ -51,9 +51,9 @@ class ExplorerController:
                 "latitude": median_lat,
                 "longitude": median_lon,
                 "bbox": bounding_box_for_biggest_cluster(
-                    cluster_state.cluster_tiles.values(), zoom
+                    cluster_state.clusters.values(), zoom
                 )
-                if len(cluster_state.cluster_tiles) > 0
+                if len(cluster_state.memberships) > 0
                 else {},
             },
             "explored": explored,
@@ -69,6 +69,8 @@ class ExplorerController:
 
 
 def plot_tile_evolution(tiles: pd.DataFrame) -> str:
+    if len(tiles) == 0:
+        return ""
     tiles.sort_values("first_time", inplace=True)
     tiles["count"] = np.arange(1, len(tiles) + 1)
     return (
@@ -83,6 +85,8 @@ def plot_tile_evolution(tiles: pd.DataFrame) -> str:
 
 
 def plot_cluster_evolution(cluster_evolution: pd.DataFrame) -> str:
+    if len(cluster_evolution) == 0:
+        return ""
     return (
         alt.Chart(cluster_evolution, title="Cluster")
         .mark_line(interpolate="step-after")
@@ -96,6 +100,8 @@ def plot_cluster_evolution(cluster_evolution: pd.DataFrame) -> str:
 
 
 def plot_square_evolution(square_evolution: pd.DataFrame) -> str:
+    if len(square_evolution) == 0:
+        return ""
     return (
         alt.Chart(square_evolution, title="Square")
         .mark_line(interpolate="step-after")

{geo_activity_playground-0.11.0 → geo_activity_playground-0.12.0}/geo_activity_playground/webui/templates/explorer.html.j2

@@ -48,6 +48,7 @@
             }
 
             let explorer_geojson = {{ explored.explored_geojson| safe}}
+            let square_geojson = {{ explored.square_geojson | safe }}
             let map = L.map('explorer-map', {
                 fullscreenControl: true,
                 center: [{{ center.latitude }}, {{ center.longitude }}],
@@ -75,6 +76,16 @@
                 map.fitBounds(L.geoJSON(bbox).getBounds())
             }
 
+            let explorer_square_layer = L.geoJSON(square_geojson,
+                {
+                    style: function (feature) {
+                        return {
+                            color: "blue", fill: false, weight: 2
+                        }
+                    }
+                }
+            ).addTo(map)
+
             active_layer = explorer_layer_cluster_color
 
             function changeColor(method) {
@@ -100,13 +111,19 @@
 
 <div class="row mb-3">
     <div class="col-md-4">
+        {% if plot_tile_evolution|length > 0 %}
         {{ vega_direct("plot_tile_evolution", plot_tile_evolution) }}
+        {% endif %}
     </div>
     <div class="col-md-4">
+        {% if plot_cluster_evolution|length > 0 %}
         {{ vega_direct("plot_cluster_evolution", plot_cluster_evolution) }}
+        {% endif %}
     </div>
     <div class="col-md-4">
+        {% if plot_square_evolution|length > 0 %}
         {{ vega_direct("plot_square_evolution", plot_square_evolution) }}
+        {% endif %}
     </div>
 </div>
 

{geo_activity_playground-0.11.0 → geo_activity_playground-0.12.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "geo-activity-playground"
-version = "0.11.0"
+version = "0.12.0"
 description = "Analysis of geo data activities like rides, runs or hikes."
 authors = ["Martin Ueding <mu@martin-ueding.de>"]
 license = "MIT"
@@ -40,6 +40,7 @@ black = "^22.3.0"
 mkdocs-material = "^9.4.1"
 mypy = "^0.961"
 pytest = "^7.1.2"
+py-spy = "^0.3.14"
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]