geo-activity-playground 0.13.0__tar.gz → 0.14.1__tar.gz

{geo_activity_playground-0.13.0 → geo_activity_playground-0.14.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: geo-activity-playground
-Version: 0.13.0
+Version: 0.14.1
 Summary: Analysis of geo data activities like rides, runs or hikes.
 License: MIT
 Author: Martin Ueding
@@ -35,3 +35,4 @@ Requires-Dist: tqdm (>=4.64.0,<5.0.0)
 Requires-Dist: vegafusion (>=1.4.3,<2.0.0)
 Requires-Dist: vegafusion-python-embed (>=1.4.3,<2.0.0)
 Requires-Dist: vl-convert-python (>=1.0.1,<2.0.0)
+Requires-Dist: xmltodict (>=0.13.0,<0.14.0)

{geo_activity_playground-0.13.0 → geo_activity_playground-0.14.1}/geo_activity_playground/__main__.py

@@ -1,12 +1,16 @@
 import argparse
+import logging
 import os
 import pathlib
 
 import coloredlogs
 
 from geo_activity_playground.core.activities import ActivityRepository
+from geo_activity_playground.core.activities import embellish_time_series
 from geo_activity_playground.core.cache_migrations import apply_cache_migrations
 from geo_activity_playground.core.config import get_config
+from geo_activity_playground.explorer.tile_visits import compute_tile_evolution
+from geo_activity_playground.explorer.tile_visits import compute_tile_visits
 from geo_activity_playground.explorer.video import explorer_video_main
 from geo_activity_playground.heatmap import generate_heatmaps_per_cluster
 from geo_activity_playground.importers.directory import import_from_directory
@@ -56,7 +60,17 @@ def main() -> None:
 
     subparser = subparsers.add_parser("serve", help="Launch webserver")
     subparser.set_defaults(
-        func=lambda options: webui_main(make_activity_repository(options.basedir))
+        func=lambda options: webui_main(
+            make_activity_repository(options.basedir),
+            host=options.host,
+            port=options.port,
+        )
+    )
+    subparser.add_argument(
+        "--host", default="127.0.0.1", help="IP address to listen on"
+    )
+    subparser.add_argument(
+        "--port", default=5000, type=int, help="the port to run listen on"
     )
 
     subparser = subparsers.add_parser("cache", help="Cache stuff")
@@ -69,6 +83,9 @@ def main() -> None:
         fmt="%(asctime)s %(name)s %(levelname)s %(message)s",
         level=options.loglevel.upper(),
     )
+
+    logging.getLogger("stravalib.protocol.ApiV3").setLevel(logging.WARNING)
+
     options.func(options)
 
 
@@ -81,4 +98,8 @@ def make_activity_repository(basedir: pathlib.Path) -> ActivityRepository:
     elif config:
         if "strava" in config:
             import_from_strava_api()
-    return ActivityRepository()
+    repository = ActivityRepository()
+    embellish_time_series(repository)
+    compute_tile_visits(repository)
+    compute_tile_evolution()
+    return repository

{geo_activity_playground-0.13.0 → geo_activity_playground-0.14.1}/geo_activity_playground/core/activities.py

@@ -10,8 +10,10 @@ import geojson
 import matplotlib
 import numpy as np
 import pandas as pd
+from tqdm import tqdm
 
 from geo_activity_playground.core.config import get_config
+from geo_activity_playground.core.tasks import WorkTracker
 from geo_activity_playground.core.tiles import compute_tile_float
 
 
@@ -43,6 +45,10 @@ class ActivityRepository:
         self.meta["kind"].fillna("Unknown", inplace=True)
         self.meta["equipment"].fillna("Unknown", inplace=True)
 
+    @property
+    def activity_ids(self) -> set[int]:
+        return set(self.meta["id"])
+
     def iter_activities(self, new_to_old=True) -> Iterator[ActivityMeta]:
         direction = -1 if new_to_old else 1
         for id, row in self.meta[::direction].iterrows():
@@ -103,6 +109,53 @@ class ActivityRepository:
         return df
 
 
+def embellish_time_series(repository: ActivityRepository) -> None:
+    work_tracker = WorkTracker("embellish-time-series")
+    activities_to_process = work_tracker.filter(repository.activity_ids)
+    for activity_id in tqdm(activities_to_process, desc="Embellish time series data"):
+        path = pathlib.Path(f"Cache/Activity Timeseries/{activity_id}.parquet")
+        df = pd.read_parquet(path)
+        df.name = id
+        changed = False
+        if pd.api.types.is_dtype_equal(df["time"].dtype, "int64"):
+            start = repository.get_activity_by_id(activity_id).start
+            time = df["time"]
+            del df["time"]
+            df["time"] = [start + datetime.timedelta(seconds=t) for t in time]
+            changed = True
+        assert pd.api.types.is_dtype_equal(df["time"].dtype, "datetime64[ns, UTC]")
+
+        if "distance" in df.columns:
+            if "distance/km" not in df.columns:
+                df["distance/km"] = df["distance"] / 1000
+                changed = True
+
+            if "speed" not in df.columns:
+                df["speed"] = (
+                    df["distance"].diff()
+                    / (df["time"].diff().dt.total_seconds() + 1e-3)
+                    * 3.6
+                )
+                changed = True
+
+        if "x" not in df.columns:
+            x, y = compute_tile_float(df["latitude"], df["longitude"], 0)
+            df["x"] = x
+            df["y"] = y
+            changed = True
+
+        if "segment_id" not in df.columns:
+            time_diff = (df["time"] - df["time"].shift(1)).dt.total_seconds()
+            jump_indices = time_diff >= 30
+            df["segment_id"] = np.cumsum(jump_indices)
+            changed = True
+
+        if changed:
+            df.to_parquet(path)
+        work_tracker.mark_done(activity_id)
+    work_tracker.close()
+
+
 def make_geojson_from_time_series(time_series: pd.DataFrame) -> str:
     line = geojson.LineString(
         [

{geo_activity_playground-0.13.0 → geo_activity_playground-0.14.1}/geo_activity_playground/core/activity_parsers.py

@@ -9,12 +9,59 @@ import fitdecode
 import gpxpy
 import pandas as pd
 import tcxreader.tcxreader
+import xmltodict
 
 
 class ActivityParseError(BaseException):
     pass
 
 
+def read_activity(path: pathlib.Path) -> pd.DataFrame:
+    suffixes = path.suffixes
+    if suffixes[-1] == ".gz":
+        opener = gzip.open
+        file_type = suffixes[-2]
+    else:
+        opener = open
+        file_type = suffixes[-1]
+
+    if file_type == ".gpx":
+        try:
+            df = read_gpx_activity(path, opener)
+        except gpxpy.gpx.GPXXMLSyntaxException as e:
+            raise ActivityParseError("Syntax error while parsing GPX file") from e
+    elif file_type == ".fit":
+        df = read_fit_activity(path, opener)
+    elif file_type == ".tcx":
+        try:
+            df = read_tcx_activity(path, opener)
+        except xml.etree.ElementTree.ParseError as e:
+            raise ActivityParseError("Syntax error in TCX file") from e
+    elif file_type in [".kml", ".kmz"]:
+        df = read_kml_activity(path, opener)
+    else:
+        raise ActivityParseError(f"Unsupported file format: {file_type}")
+
+    if len(df):
+        try:
+            if df.time.dt.tz is not None:
+                df.time = df.time.dt.tz_localize(None)
+        except AttributeError as e:
+            print(df)
+            print(df.dtypes)
+            types = {}
+            for elem in df["time"]:
+                t = str(type(elem))
+                if t not in types:
+                    types[t] = elem
+            print(types)
+            raise ActivityParseError(
+                "It looks like the date parsing has gone wrong."
+            ) from e
+    df.name = path.stem.split(".")[0]
+    return df
+
+
 def read_fit_activity(path: pathlib.Path, open) -> pd.DataFrame:
     """
     {'timestamp': datetime.datetime(2023, 11, 11, 16, 29, 49, tzinfo=datetime.timezone.utc),
@@ -45,8 +92,11 @@ def read_fit_activity(path: pathlib.Path, open) -> pd.DataFrame:
                         and fields.get("position_lat", None)
                         and fields.get("position_long", None)
                     ):
+                        time = fields["timestamp"]
+                        assert isinstance(time, datetime.datetime)
+                        time = time.astimezone(datetime.timezone.utc)
                         row = {
-                            "time": fields["timestamp"],
+                            "time": time,
                             "latitude": fields["position_lat"] / ((2**32) / 360),
                             "longitude": fields["position_long"] / ((2**32) / 360),
                         }
@@ -82,6 +132,8 @@ def read_gpx_activity(path: pathlib.Path, open) -> pd.DataFrame:
                         time = point.time
                     else:
                         time = dateutil.parser.parse(str(point.time))
+                    assert isinstance(time, datetime.datetime)
+                    time = time.astimezone(datetime.timezone.utc)
                     points.append((time, point.latitude, point.longitude))
 
     return pd.DataFrame(points, columns=["time", "latitude", "longitude"])
@@ -110,8 +162,11 @@ def read_tcx_activity(path: pathlib.Path, open) -> pd.DataFrame:
 
     for trackpoint in data.trackpoints:
         if trackpoint.latitude and trackpoint.longitude:
+            time = trackpoint.time
+            assert isinstance(time, datetime.datetime)
+            time = time.astimezone(datetime.timezone.utc)
             row = {
-                "time": trackpoint.time,
+                "time": time,
                 "latitude": trackpoint.latitude,
                 "longitude": trackpoint.longitude,
             }
@@ -128,45 +183,24 @@ def read_tcx_activity(path: pathlib.Path, open) -> pd.DataFrame:
     return df
 
 
-def read_activity(path: pathlib.Path) -> pd.DataFrame:
-    suffixes = path.suffixes
-    if suffixes[-1] == ".gz":
-        opener = gzip.open
-        file_type = suffixes[-2]
-    else:
-        opener = open
-        file_type = suffixes[-1]
-
-    if file_type == ".gpx":
-        try:
-            df = read_gpx_activity(path, opener)
-        except gpxpy.gpx.GPXXMLSyntaxException as e:
-            raise ActivityParseError("Syntax error while parsing GPX file") from e
-    elif file_type == ".fit":
-        df = read_fit_activity(path, opener)
-    elif file_type == ".tcx":
-        try:
-            df = read_tcx_activity(path, opener)
-        except xml.etree.ElementTree.ParseError as e:
-            raise ActivityParseError("Syntax error in TCX file") from e
-    else:
-        raise ActivityParseError(f"Unsupported file format: {file_type}")
-
-    if len(df):
-        try:
-            if df.time.dt.tz is not None:
-                df.time = df.time.dt.tz_localize(None)
-        except AttributeError as e:
-            print(df)
-            print(df.dtypes)
-            types = {}
-            for elem in df["time"]:
-                t = str(type(elem))
-                if t not in types:
-                    types[t] = elem
-            print(types)
-            raise ActivityParseError(
-                "It looks like the date parsing has gone wrong."
-            ) from e
-    df.name = path.stem.split(".")[0]
-    return df
+def read_kml_activity(path: pathlib.Path, opener) -> pd.DataFrame:
+    with opener(path, "rb") as f:
+        kml_dict = xmltodict.parse(f)
+    doc = kml_dict["kml"]["Document"]
+    keypoint_folder = doc["Folder"]
+    placemark = keypoint_folder["Placemark"]
+    track = placemark["gx:Track"]
+    rows = []
+    for when, where in zip(track["when"], track["gx:coord"]):
+        time = dateutil.parser.parse(when).astimezone(datetime.timezone.utc)
+        parts = where.split(" ")
+        if len(parts) == 2:
+            lon, lat = parts
+            alt = None
+        if len(parts) == 3:
+            lon, lat, alt = parts
+        row = {"time": time, "latitude": float(lat), "longitude": float(lon)}
+        if alt is not None:
+            row["altitude"] = float(alt)
+        rows.append(row)
+    return pd.DataFrame(rows)

{geo_activity_playground-0.13.0 → geo_activity_playground-0.14.1}/geo_activity_playground/core/heatmap.py

@@ -5,7 +5,6 @@ import dataclasses
 import functools
 import logging
 import pathlib
-import pickle
 
 import matplotlib.pyplot as pl
 import numpy as np
@@ -16,46 +15,11 @@ from geo_activity_playground.core.tasks import work_tracker
 from geo_activity_playground.core.tiles import compute_tile_float
 from geo_activity_playground.core.tiles import get_tile
 from geo_activity_playground.core.tiles import get_tile_upper_left_lat_lon
-from geo_activity_playground.explorer.converters import get_first_tiles
 
 
 logger = logging.getLogger(__name__)
 
 
-@functools.cache
-def compute_activities_per_tile(
-    repository: ActivityRepository,
-) -> dict[int, dict[tuple[int, int], set[int]]]:
-    logger.info("Extracting activities per tile …")
-    cache_path = pathlib.Path("Cache/activities-per-tile.pickle")
-    if cache_path.exists():
-        with open(cache_path, "rb") as f:
-            data = pickle.load(f)
-    else:
-        data: dict[int, dict[tuple[int, int], set[int]]] = {}
-    with work_tracker(pathlib.Path("Cache/activities-per-tile-task.json")) as tracker:
-        for activity in repository.iter_activities():
-            if activity.id in tracker:
-                continue
-            tracker.add(activity.id)
-
-            logger.info(f"Add activity {activity.id} to all zoom levels …")
-            for zoom in range(1, 20):
-                if zoom not in data:
-                    data[zoom] = {}
-                tiles_this_activity = get_first_tiles(activity.id, repository, zoom)
-                for _, row in tiles_this_activity.iterrows():
-                    tile = (row["tile_x"], row["tile_y"])
-                    if tile not in data[zoom]:
-                        data[zoom][tile] = set()
-                    data[zoom][tile].add(activity.id)
-
-    with open(cache_path, "wb") as f:
-        pickle.dump(data, f)
-
-    return data
-
-
 @functools.cache
 def get_all_points(repository: ActivityRepository) -> pd.DataFrame:
     logger.info("Gathering all points …")
@@ -77,12 +41,6 @@ def get_all_points(repository: ActivityRepository) -> pd.DataFrame:
                 continue
             shard = time_series[["latitude", "longitude"]].copy()
             shard["activity_id"] = activity.id
-            x, y = compute_tile_float(
-                shard["latitude"],
-                shard["longitude"],
-            )
-            shard["x"] = x
-            shard["y"] = y
             new_shards.append(shard)
     logger.info("Concatenating shards …")
     all_points = pd.concat([all_points] + new_shards)

geo_activity_playground-0.14.1/geo_activity_playground/core/tasks.py

@@ -0,0 +1,49 @@
+import contextlib
+import json
+import pathlib
+import pickle
+from typing import Any
+
+
+@contextlib.contextmanager
+def work_tracker(path: pathlib.Path):
+    if path.exists():
+        with open(path) as f:
+            s = set(json.load(f))
+    else:
+        s = set()
+
+    yield s
+
+    with open(path, "w") as f:
+        json.dump(list(s), f)
+
+
+class WorkTracker:
+    def __init__(self, name: str) -> None:
+        self._path = pathlib.Path(f"Cache/work-tracker-{name}.pickle")
+
+        if self._path.exists():
+            with open(self._path, "rb") as f:
+                self._done = pickle.load(f)
+        else:
+            self._done = set()
+
+    def filter(self, ids: list[int]) -> set[int]:
+        return set(ids) - self._done
+
+    def mark_done(self, id: int) -> None:
+        self._done.add(id)
+
+    def close(self) -> None:
+        with open(self._path, "wb") as f:
+            pickle.dump(self._done, f)
+
+
+def try_load_pickle(path: pathlib.Path) -> Any:
+    if path.exists():
+        try:
+            with open(path, "rb") as f:
+                return pickle.load(f)
+        except ModuleNotFoundError:
+            pass

{geo_activity_playground-0.13.0 → geo_activity_playground-0.14.1}/geo_activity_playground/core/tiles.py

@@ -3,6 +3,7 @@ import logging
 import math
 import pathlib
 import time
+from typing import Iterator
 from typing import Optional
 
 import numpy as np
@@ -21,7 +22,7 @@ def compute_tile(lat: float, lon: float, zoom: int) -> tuple[int, int]:
     return int(x * n), int(y * n)
 
 
-def compute_tile_float(lat: float, lon: float, zoom: int) -> tuple[int, int]:
+def compute_tile_float(lat: float, lon: float, zoom: int) -> tuple[float, float]:
     x = np.radians(lon)
     y = np.arcsinh(np.tan(np.radians(lat)))
     x = (1 + x / np.pi) / 2
@@ -98,3 +99,11 @@ def interpolate_missing_tile(
         return (int(x2), y_hat)
     else:
         return (int(x1), y_hat)
+
+
+def adjacent_to(tile: tuple[int, int]) -> Iterator[tuple[int, int]]:
+    x, y = tile
+    yield (x + 1, y)
+    yield (x - 1, y)
+    yield (x, y + 1)
+    yield (x, y - 1)

geo_activity_playground-0.14.1/geo_activity_playground/explorer/grid_file.py

@@ -0,0 +1,102 @@
+import logging
+from typing import Iterator
+from typing import Optional
+
+import geojson
+import gpxpy
+import pandas as pd
+
+from geo_activity_playground.core.coordinates import Bounds
+from geo_activity_playground.core.tiles import adjacent_to
+from geo_activity_playground.core.tiles import get_tile_upper_left_lat_lon
+
+
+logger = logging.getLogger(__name__)
+
+
+def get_border_tiles(
+    tiles: pd.DataFrame, zoom: int, tile_bounds: Bounds
+) -> list[list[list[float]]]:
+    logger.info("Generate border tiles …")
+    tile_set = set(zip(tiles["tile_x"], tiles["tile_y"]))
+    border_tiles = set()
+    for tile in tile_set:
+        for neighbor in adjacent_to(tile):
+            if neighbor not in tile_set:
+                for neighbor2 in adjacent_to(neighbor):
+                    if neighbor2 not in tile_set and tile_bounds.contains(*neighbor):
+                        border_tiles.add(neighbor2)
+    return make_grid_points(border_tiles, zoom)
+
+
+def get_explored_tiles(tiles: pd.DataFrame, zoom: int) -> list[list[list[float]]]:
+    return make_grid_points(zip(tiles["tile_x"], tiles["tile_y"]), zoom)
+
+
+def make_explorer_tile(
+    tile_x: int, tile_y: int, properties: dict, zoom: int
+) -> geojson.Feature:
+    return make_explorer_rectangle(
+        tile_x, tile_y, tile_x + 1, tile_y + 1, zoom, properties
+    )
+
+
+def make_explorer_rectangle(
+    x1: int, y1: int, x2: int, y2: int, zoom: int, properties: Optional[dict] = None
+) -> geojson.Feature:
+    corners = [
+        get_tile_upper_left_lat_lon(*args)
+        for args in [
+            (x1, y1, zoom),
+            (x2, y1, zoom),
+            (x2, y2, zoom),
+            (x1, y2, zoom),
+            (x1, y1, zoom),
+        ]
+    ]
+    return geojson.Feature(
+        geometry=geojson.Polygon([[(coord[1], coord[0]) for coord in corners]]),
+        properties=properties,
+    )
+
+
+def make_grid_points(
+    tile_iterator: Iterator[tuple[int, int]], zoom: int
+) -> list[list[list[float]]]:
+    result = []
+    for tile_x, tile_y in tile_iterator:
+        tile = [
+            get_tile_upper_left_lat_lon(tile_x, tile_y, zoom),
+            get_tile_upper_left_lat_lon(tile_x + 1, tile_y, zoom),
+            get_tile_upper_left_lat_lon(tile_x + 1, tile_y + 1, zoom),
+            get_tile_upper_left_lat_lon(tile_x, tile_y + 1, zoom),
+            get_tile_upper_left_lat_lon(tile_x, tile_y, zoom),
+        ]
+        result.append(tile)
+    return result
+
+
+def make_grid_file_gpx(grid_points: list[list[list[float]]]) -> str:
+    gpx = gpxpy.gpx.GPX()
+    gpx_track = gpxpy.gpx.GPXTrack()
+    gpx.tracks.append(gpx_track)
+
+    for points in grid_points:
+        gpx_segment = gpxpy.gpx.GPXTrackSegment()
+        gpx_track.segments.append(gpx_segment)
+        for point in points:
+            gpx_segment.points.append(gpxpy.gpx.GPXTrackPoint(*point))
+    return gpx.to_xml()
+
+
+def make_grid_file_geojson(grid_points: list[list[list[float]]]) -> str:
+    fc = geojson.FeatureCollection(
+        [
+            geojson.Feature(
+                geometry=geojson.Polygon([[[lon, lat] for lat, lon in points]])
+            )
+            for points in grid_points
+        ]
+    )
+    result = geojson.dumps(fc, sort_keys=True, indent=4, ensure_ascii=False)
+    return result