geo-activity-playground 0.19.1__tar.gz → 0.21.0__tar.gz

{geo_activity_playground-0.19.1 → geo_activity_playground-0.21.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: geo-activity-playground
-Version: 0.19.1
+Version: 0.21.0
 Summary: Analysis of geo data activities like rides, runs or hikes.
 License: MIT
 Author: Martin Ueding
@@ -32,7 +32,7 @@ Requires-Dist: stravalib (>=1.3.3,<2.0.0)
 Requires-Dist: tcxreader (>=0.4.5,<0.5.0)
 Requires-Dist: tomli (>=2.0.1,<3.0.0) ; python_version < "3.11"
 Requires-Dist: tqdm (>=4.64.0,<5.0.0)
-Requires-Dist: vegafusion (>=1.4.3,<2.0.0)
 Requires-Dist: vegafusion-python-embed (>=1.4.3,<2.0.0)
+Requires-Dist: vegafusion[embed] (>=1.4.3,<2.0.0)
 Requires-Dist: vl-convert-python (>=1.0.1,<2.0.0)
 Requires-Dist: xmltodict (>=0.13.0,<0.14.0)

{geo_activity_playground-0.19.1 → geo_activity_playground-0.21.0}/geo_activity_playground/__main__.py

@@ -104,15 +104,30 @@ def make_activity_repository(
     apply_cache_migrations()
     config = get_config()
 
+    if not config.get("prefer_metadata_from_file", True):
+        logger.error(
+            "The config option `prefer_metadata_from_file` is deprecated. If you want to prefer extract metadata from the activity file paths, please use the new `metadata_extraction_regexes` as explained at https://martin-ueding.github.io/geo-activity-playground/getting-started/using-activity-files/#directory-structure."
+        )
+        sys.exit(1)
+
     repository = ActivityRepository()
 
     if pathlib.Path("Activities").exists():
-        import_from_directory(repository, config.get("prefer_metadata_from_file", True))
+        import_from_directory(
+            repository,
+            config.get("metadata_extraction_regexes", []),
+        )
     if pathlib.Path("Strava Export").exists():
         import_from_strava_checkout(repository)
     if "strava" in config and not skip_strava:
         import_from_strava_api(repository)
 
+    if len(repository) == 0:
+        logger.error(
+            f"No activities found. You need to either add activity files (GPX, FIT, …) to {basedir/'Activities'} or set up the Strava API. Starting without any activities is unfortunately not supported."
+        )
+        sys.exit(1)
+
     embellish_time_series(repository)
     compute_tile_visits(repository)
     compute_tile_evolution()

{geo_activity_playground-0.19.1 → geo_activity_playground-0.21.0}/geo_activity_playground/core/activities.py

@@ -1,6 +1,7 @@
 import datetime
 import functools
 import logging
+import pathlib
 from typing import Iterator
 from typing import Optional
 from typing import TypedDict
@@ -12,6 +13,7 @@ import pandas as pd
 from tqdm import tqdm
 
 from geo_activity_playground.core.config import get_config
+from geo_activity_playground.core.coordinates import get_distance
 from geo_activity_playground.core.paths import activities_path
 from geo_activity_playground.core.paths import activity_timeseries_path
 from geo_activity_playground.core.tasks import WorkTracker
@@ -25,12 +27,17 @@ class ActivityMeta(TypedDict):
     commute: bool
     distance_km: float
     elapsed_time: datetime.timedelta
+    end_latitude: float
+    end_longitude: float
     equipment: str
     id: int
     kind: str
     name: str
     path: str
+    start_latitude: float
+    start_longitude: float
     start: datetime.datetime
+    steps: int
 
 
 class ActivityRepository:
@@ -43,15 +50,27 @@ class ActivityRepository:
             self.meta = pd.DataFrame()
 
         self._loose_activities: list[ActivityMeta] = []
+        self._loose_activity_ids: set[int] = set()
 
     def __len__(self) -> int:
         return len(self.meta)
 
     def add_activity(self, activity_meta: ActivityMeta) -> None:
-        assert not self.has_activity(
-            activity_meta["id"]
-        ), f"Trying to add the following activity which already exists: {activity_meta}"
+        _extend_metadata_from_timeseries(activity_meta)
+        if activity_meta["id"] in self._loose_activity_ids:
+            logger.error(f"Activity with the same file already exists. New activity:")
+            print(activity_meta)
+            print("Existing activity:")
+            print(
+                [
+                    activity
+                    for activity in self._loose_activities
+                    if activity["id"] == activity_meta["id"]
+                ]
+            )
+            raise ValueError("Activity with the same file already exists.")
         self._loose_activities.append(activity_meta)
+        self._loose_activity_ids.add(activity_meta["id"])
 
     def commit(self) -> None:
         if self._loose_activities:
@@ -59,7 +78,15 @@ class ActivityRepository:
                 f"Adding {len(self._loose_activities)} activities to the repository …"
             )
             new_df = pd.DataFrame(self._loose_activities)
-            self.meta = pd.concat([self.meta, new_df])
+            if len(self.meta):
+                new_ids_set = set(new_df["id"])
+                is_kept = [
+                    activity_id not in new_ids_set for activity_id in self.meta["id"]
+                ]
+                old_df = self.meta.loc[is_kept]
+            else:
+                old_df = self.meta
+            self.meta = pd.concat([old_df, new_df])
             assert pd.api.types.is_dtype_equal(
                 self.meta["start"].dtype, "datetime64[ns, UTC]"
             ), self.meta["start"].dtype
@@ -86,6 +113,8 @@ class ActivityRepository:
     def last_activity_date(self) -> Optional[datetime.datetime]:
         if len(self.meta):
             return self.meta.iloc[-1]["start"]
+        else:
+            return None
 
     @property
     def activity_ids(self) -> set[int]:
@@ -122,58 +151,88 @@ def embellish_time_series(repository: ActivityRepository) -> None:
         path = activity_timeseries_path(activity_id)
         df = pd.read_parquet(path)
         df.name = id
-        changed = False
-        if pd.api.types.is_dtype_equal(df["time"].dtype, "int64"):
-            start = repository.get_activity_by_id(activity_id)["start"]
-            time = df["time"]
-            del df["time"]
-            df["time"] = [start + datetime.timedelta(seconds=t) for t in time]
-            changed = True
-        assert pd.api.types.is_dtype_equal(df["time"].dtype, "datetime64[ns, UTC]")
-
-        if "distance_km" in df.columns:
-            if "speed" not in df.columns:
-                df["speed"] = (
-                    df["distance_km"].diff()
-                    / (df["time"].diff().dt.total_seconds() + 1e-3)
-                    * 3600
-                )
-                changed = True
-
-            potential_jumps = (df["speed"] > 40) & (df["speed"].diff() > 10)
-            if np.any(potential_jumps):
-                df = df.loc[~potential_jumps]
-                changed = True
-
-        if "x" not in df.columns:
-            x, y = compute_tile_float(df["latitude"], df["longitude"], 0)
-            df["x"] = x
-            df["y"] = y
-            changed = True
-
-        if "segment_id" not in df.columns:
-            time_diff = (df["time"] - df["time"].shift(1)).dt.total_seconds()
-            jump_indices = time_diff >= 30
-            df["segment_id"] = np.cumsum(jump_indices)
-            changed = True
-
+        df, changed = embellish_single_time_series(
+            df, repository.get_activity_by_id(activity_id)["start"]
+        )
         if changed:
             df.to_parquet(path)
         work_tracker.mark_done(activity_id)
     work_tracker.close()
 
 
+def embellish_single_time_series(
+    timeseries: pd.DataFrame, start: Optional[datetime.datetime] = None
+) -> bool:
+    changed = False
+
+    if start is not None and pd.api.types.is_dtype_equal(
+        timeseries["time"].dtype, "int64"
+    ):
+        time = timeseries["time"]
+        del timeseries["time"]
+        timeseries["time"] = [start + datetime.timedelta(seconds=t) for t in time]
+        changed = True
+    assert pd.api.types.is_dtype_equal(timeseries["time"].dtype, "datetime64[ns, UTC]")
+
+    distances = get_distance(
+        timeseries["latitude"].shift(1),
+        timeseries["longitude"].shift(1),
+        timeseries["latitude"],
+        timeseries["longitude"],
+    ).fillna(0.0)
+    time_diff_threshold_seconds = 30
+    time_diff = (timeseries["time"] - timeseries["time"].shift(1)).dt.total_seconds()
+    jump_indices = (time_diff >= time_diff_threshold_seconds) & (distances > 100)
+    distances.loc[jump_indices] = 0.0
+
+    if not "distance_km" in timeseries.columns:
+        timeseries["distance_km"] = pd.Series(np.cumsum(distances)) / 1000
+        changed = True
+
+    if "speed" not in timeseries.columns:
+        timeseries["speed"] = (
+            timeseries["distance_km"].diff()
+            / (timeseries["time"].diff().dt.total_seconds() + 1e-3)
+            * 3600
+        )
+        changed = True
+
+    potential_jumps = (timeseries["speed"] > 40) & (timeseries["speed"].diff() > 10)
+    if np.any(potential_jumps):
+        timeseries = timeseries.loc[~potential_jumps].copy()
+        changed = True
+
+    if "segment_id" not in timeseries.columns:
+        timeseries["segment_id"] = np.cumsum(jump_indices)
+        changed = True
+
+    if "x" not in timeseries.columns:
+        x, y = compute_tile_float(timeseries["latitude"], timeseries["longitude"], 0)
+        timeseries["x"] = x
+        timeseries["y"] = y
+        changed = True
+
+    return timeseries, changed
+
+
 def make_geojson_from_time_series(time_series: pd.DataFrame) -> str:
-    line = geojson.LineString(
-        [
-            (lon, lat)
-            for lat, lon in zip(time_series["latitude"], time_series["longitude"])
+    fc = geojson.FeatureCollection(
+        features=[
+            geojson.LineString(
+                [(lon, lat) for lat, lon in zip(group["latitude"], group["longitude"])]
+            )
+            for _, group in time_series.groupby("segment_id")
         ]
     )
-    return geojson.dumps(line)
+    return geojson.dumps(fc)
 
 
 def make_geojson_color_line(time_series: pd.DataFrame) -> str:
+    speed_without_na = time_series["speed"].dropna()
+    low = min(speed_without_na)
+    high = max(speed_without_na)
+    clamp_speed = lambda speed: min(max((speed - low) / (high - low), 0.0), 1.0)
+
     cmap = matplotlib.colormaps["viridis"]
     features = [
         geojson.Feature(
@@ -185,17 +244,29 @@ def make_geojson_color_line(time_series: pd.DataFrame) -> str:
             ),
             properties={
                 "speed": next["speed"] if np.isfinite(next["speed"]) else 0.0,
-                "color": matplotlib.colors.to_hex(cmap(min(next["speed"] / 35, 1.0))),
+                "color": matplotlib.colors.to_hex(cmap(clamp_speed(next["speed"]))),
             },
         )
-        for (_, row), (_, next) in zip(
-            time_series.iterrows(), time_series.iloc[1:].iterrows()
-        )
+        for _, group in time_series.groupby("segment_id")
+        for (_, row), (_, next) in zip(group.iterrows(), group.iloc[1:].iterrows())
     ]
     feature_collection = geojson.FeatureCollection(features)
     return geojson.dumps(feature_collection)
 
 
+def make_speed_color_bar(time_series: pd.DataFrame) -> dict[str, str]:
+    speed_without_na = time_series["speed"].dropna()
+    low = min(speed_without_na)
+    high = max(speed_without_na)
+    cmap = matplotlib.colormaps["viridis"]
+    clamp_speed = lambda speed: min(max((speed - low) / (high - low), 0.0), 1.0)
+    colors = [
+        (f"{speed:.1f}", matplotlib.colors.to_hex(cmap(clamp_speed(speed))))
+        for speed in np.linspace(low, high, 10)
+    ]
+    return {"low": low, "high": high, "colors": colors}
+
+
 def extract_heart_rate_zones(time_series: pd.DataFrame) -> Optional[pd.DataFrame]:
     if "heartrate" not in time_series:
         return None
@@ -236,3 +307,14 @@ def extract_heart_rate_zones(time_series: pd.DataFrame) -> Optional[pd.DataFrame
             duration_per_zone.loc[i] = 0.0
     result = duration_per_zone.reset_index()
     return result
+
+
+def _extend_metadata_from_timeseries(metadata: ActivityMeta) -> None:
+    timeseries = pd.read_parquet(
+        pathlib.Path("Cache/Activity Timeseries") / f"{metadata['id']}.parquet"
+    )
+
+    metadata["start_latitude"] = timeseries["latitude"].iloc[0]
+    metadata["end_latitude"] = timeseries["latitude"].iloc[-1]
+    metadata["start_longitude"] = timeseries["longitude"].iloc[0]
+    metadata["end_longitude"] = timeseries["longitude"].iloc[-1]

{geo_activity_playground-0.19.1 → geo_activity_playground-0.21.0}/geo_activity_playground/core/activity_parsers.py

@@ -14,6 +14,7 @@ import tcxreader.tcxreader
 import xmltodict
 
 from geo_activity_playground.core.activities import ActivityMeta
+from geo_activity_playground.core.activities import embellish_single_time_series
 from geo_activity_playground.core.coordinates import get_distance
 
 logger = logging.getLogger(__name__)
@@ -74,18 +75,7 @@ def read_activity(path: pathlib.Path) -> tuple[ActivityMeta, pd.DataFrame]:
                 "It looks like the date parsing has gone wrong."
             ) from e
 
-        # Add distance column if missing.
-        if "distance_km" not in timeseries.columns:
-            distances = [0] + [
-                get_distance(lat_1, lon_1, lat_2, lon_2)
-                for lat_1, lon_1, lat_2, lon_2 in zip(
-                    timeseries["latitude"],
-                    timeseries["longitude"],
-                    timeseries["latitude"].iloc[1:],
-                    timeseries["longitude"].iloc[1:],
-                )
-            ]
-            timeseries["distance_km"] = pd.Series(np.cumsum(distances)) / 1000
+        timeseries, changed = embellish_single_time_series(timeseries)
 
         # Extract some meta data from the time series.
         metadata["start"] = timeseries["time"].iloc[0]
@@ -178,6 +168,10 @@ def read_fit_activity(path: pathlib.Path, open) -> tuple[ActivityMeta, pd.DataFr
                         metadata["kind"] = str(values["sport"])
                         if "sub_sport" in values:
                             metadata["kind"] += " " + str(values["sub_sport"])
+                    if "total_calories" in fields:
+                        metadata["calories"] = values["total_calories"]
+                    if "total_strides" in fields:
+                        metadata["steps"] = 2 * int(values["total_strides"])
 
     return metadata, pd.DataFrame(rows)
 
@@ -212,9 +206,13 @@ def read_gpx_activity(path: pathlib.Path, open) -> pd.DataFrame:
                     time = dateutil.parser.parse(str(point.time))
                 assert isinstance(time, datetime.datetime)
                 time = time.astimezone(datetime.timezone.utc)
-                points.append((time, point.latitude, point.longitude))
+                points.append((time, point.latitude, point.longitude, point.elevation))
 
-    return pd.DataFrame(points, columns=["time", "latitude", "longitude"])
+    df = pd.DataFrame(points, columns=["time", "latitude", "longitude", "altitude"])
+    # Some files don't have altitude information. In these cases we remove the column.
+    if not df["altitude"].any():
+        del df["altitude"]
+    return df
 
 
 def read_tcx_activity(path: pathlib.Path, opener) -> pd.DataFrame:

{geo_activity_playground-0.19.1 → geo_activity_playground-0.21.0}/geo_activity_playground/core/cache_migrations.py

@@ -95,6 +95,9 @@ def apply_cache_migrations() -> None:
         delete_activity_metadata,
         delete_activity_metadata,
         convert_distances_to_km,
+        delete_activity_metadata,
+        delete_tile_visits,
+        delete_heatmap_cache,
     ]
 
     for migration in migrations[cache_status["num_applied_migrations"] :]:

{geo_activity_playground-0.19.1 → geo_activity_playground-0.21.0}/geo_activity_playground/core/config.py

@@ -21,7 +21,4 @@ def get_config() -> dict:
     with open(config_path, "rb") as f:
         config = tomllib.load(f)
 
-    # Filter out empty config groups.
-    config = {key: value for key, value in config.items() if value}
-
     return config

{geo_activity_playground-0.19.1 → geo_activity_playground-0.21.0}/geo_activity_playground/core/heatmap.py

@@ -4,7 +4,6 @@ This code is based on https://github.com/remisalmon/Strava-local-heatmap.
 import dataclasses
 import logging
 
-import matplotlib.pyplot as pl
 import numpy as np
 
 from geo_activity_playground.core.tiles import compute_tile_float
@@ -147,32 +146,3 @@ def crop_image_to_bounds(
     max_y = int((max_y - tile_bounds.y_tile_min) * OSM_TILE_SIZE)
     image = image[min_y:max_y, min_x:max_x, :]
     return image
-
-
-def gaussian_filter(image, sigma):
-    # returns image filtered with a gaussian function of variance sigma**2
-    #
-    # input: image = numpy.ndarray
-    #        sigma = float
-    # output: image = numpy.ndarray
-
-    i, j = np.meshgrid(
-        np.arange(image.shape[0]), np.arange(image.shape[1]), indexing="ij"
-    )
-
-    mu = (int(image.shape[0] / 2.0), int(image.shape[1] / 2.0))
-
-    gaussian = (
-        1.0
-        / (2.0 * np.pi * sigma * sigma)
-        * np.exp(-0.5 * (((i - mu[0]) / sigma) ** 2 + ((j - mu[1]) / sigma) ** 2))
-    )
-
-    gaussian = np.roll(gaussian, (-mu[0], -mu[1]), axis=(0, 1))
-
-    image_fft = np.fft.rfft2(image)
-    gaussian_fft = np.fft.rfft2(gaussian)
-
-    image = np.fft.irfft2(image_fft * gaussian_fft)
-
-    return image

geo_activity_playground-0.21.0/geo_activity_playground/core/similarity.py

@@ -0,0 +1,93 @@
+import pathlib
+import pickle
+
+import imagehash
+import numpy as np
+import pandas as pd
+from PIL import Image
+from PIL import ImageDraw
+from tqdm import tqdm
+
+from .activities import ActivityRepository
+from .coordinates import get_distance
+from geo_activity_playground.core.tasks import stored_object
+
+
+fingerprint_path = pathlib.Path("Cache/activity_fingerprints.pickle")
+distances_path = pathlib.Path("Cache/activity_distances.pickle")
+
+
+def add_distance(distances, this, other, distance) -> None:
+    if this not in distances:
+        distances[this] = {}
+    if distance not in distances[this]:
+        distances[this][distance] = set()
+    distances[this][distance].add(other)
+
+
+def precompute_activity_distances(repository: ActivityRepository) -> None:
+    with stored_object(fingerprint_path, {}) as fingerprints, stored_object(
+        distances_path, {}
+    ) as distances:
+        activity_ids = repository.activity_ids
+
+        activity_ids_without_fingerprint = [
+            activity_id
+            for activity_id in activity_ids
+            if activity_id not in fingerprints
+        ]
+        for activity_id in tqdm(
+            activity_ids_without_fingerprint, desc="Compute activity fingerprints"
+        ):
+            ts = repository.get_time_series(activity_id)
+            ts_hash = _compute_image_hash(ts)
+            fingerprints[activity_id] = ts_hash
+
+        for this in tqdm(
+            activity_ids_without_fingerprint, desc="Compute activity distances"
+        ):
+            for other in activity_ids:
+                distance = _hamming_distance(fingerprints[this], fingerprints[other])
+                add_distance(distances, this, other, distance)
+                add_distance(distances, other, this, distance)
+
+
+def asymmetric_activity_overlap(
+    activity: pd.DataFrame, reference: pd.DataFrame
+) -> float:
+    sample = activity.iloc[np.linspace(0, len(activity) - 1, 50, dtype=np.int64)]
+    min_distances = [
+        _get_min_distance(latitude, longitude, reference)
+        for (latitude, longitude) in zip(sample["latitude"], sample["longitude"])
+    ]
+    return sum(distance < 25 for distance in min_distances) / len(min_distances)
+
+
+def _get_min_distance(latitude: float, longitude: float, other: pd.DataFrame) -> float:
+    distances = get_distance(latitude, longitude, other["latitude"], other["longitude"])
+    return np.min(distances)
+
+
+def _compute_image_hash(time_series) -> int:
+    z = 12 + 8
+    x = time_series["x"] * 2**z
+    y = time_series["y"] * 2**z
+    xy_pixels = np.array([x - x.min(), y - y.min()]).T
+    dim = xy_pixels.max(axis=0)
+    # Some activities have bogus data in them which makes them require a huge image. We just skip those outright and return a dummy hash value.
+    if max(dim) > 6000:
+        return 0
+    im = Image.new("L", tuple(map(int, dim)))
+    draw = ImageDraw.Draw(im)
+    pixels = list(map(int, xy_pixels.flatten()))
+    draw.line(pixels, fill=255, width=5)
+    return int(str(imagehash.dhash(im)), 16)
+
+
+def _hamming_distance(a: int, b: int) -> int:
+    diff = a ^ b
+    result = 0
+    while diff:
+        result += diff % 2
+        diff //= 2
+    return result

{geo_activity_playground-0.19.1 → geo_activity_playground-0.21.0}/geo_activity_playground/core/tasks.py

@@ -2,11 +2,35 @@ import contextlib
 import json
 import pathlib
 import pickle
+from collections.abc import Iterable
 from typing import Any
+from typing import Generic
+from typing import Sequence
+from typing import TypeVar
 
 from geo_activity_playground.core.paths import cache_dir
 
 
+T = TypeVar("T")
+
+
+@contextlib.contextmanager
+def stored_object(path: pathlib.Path, default):
+    if path.exists():
+        with open(path, "rb") as f:
+            payload = pickle.load(f)
+    else:
+        payload = default
+
+    yield payload
+
+    temp_location = path.with_suffix(".tmp")
+    with open(temp_location, "wb") as f:
+        pickle.dump(payload, f)
+    path.unlink(missing_ok=True)
+    temp_location.rename(path)
+
+
 def work_tracker_path(name: str) -> pathlib.Path:
     return cache_dir() / f"work-tracker-{name}.pickle"
 
@@ -35,7 +59,7 @@ class WorkTracker:
         else:
             self._done = set()
 
-    def filter(self, ids: list[int]) -> set[int]:
+    def filter(self, ids: Iterable[int]) -> set[int]:
         return set(ids) - self._done
 
     def mark_done(self, id: int) -> None:

{geo_activity_playground-0.19.1 → geo_activity_playground-0.21.0}/geo_activity_playground/explorer/grid_file.py

@@ -1,6 +1,6 @@
 import json
 import logging
-from typing import Iterator
+from collections.abc import Iterable
 from typing import Optional
 
 import geojson
@@ -8,7 +8,6 @@ import gpxpy
 import pandas as pd
 
 from geo_activity_playground.core.coordinates import Bounds
-from geo_activity_playground.core.tiles import adjacent_to
 from geo_activity_playground.core.tiles import get_tile_upper_left_lat_lon
 
 
@@ -66,10 +65,10 @@ def make_explorer_rectangle(
 
 
 def make_grid_points(
-    tile_iterator: Iterator[tuple[int, int]], zoom: int
+    tiles: Iterable[tuple[int, int]], zoom: int
 ) -> list[list[list[float]]]:
     result = []
-    for tile_x, tile_y in tile_iterator:
+    for tile_x, tile_y in tiles:
         tile = [
             get_tile_upper_left_lat_lon(tile_x, tile_y, zoom),
             get_tile_upper_left_lat_lon(tile_x + 1, tile_y, zoom),

{geo_activity_playground-0.19.1 → geo_activity_playground-0.21.0}/geo_activity_playground/explorer/tile_visits.py

@@ -6,6 +6,7 @@ import pathlib
 import pickle
 from typing import Any
 from typing import Iterator
+from typing import Optional
 
 import pandas as pd
 from tqdm import tqdm
@@ -116,10 +117,10 @@ class TileEvolutionState:
         self.square_start = 0
         self.cluster_start = 0
         self.max_square_size = 0
-        self.visited_tiles = set()
+        self.visited_tiles: set[tuple[int, int]] = set()
         self.square_evolution = pd.DataFrame()
-        self.square_x = None
-        self.square_y = None
+        self.square_x: Optional[int] = None
+        self.square_y: Optional[int] = None
 
 
 def compute_tile_evolution() -> None: