geo-activity-playground 0.20.0__tar.gz → 0.21.0__tar.gz

{geo_activity_playground-0.20.0 → geo_activity_playground-0.21.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: geo-activity-playground
-Version: 0.20.0
+Version: 0.21.0
 Summary: Analysis of geo data activities like rides, runs or hikes.
 License: MIT
 Author: Martin Ueding
@@ -20,7 +20,6 @@ Requires-Dist: fitdecode (>=0.10.0,<0.11.0)
 Requires-Dist: flask (>=3.0.0,<4.0.0)
 Requires-Dist: geojson (>=3.0.1,<4.0.0)
 Requires-Dist: gpxpy (>=1.5.0,<2.0.0)
-Requires-Dist: imagehash (>=4.3.1,<5.0.0)
 Requires-Dist: jinja2 (>=3.1.2,<4.0.0)
 Requires-Dist: matplotlib (>=3.6.3,<4.0.0)
 Requires-Dist: numpy (>=1.22.4,<2.0.0)

{geo_activity_playground-0.20.0 → geo_activity_playground-0.21.0}/geo_activity_playground/__main__.py

@@ -6,7 +6,6 @@ import sys
 
 import coloredlogs
 
-from .core.similarity import precompute_activity_distances
 from .importers.strava_checkout import convert_strava_checkout
 from .importers.strava_checkout import import_from_strava_checkout
 from geo_activity_playground.core.activities import ActivityRepository
@@ -105,17 +104,31 @@ def make_activity_repository(
     apply_cache_migrations()
     config = get_config()
 
+    if not config.get("prefer_metadata_from_file", True):
+        logger.error(
+            "The config option `prefer_metadata_from_file` is deprecated. If you want to prefer extract metadata from the activity file paths, please use the new `metadata_extraction_regexes` as explained at https://martin-ueding.github.io/geo-activity-playground/getting-started/using-activity-files/#directory-structure."
+        )
+        sys.exit(1)
+
     repository = ActivityRepository()
 
     if pathlib.Path("Activities").exists():
-        import_from_directory(repository, config.get("prefer_metadata_from_file", True))
+        import_from_directory(
+            repository,
+            config.get("metadata_extraction_regexes", []),
+        )
     if pathlib.Path("Strava Export").exists():
         import_from_strava_checkout(repository)
     if "strava" in config and not skip_strava:
         import_from_strava_api(repository)
 
+    if len(repository) == 0:
+        logger.error(
+            f"No activities found. You need to either add activity files (GPX, FIT, …) to {basedir/'Activities'} or set up the Strava API. Starting without any activities is unfortunately not supported."
+        )
+        sys.exit(1)
+
     embellish_time_series(repository)
-    precompute_activity_distances(repository)
     compute_tile_visits(repository)
     compute_tile_evolution()
     return repository

{geo_activity_playground-0.20.0 → geo_activity_playground-0.21.0}/geo_activity_playground/core/activities.py

@@ -37,6 +37,7 @@ class ActivityMeta(TypedDict):
     start_latitude: float
     start_longitude: float
     start: datetime.datetime
+    steps: int
 
 
 class ActivityRepository:
@@ -163,9 +164,6 @@ def embellish_single_time_series(
     timeseries: pd.DataFrame, start: Optional[datetime.datetime] = None
 ) -> bool:
     changed = False
-    time_diff_threshold_seconds = 30
-    time_diff = (timeseries["time"] - timeseries["time"].shift(1)).dt.total_seconds()
-    jump_indices = time_diff >= time_diff_threshold_seconds
 
     if start is not None and pd.api.types.is_dtype_equal(
         timeseries["time"].dtype, "int64"
@@ -176,31 +174,37 @@ def embellish_single_time_series(
         changed = True
     assert pd.api.types.is_dtype_equal(timeseries["time"].dtype, "datetime64[ns, UTC]")
 
-    # Add distance column if missing.
-    if "distance_km" not in timeseries.columns:
-        distances = get_distance(
-            timeseries["latitude"].shift(1),
-            timeseries["longitude"].shift(1),
-            timeseries["latitude"],
-            timeseries["longitude"],
-        ).fillna(0.0)
-        distances.loc[jump_indices] = 0.0
+    distances = get_distance(
+        timeseries["latitude"].shift(1),
+        timeseries["longitude"].shift(1),
+        timeseries["latitude"],
+        timeseries["longitude"],
+    ).fillna(0.0)
+    time_diff_threshold_seconds = 30
+    time_diff = (timeseries["time"] - timeseries["time"].shift(1)).dt.total_seconds()
+    jump_indices = (time_diff >= time_diff_threshold_seconds) & (distances > 100)
+    distances.loc[jump_indices] = 0.0
+
+    if not "distance_km" in timeseries.columns:
         timeseries["distance_km"] = pd.Series(np.cumsum(distances)) / 1000
         changed = True
 
-    if "distance_km" in timeseries.columns:
-        if "speed" not in timeseries.columns:
-            timeseries["speed"] = (
-                timeseries["distance_km"].diff()
-                / (timeseries["time"].diff().dt.total_seconds() + 1e-3)
-                * 3600
-            )
-            changed = True
+    if "speed" not in timeseries.columns:
+        timeseries["speed"] = (
+            timeseries["distance_km"].diff()
+            / (timeseries["time"].diff().dt.total_seconds() + 1e-3)
+            * 3600
+        )
+        changed = True
 
-        potential_jumps = (timeseries["speed"] > 40) & (timeseries["speed"].diff() > 10)
-        if np.any(potential_jumps):
-            timeseries = timeseries.loc[~potential_jumps]
-            changed = True
+    potential_jumps = (timeseries["speed"] > 40) & (timeseries["speed"].diff() > 10)
+    if np.any(potential_jumps):
+        timeseries = timeseries.loc[~potential_jumps].copy()
+        changed = True
+
+    if "segment_id" not in timeseries.columns:
+        timeseries["segment_id"] = np.cumsum(jump_indices)
+        changed = True
 
     if "x" not in timeseries.columns:
         x, y = compute_tile_float(timeseries["latitude"], timeseries["longitude"], 0)
@@ -208,10 +212,6 @@ def embellish_single_time_series(
         timeseries["y"] = y
         changed = True
 
-    if "segment_id" not in timeseries.columns:
-        timeseries["segment_id"] = np.cumsum(jump_indices)
-        changed = True
-
     return timeseries, changed
 
 
@@ -228,6 +228,11 @@ def make_geojson_from_time_series(time_series: pd.DataFrame) -> str:
 
 
 def make_geojson_color_line(time_series: pd.DataFrame) -> str:
+    speed_without_na = time_series["speed"].dropna()
+    low = min(speed_without_na)
+    high = max(speed_without_na)
+    clamp_speed = lambda speed: min(max((speed - low) / (high - low), 0.0), 1.0)
+
     cmap = matplotlib.colormaps["viridis"]
     features = [
         geojson.Feature(
@@ -239,7 +244,7 @@ def make_geojson_color_line(time_series: pd.DataFrame) -> str:
             ),
             properties={
                 "speed": next["speed"] if np.isfinite(next["speed"]) else 0.0,
-                "color": matplotlib.colors.to_hex(cmap(min(next["speed"] / 35, 1.0))),
+                "color": matplotlib.colors.to_hex(cmap(clamp_speed(next["speed"]))),
             },
         )
         for _, group in time_series.groupby("segment_id")
@@ -249,6 +254,19 @@ def make_geojson_color_line(time_series: pd.DataFrame) -> str:
     return geojson.dumps(feature_collection)
 
 
+def make_speed_color_bar(time_series: pd.DataFrame) -> dict[str, str]:
+    speed_without_na = time_series["speed"].dropna()
+    low = min(speed_without_na)
+    high = max(speed_without_na)
+    cmap = matplotlib.colormaps["viridis"]
+    clamp_speed = lambda speed: min(max((speed - low) / (high - low), 0.0), 1.0)
+    colors = [
+        (f"{speed:.1f}", matplotlib.colors.to_hex(cmap(clamp_speed(speed))))
+        for speed in np.linspace(low, high, 10)
+    ]
+    return {"low": low, "high": high, "colors": colors}
+
+
 def extract_heart_rate_zones(time_series: pd.DataFrame) -> Optional[pd.DataFrame]:
     if "heartrate" not in time_series:
         return None

{geo_activity_playground-0.20.0 → geo_activity_playground-0.21.0}/geo_activity_playground/core/activity_parsers.py

@@ -168,6 +168,10 @@ def read_fit_activity(path: pathlib.Path, open) -> tuple[ActivityMeta, pd.DataFr
                         metadata["kind"] = str(values["sport"])
                         if "sub_sport" in values:
                             metadata["kind"] += " " + str(values["sub_sport"])
+                    if "total_calories" in fields:
+                        metadata["calories"] = values["total_calories"]
+                    if "total_strides" in fields:
+                        metadata["steps"] = 2 * int(values["total_strides"])
 
     return metadata, pd.DataFrame(rows)
 

geo_activity_playground-0.21.0/geo_activity_playground/importers/directory.py

@@ -0,0 +1,127 @@
+import hashlib
+import logging
+import multiprocessing
+import pathlib
+import pickle
+import re
+import sys
+import traceback
+from typing import Optional
+
+import pandas as pd
+from tqdm import tqdm
+
+from geo_activity_playground.core.activities import ActivityMeta
+from geo_activity_playground.core.activities import ActivityRepository
+from geo_activity_playground.core.activity_parsers import ActivityParseError
+from geo_activity_playground.core.activity_parsers import read_activity
+from geo_activity_playground.core.tasks import WorkTracker
+
+logger = logging.getLogger(__name__)
+
+ACTIVITY_DIR = pathlib.Path("Activities")
+
+
+def import_from_directory(
+    repository: ActivityRepository, metadata_extraction_regexes: list[str] = []
+) -> None:
+    paths_with_errors = []
+    work_tracker = WorkTracker("parse-activity-files")
+
+    activity_paths = [
+        path
+        for path in ACTIVITY_DIR.rglob("*.*")
+        if path.is_file() and path.suffixes and not path.stem.startswith(".")
+    ]
+    new_activity_paths = work_tracker.filter(activity_paths)
+
+    activity_stream_dir = pathlib.Path("Cache/Activity Timeseries")
+    activity_stream_dir.mkdir(exist_ok=True, parents=True)
+    file_metadata_dir = pathlib.Path("Cache/Activity Metadata")
+    file_metadata_dir.mkdir(exist_ok=True, parents=True)
+
+    with multiprocessing.Pool() as pool:
+        paths_with_errors = tqdm(
+            pool.imap(_cache_single_file, new_activity_paths),
+            desc="Parse activity metadata",
+            total=len(new_activity_paths),
+        )
+        paths_with_errors = [error for error in paths_with_errors if error]
+
+    for path in tqdm(new_activity_paths, desc="Collate activity metadata"):
+        activity_id = _get_file_hash(path)
+        file_metadata_path = file_metadata_dir / f"{activity_id}.pickle"
+        work_tracker.mark_done(activity_id)
+
+        if not file_metadata_path.exists():
+            continue
+
+        with open(file_metadata_path, "rb") as f:
+            activity_meta_from_file = pickle.load(f)
+
+        activity_meta = ActivityMeta(
+            id=activity_id,
+            # https://stackoverflow.com/a/74718395/653152
+            name=path.name.removesuffix("".join(path.suffixes)),
+            path=str(path),
+            kind="Unknown",
+            equipment="Unknown",
+        )
+        activity_meta.update(activity_meta_from_file)
+        activity_meta.update(_get_metadata_from_path(path, metadata_extraction_regexes))
+        repository.add_activity(activity_meta)
+
+    if paths_with_errors:
+        logger.warning(
+            "There were errors while parsing some of the files. These were skipped and tried again next time."
+        )
+        for path, error in paths_with_errors:
+            logger.error(f"{path}: {error}")
+
+    repository.commit()
+
+    work_tracker.close()
+
+
+def _cache_single_file(path: pathlib.Path) -> Optional[tuple[pathlib.Path, str]]:
+    activity_stream_dir = pathlib.Path("Cache/Activity Timeseries")
+    file_metadata_dir = pathlib.Path("Cache/Activity Metadata")
+
+    activity_id = _get_file_hash(path)
+    timeseries_path = activity_stream_dir / f"{activity_id}.parquet"
+    file_metadata_path = file_metadata_dir / f"{activity_id}.pickle"
+
+    if not timeseries_path.exists():
+        try:
+            activity_meta_from_file, timeseries = read_activity(path)
+        except ActivityParseError as e:
+            logger.error(f"Error while parsing file {path}:")
+            traceback.print_exc()
+            return (path, str(e))
+        except:
+            logger.error(f"Encountered a problem with {path=}, see details below.")
+            raise
+
+        if len(timeseries) == 0:
+            return
+
+        timeseries.to_parquet(timeseries_path)
+        with open(file_metadata_path, "wb") as f:
+            pickle.dump(activity_meta_from_file, f)
+
+
+def _get_file_hash(path: pathlib.Path) -> int:
+    file_hash = hashlib.blake2s()
+    with open(path, "rb") as f:
+        while chunk := f.read(8192):
+            file_hash.update(chunk)
+    return int(file_hash.hexdigest(), 16) % 2**62
+
+
+def _get_metadata_from_path(
+    path: pathlib.Path, metadata_extraction_regexes: list[str]
+) -> dict[str, str]:
+    for regex in metadata_extraction_regexes:
+        if m := re.search(regex, str(path.relative_to(ACTIVITY_DIR))):
+            return m.groupdict()
+    return {}

{geo_activity_playground-0.20.0 → geo_activity_playground-0.21.0}/geo_activity_playground/importers/strava_checkout.py

@@ -3,6 +3,8 @@ import logging
 import pathlib
 import shutil
 import traceback
+from typing import Optional
+from typing import Union
 
 import dateutil.parser
 import numpy as np
@@ -116,6 +118,13 @@ EXPECTED_COLUMNS = [
 ]
 
 
+def float_or_none(x: Union[float, str]) -> Optional[float]:
+    try:
+        return float(x)
+    except ValueError:
+        return None
+
+
 def import_from_strava_checkout(repository: ActivityRepository) -> None:
     checkout_path = pathlib.Path("Strava Export")
     activities = pd.read_csv(checkout_path / "activities.csv")
@@ -143,7 +152,7 @@ def import_from_strava_checkout(repository: ActivityRepository) -> None:
         row = activities.loc[activity_id]
         activity_file = checkout_path / row["Filename"]
         table_activity_meta = {
-            "calories": row["Calories"],
+            "calories": float_or_none(row["Calories"]),
             "commute": row["Commute"] == "true",
             "distance_km": row["Distance"],
             "elapsed_time": datetime.timedelta(seconds=int(row["Elapsed Time"])),

geo_activity_playground-0.21.0/geo_activity_playground/importers/test_directory.py

@@ -0,0 +1,22 @@
+import pathlib
+
+from geo_activity_playground.importers.directory import _get_metadata_from_path
+
+
+def test_get_metadata_from_path() -> None:
+    expected = {
+        "kind": "Radfahrt",
+        "equipment": "Bike 2019",
+        "name": "Foo-Bar to Baz24",
+    }
+    actual = _get_metadata_from_path(
+        pathlib.Path(
+            "Activities/Radfahrt/Bike 2019/Foo-Bar to Baz24/2024-03-03-17-42-10 Something something.fit"
+        ),
+        [
+            r"(?P<kind>[^/]+)/(?P<equipment>[^/]+)/(?P<name>[^/]+)/",
+            r"(?P<kind>[^/]+)/(?P<equipment>[^/]+)/[-\d_ ]+(?P<name>[^/\.]+)(?:\.\w+)+$",
+            r"(?P<kind>[^/]+)/[-\d_ ]+(?P<name>[^/\.]+)(?:\.\w+)+$",
+        ],
+    )
+    assert actual == expected