geo-activity-playground 0.24.2__tar.gz → 0.26.0__tar.gz

{geo_activity_playground-0.24.2 → geo_activity_playground-0.26.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: geo-activity-playground
-Version: 0.24.2
+Version: 0.26.0
 Summary: Analysis of geo data activities like rides, runs or hikes.
 License: MIT
 Author: Martin Ueding

{geo_activity_playground-0.24.2 → geo_activity_playground-0.26.0}/geo_activity_playground/__main__.py

@@ -8,7 +8,6 @@ import coloredlogs
 
 from .importers.strava_checkout import convert_strava_checkout
 from geo_activity_playground.core.activities import ActivityRepository
-from geo_activity_playground.core.cache_migrations import apply_cache_migrations
 from geo_activity_playground.core.config import get_config
 from geo_activity_playground.explorer.tile_visits import TileVisitAccessor
 from geo_activity_playground.explorer.video import explorer_video_main
@@ -97,7 +96,6 @@ def make_activity_repository(
     basedir: pathlib.Path, skip_strava: bool
 ) -> tuple[ActivityRepository, TileVisitAccessor, dict]:
     os.chdir(basedir)
-    apply_cache_migrations()
     config = get_config()
 
     if not config.get("prefer_metadata_from_file", True):

geo_activity_playground-0.26.0/geo_activity_playground/core/activities.py

@@ -0,0 +1,254 @@
+import datetime
+import functools
+import logging
+import pickle
+from typing import Iterator
+from typing import Optional
+from typing import TypedDict
+
+import geojson
+import matplotlib
+import numpy as np
+import pandas as pd
+from tqdm import tqdm
+
+from geo_activity_playground.core.config import get_config
+from geo_activity_playground.core.paths import activities_file
+from geo_activity_playground.core.paths import activity_enriched_meta_dir
+from geo_activity_playground.core.paths import activity_enriched_time_series_dir
+
+logger = logging.getLogger(__name__)
+
+
+class ActivityMeta(TypedDict):
+    calories: float
+    commute: bool
+    consider_for_achievements: bool
+    distance_km: float
+    elapsed_time: datetime.timedelta
+    end_latitude: float
+    end_longitude: float
+    equipment: str
+    id: int
+    kind: str
+    moving_time: datetime.timedelta
+    name: str
+    path: str
+    start_latitude: float
+    start_longitude: float
+    start: datetime.datetime
+    steps: int
+
+
+def make_activity_meta() -> ActivityMeta:
+    return ActivityMeta(
+        calories=None,
+        commute=False,
+        consider_for_achievements=True,
+        equipment="Unknown",
+        kind="Unknown",
+        steps=None,
+    )
+
+
+def build_activity_meta() -> None:
+    if activities_file().exists():
+        meta = pd.read_parquet(activities_file())
+        present_ids = set(meta["id"])
+    else:
+        meta = pd.DataFrame(columns=["id"])
+        present_ids = set()
+
+    available_ids = {
+        int(path.stem) for path in activity_enriched_meta_dir().glob("*.pickle")
+    }
+    new_ids = available_ids - present_ids
+    deleted_ids = present_ids - available_ids
+
+    # Remove updated activities and read these again.
+    if activities_file().exists():
+        meta_mtime = activities_file().stat().st_mtime
+        updated_ids = {
+            int(path.stem)
+            for path in activity_enriched_meta_dir().glob("*.pickle")
+            if path.stat().st_mtime > meta_mtime
+        }
+        new_ids.update(updated_ids)
+        deleted_ids.update(updated_ids & present_ids)
+
+    if deleted_ids:
+        logger.debug(f"Removing activities {deleted_ids} from repository.")
+        meta.drop(sorted(deleted_ids), axis="index", inplace=True)
+
+    rows = []
+    for new_id in tqdm(new_ids, desc="Register new activities"):
+        with open(activity_enriched_meta_dir() / f"{new_id}.pickle", "rb") as f:
+            rows.append(pickle.load(f))
+
+    if rows:
+        new_shard = pd.DataFrame(rows)
+        new_shard.index = new_shard["id"]
+        new_shard.index.name = "index"
+        meta = pd.concat([meta, new_shard])
+
+    if len(meta):
+        assert pd.api.types.is_dtype_equal(meta["start"].dtype, "datetime64[ns]"), (
+            meta["start"].dtype,
+            meta["start"].iloc[0],
+        )
+
+        meta.sort_values("start", inplace=True)
+
+    meta.to_parquet(activities_file())
+
+
+class ActivityRepository:
+    def __init__(self) -> None:
+        self.meta = None
+
+    def __len__(self) -> int:
+        return len(self.meta)
+
+    def reload(self) -> None:
+        self.meta = pd.read_parquet(activities_file())
+
+    def has_activity(self, activity_id: int) -> bool:
+        if len(self.meta):
+            if activity_id in self.meta["id"]:
+                return True
+
+        for activity_meta in self._loose_activities:
+            if activity_meta["id"] == activity_id:
+                return True
+
+        return False
+
+    def last_activity_date(self) -> Optional[datetime.datetime]:
+        if len(self.meta):
+            return self.meta.iloc[-1]["start"]
+        else:
+            return None
+
+    def get_activity_ids(self, only_achievements: bool = False) -> set[int]:
+        if only_achievements:
+            return set(self.meta.loc[self.meta["consider_for_achievements"]].index)
+        else:
+            return set(self.meta.index)
+
+    def iter_activities(self, new_to_old=True, dropna=False) -> Iterator[ActivityMeta]:
+        direction = -1 if new_to_old else 1
+        for index, row in self.meta[::direction].iterrows():
+            if not dropna or not pd.isna(row["start"]):
+                yield row
+
+    @functools.lru_cache()
+    def get_activity_by_id(self, id: int) -> ActivityMeta:
+        activity = self.meta.loc[id]
+        assert isinstance(activity["name"], str), activity["name"]
+        return activity
+
+    @functools.lru_cache(maxsize=3000)
+    def get_time_series(self, id: int) -> pd.DataFrame:
+        path = activity_enriched_time_series_dir() / f"{id}.parquet"
+        try:
+            df = pd.read_parquet(path)
+        except OSError as e:
+            logger.error(f"Error while reading {path}, deleting cache file …")
+            path.unlink(missing_ok=True)
+            raise
+
+        return df
+
+
+def make_geojson_from_time_series(time_series: pd.DataFrame) -> str:
+    fc = geojson.FeatureCollection(
+        features=[
+            geojson.LineString(
+                [(lon, lat) for lat, lon in zip(group["latitude"], group["longitude"])]
+            )
+            for _, group in time_series.groupby("segment_id")
+        ]
+    )
+    return geojson.dumps(fc)
+
+
+def make_geojson_color_line(time_series: pd.DataFrame) -> str:
+    speed_without_na = time_series["speed"].dropna()
+    low = min(speed_without_na)
+    high = max(speed_without_na)
+    clamp_speed = lambda speed: min(max((speed - low) / (high - low), 0.0), 1.0)
+
+    cmap = matplotlib.colormaps["viridis"]
+    features = [
+        geojson.Feature(
+            geometry=geojson.LineString(
+                coordinates=[
+                    [row["longitude"], row["latitude"]],
+                    [next["longitude"], next["latitude"]],
+                ]
+            ),
+            properties={
+                "speed": next["speed"] if np.isfinite(next["speed"]) else 0.0,
+                "color": matplotlib.colors.to_hex(cmap(clamp_speed(next["speed"]))),
+            },
+        )
+        for _, group in time_series.groupby("segment_id")
+        for (_, row), (_, next) in zip(group.iterrows(), group.iloc[1:].iterrows())
+    ]
+    feature_collection = geojson.FeatureCollection(features)
+    return geojson.dumps(feature_collection)
+
+
+def make_speed_color_bar(time_series: pd.DataFrame) -> dict[str, str]:
+    speed_without_na = time_series["speed"].dropna()
+    low = min(speed_without_na)
+    high = max(speed_without_na)
+    cmap = matplotlib.colormaps["viridis"]
+    clamp_speed = lambda speed: min(max((speed - low) / (high - low), 0.0), 1.0)
+    colors = [
+        (f"{speed:.1f}", matplotlib.colors.to_hex(cmap(clamp_speed(speed))))
+        for speed in np.linspace(low, high, 10)
+    ]
+    return {"low": low, "high": high, "colors": colors}
+
+
+def extract_heart_rate_zones(time_series: pd.DataFrame) -> Optional[pd.DataFrame]:
+    if "heartrate" not in time_series:
+        return None
+    config = get_config()
+    try:
+        heart_config = config["heart"]
+    except KeyError:
+        logger.warning(
+            "Missing config entry `heart`, cannot determine heart rate zones."
+        )
+        return None
+
+    birthyear = heart_config.get("birthyear", None)
+    maximum = heart_config.get("maximum", None)
+    resting = heart_config.get("resting", None)
+
+    if not maximum and birthyear:
+        age = time_series["time"].iloc[0].year - birthyear
+        maximum = 220 - age
+    if not resting:
+        resting = 0
+    if not maximum:
+        logger.warning(
+            "Missing config entry `heart.maximum` or `heart.birthyear`, cannot determine heart rate zones."
+        )
+        return None
+
+    zones: pd.Series = (time_series["heartrate"] - resting) * 10 // (
+        maximum - resting
+    ) - 4
+    zones.loc[zones < 0] = 0
+    zones.loc[zones > 5] = 5
+    df = pd.DataFrame({"heartzone": zones, "step": time_series["time"].diff()}).dropna()
+    duration_per_zone = df.groupby("heartzone").sum()["step"].dt.total_seconds() / 60
+    duration_per_zone.name = "minutes"
+    for i in range(6):
+        if i not in duration_per_zone:
+            duration_per_zone.loc[i] = 0.0
+    result = duration_per_zone.reset_index()
+    return result

geo_activity_playground-0.26.0/geo_activity_playground/core/enrichment.py

@@ -0,0 +1,164 @@
+import datetime
+import logging
+import pickle
+from typing import Any
+from typing import Optional
+
+import numpy as np
+import pandas as pd
+from tqdm import tqdm
+
+from geo_activity_playground.core.activities import ActivityMeta
+from geo_activity_playground.core.activities import make_activity_meta
+from geo_activity_playground.core.coordinates import get_distance
+from geo_activity_playground.core.paths import activity_enriched_meta_dir
+from geo_activity_playground.core.paths import activity_enriched_time_series_dir
+from geo_activity_playground.core.paths import activity_extracted_meta_dir
+from geo_activity_playground.core.paths import activity_extracted_time_series_dir
+from geo_activity_playground.core.tiles import compute_tile_float
+from geo_activity_playground.core.time_conversion import convert_to_datetime_ns
+
+logger = logging.getLogger(__name__)
+
+
+def enrich_activities(kind_defaults: dict[dict[str, Any]]) -> None:
+    # Delete removed activities.
+    for enriched_metadata_path in activity_enriched_meta_dir().glob("*.pickle"):
+        if not (activity_extracted_meta_dir() / enriched_metadata_path.name).exists():
+            logger.warning(f"Deleting {enriched_metadata_path}")
+            enriched_metadata_path.unlink()
+    for enriched_time_series_path in activity_enriched_time_series_dir().glob(
+        "*.parquet"
+    ):
+        if not (
+            activity_extracted_time_series_dir() / enriched_time_series_path.name
+        ).exists():
+            logger.warning(f"Deleting {enriched_time_series_path}")
+            enriched_time_series_path.unlink()
+
+    # Get new metadata paths.
+    new_extracted_metadata_paths = []
+    for extracted_metadata_path in activity_extracted_meta_dir().glob("*.pickle"):
+        enriched_metadata_path = (
+            activity_enriched_meta_dir() / extracted_metadata_path.name
+        )
+        if (
+            not enriched_metadata_path.exists()
+            or enriched_metadata_path.stat().st_mtime
+            < extracted_metadata_path.stat().st_mtime
+        ):
+            new_extracted_metadata_paths.append(extracted_metadata_path)
+
+    for extracted_metadata_path in tqdm(
+        new_extracted_metadata_paths, desc="Enrich new activity data"
+    ):
+        # Read extracted data.
+        activity_id = extracted_metadata_path.stem
+        extracted_time_series_path = (
+            activity_extracted_time_series_dir() / f"{activity_id}.parquet"
+        )
+        time_series = pd.read_parquet(extracted_time_series_path)
+        with open(extracted_metadata_path, "rb") as f:
+            extracted_metadata = pickle.load(f)
+
+        metadata = make_activity_meta()
+        metadata.update(extracted_metadata)
+
+        # Enrich time series.
+        metadata.update(kind_defaults.get(metadata["kind"], {}))
+        time_series = _embellish_single_time_series(
+            time_series, metadata.get("start", None)
+        )
+        metadata.update(_get_metadata_from_timeseries(time_series))
+
+        # Write enriched data.
+        enriched_metadata_path = activity_enriched_meta_dir() / f"{activity_id}.pickle"
+        enriched_time_series_path = (
+            activity_enriched_time_series_dir() / f"{activity_id}.parquet"
+        )
+        with open(enriched_metadata_path, "wb") as f:
+            pickle.dump(metadata, f)
+        time_series.to_parquet(enriched_time_series_path)
+
+
+def _get_metadata_from_timeseries(timeseries: pd.DataFrame) -> ActivityMeta:
+    metadata = ActivityMeta()
+
+    # Extract some meta data from the time series.
+    metadata["start"] = timeseries["time"].iloc[0]
+    metadata["elapsed_time"] = timeseries["time"].iloc[-1] - timeseries["time"].iloc[0]
+    metadata["distance_km"] = timeseries["distance_km"].iloc[-1]
+    if "calories" in timeseries.columns:
+        metadata["calories"] = timeseries["calories"].iloc[-1]
+    metadata["moving_time"] = _compute_moving_time(timeseries)
+
+    metadata["start_latitude"] = timeseries["latitude"].iloc[0]
+    metadata["end_latitude"] = timeseries["latitude"].iloc[-1]
+    metadata["start_longitude"] = timeseries["longitude"].iloc[0]
+    metadata["end_longitude"] = timeseries["longitude"].iloc[-1]
+
+    return metadata
+
+
+def _compute_moving_time(time_series: pd.DataFrame) -> datetime.timedelta:
+    def moving_time(group) -> datetime.timedelta:
+        selection = group["speed"] > 1.0
+        time_diff = group["time"].diff().loc[selection]
+        return time_diff.sum()
+
+    return (
+        time_series.groupby("segment_id").apply(moving_time, include_groups=False).sum()
+    )
+
+
+def _embellish_single_time_series(
+    timeseries: pd.DataFrame, start: Optional[datetime.datetime] = None
+) -> pd.DataFrame:
+    if start is not None and pd.api.types.is_dtype_equal(
+        timeseries["time"].dtype, "int64"
+    ):
+        time = timeseries["time"]
+        del timeseries["time"]
+        timeseries["time"] = [
+            convert_to_datetime_ns(start + datetime.timedelta(seconds=t)) for t in time
+        ]
+    timeseries["time"] = convert_to_datetime_ns(timeseries["time"])
+    assert pd.api.types.is_dtype_equal(timeseries["time"].dtype, "datetime64[ns]"), (
+        timeseries["time"].dtype,
+        timeseries["time"].iloc[0],
+    )
+
+    distances = get_distance(
+        timeseries["latitude"].shift(1),
+        timeseries["longitude"].shift(1),
+        timeseries["latitude"],
+        timeseries["longitude"],
+    ).fillna(0.0)
+    time_diff_threshold_seconds = 30
+    time_diff = (timeseries["time"] - timeseries["time"].shift(1)).dt.total_seconds()
+    jump_indices = time_diff >= time_diff_threshold_seconds
+    distances.loc[jump_indices] = 0.0
+
+    if "distance_km" not in timeseries.columns:
+        timeseries["distance_km"] = pd.Series(np.cumsum(distances)) / 1000
+
+    if "speed" not in timeseries.columns:
+        timeseries["speed"] = (
+            timeseries["distance_km"].diff()
+            / (timeseries["time"].diff().dt.total_seconds() + 1e-3)
+            * 3600
+        )
+
+    potential_jumps = (timeseries["speed"] > 40) & (timeseries["speed"].diff() > 10)
+    if np.any(potential_jumps):
+        timeseries = timeseries.loc[~potential_jumps].copy()
+
+    if "segment_id" not in timeseries.columns:
+        timeseries["segment_id"] = np.cumsum(jump_indices)
+
+    if "x" not in timeseries.columns:
+        x, y = compute_tile_float(timeseries["latitude"], timeseries["longitude"], 0)
+        timeseries["x"] = x
+        timeseries["y"] = y
+
+    return timeseries

geo_activity_playground-0.26.0/geo_activity_playground/core/paths.py

@@ -0,0 +1,56 @@
+"""
+Paths within the playground and cache.
+"""
+import functools
+import pathlib
+import typing
+
+
+def dir_wrapper(path: pathlib.Path) -> typing.Callable[[], pathlib.Path]:
+    @functools.cache
+    def wrapper() -> pathlib.Path:
+        path.mkdir(exist_ok=True, parents=True)
+        return path
+
+    return wrapper
+
+
+def file_wrapper(path: pathlib.Path) -> typing.Callable[[], pathlib.Path]:
+    @functools.cache
+    def wrapper() -> pathlib.Path:
+        path.parent.mkdir(exist_ok=True, parents=True)
+        return path
+
+    return wrapper
+
+
+_cache_dir = pathlib.Path("Cache")
+
+_activity_dir = _cache_dir / "Activity"
+_activity_extracted_dir = _activity_dir / "Extracted"
+_activity_extracted_meta_dir = _activity_extracted_dir / "Meta"
+_activity_extracted_time_series_dir = _activity_extracted_dir / "Time Series"
+
+_activity_enriched_dir = _activity_dir / "Enriched"
+_activity_enriched_meta_dir = _activity_enriched_dir / "Meta"
+_activity_enriched_time_series_dir = _activity_enriched_dir / "Time Series"
+_activities_file = _activity_dir / "activities.parquet"
+
+_tiles_per_time_series = _cache_dir / "Tiles" / "Tiles Per Time Series"
+
+_strava_api_dir = pathlib.Path("Strava API")
+_strava_dynamic_config_path = _strava_api_dir / "strava-client-id.json"
+
+
+cache_dir = dir_wrapper(_cache_dir)
+
+activity_extracted_dir = dir_wrapper(_activity_extracted_dir)
+activity_extracted_meta_dir = dir_wrapper(_activity_extracted_meta_dir)
+activity_extracted_time_series_dir = dir_wrapper(_activity_extracted_time_series_dir)
+activity_enriched_meta_dir = dir_wrapper(_activity_enriched_meta_dir)
+activity_enriched_time_series_dir = dir_wrapper(_activity_enriched_time_series_dir)
+tiles_per_time_series = dir_wrapper(_tiles_per_time_series)
+strava_api_dir = dir_wrapper(_strava_api_dir)
+
+activities_file = file_wrapper(_activities_file)
+strava_dynamic_config_path = file_wrapper(_strava_dynamic_config_path)

{geo_activity_playground-0.24.2 → geo_activity_playground-0.26.0}/geo_activity_playground/core/tasks.py

@@ -46,12 +46,12 @@ def work_tracker(path: pathlib.Path):
     yield s
 
     with open(path, "w") as f:
-        json.dump(list(s), f)
+        json.dump(list(s), f, indent=2, sort_keys=True)
 
 
 class WorkTracker:
-    def __init__(self, name: str) -> None:
-        self._path = work_tracker_path(name)
+    def __init__(self, path: pathlib.Path) -> None:
+        self._path = path
 
         if self._path.exists():
             with open(self._path, "rb") as f:
@@ -59,12 +59,15 @@ class WorkTracker:
         else:
             self._done = set()
 
-    def filter(self, ids: Iterable[int]) -> set[int]:
+    def filter(self, ids: Iterable) -> set:
         return set(ids) - self._done
 
     def mark_done(self, id: int) -> None:
         self._done.add(id)
 
+    def discard(self, id) -> None:
+        self._done.discard(id)
+
     def close(self) -> None:
         with open(self._path, "wb") as f:
             pickle.dump(self._done, f)
@@ -77,3 +80,23 @@ def try_load_pickle(path: pathlib.Path) -> Any:
                 return pickle.load(f)
         except ModuleNotFoundError:
             pass
+
+
+class TransformVersion:
+    def __init__(self, path: pathlib.Path, code_version: int) -> None:
+        self._path = path
+        self._code_version = code_version
+
+        with open(path) as f:
+            self._actual_version = json.load(f)
+
+        assert (
+            self._actual_version <= self._code_version
+        ), "You attempt to use a more modern playground with an older code version, that is not supported."
+
+    def outdated(self) -> bool:
+        return self._actual_version < self._code_version
+
+    def write(self) -> None:
+        with open(self._path, "w") as f:
+            json.dump(self._code_version, f)