geo-activity-playground 0.24.1__py3-none-any.whl → 0.25.0__py3-none-any.whl

geo_activity_playground/__main__.py

@@ -8,7 +8,6 @@ import coloredlogs
 
 from .importers.strava_checkout import convert_strava_checkout
 from geo_activity_playground.core.activities import ActivityRepository
-from geo_activity_playground.core.cache_migrations import apply_cache_migrations
 from geo_activity_playground.core.config import get_config
 from geo_activity_playground.explorer.tile_visits import TileVisitAccessor
 from geo_activity_playground.explorer.video import explorer_video_main
@@ -97,7 +96,6 @@ def make_activity_repository(
     basedir: pathlib.Path, skip_strava: bool
 ) -> tuple[ActivityRepository, TileVisitAccessor, dict]:
     os.chdir(basedir)
-    apply_cache_migrations()
     config = get_config()
 
     if not config.get("prefer_metadata_from_file", True):

geo_activity_playground/core/activities.py

@@ -1,7 +1,7 @@
 import datetime
 import functools
 import logging
-import pathlib
+import pickle
 from typing import Iterator
 from typing import Optional
 from typing import TypedDict
@@ -13,12 +13,9 @@ import pandas as pd
 from tqdm import tqdm
 
 from geo_activity_playground.core.config import get_config
-from geo_activity_playground.core.coordinates import get_distance
-from geo_activity_playground.core.paths import activities_path
-from geo_activity_playground.core.paths import activity_timeseries_path
-from geo_activity_playground.core.tasks import WorkTracker
-from geo_activity_playground.core.tiles import compute_tile_float
-from geo_activity_playground.core.time_conversion import convert_to_datetime_ns
+from geo_activity_playground.core.paths import activities_file
+from geo_activity_playground.core.paths import activity_enriched_meta_dir
+from geo_activity_playground.core.paths import activity_enriched_time_series_dir
 
 logger = logging.getLogger(__name__)
 
@@ -34,6 +31,7 @@ class ActivityMeta(TypedDict):
     equipment: str
     id: int
     kind: str
+    moving_time: datetime.timedelta
     name: str
     path: str
     start_latitude: float
@@ -42,69 +40,77 @@ class ActivityMeta(TypedDict):
     steps: int
 
 
+def make_activity_meta() -> ActivityMeta:
+    return ActivityMeta(
+        calories=None,
+        commute=False,
+        consider_for_achievements=True,
+        equipment="Unknown",
+        kind="Unknown",
+        steps=None,
+    )
+
+
+def build_activity_meta() -> None:
+    if activities_file().exists():
+        meta = pd.read_parquet(activities_file())
+        present_ids = set(meta["id"])
+    else:
+        meta = pd.DataFrame(columns=["id"])
+        present_ids = set()
+
+    available_ids = {
+        int(path.stem) for path in activity_enriched_meta_dir().glob("*.pickle")
+    }
+    new_ids = available_ids - present_ids
+    deleted_ids = present_ids - available_ids
+
+    # Remove updated activities and read these again.
+    if activities_file().exists():
+        meta_mtime = activities_file().stat().st_mtime
+        updated_ids = {
+            int(path.stem)
+            for path in activity_enriched_meta_dir().glob("*.pickle")
+            if path.stat().st_mtime > meta_mtime
+        }
+        new_ids.update(updated_ids)
+        deleted_ids.update(updated_ids & present_ids)
+
+    if deleted_ids:
+        logger.debug(f"Removing activities {deleted_ids} from repository.")
+        meta.drop(sorted(deleted_ids), axis="index", inplace=True)
+
+    rows = []
+    for new_id in tqdm(new_ids, desc="Register new activities"):
+        with open(activity_enriched_meta_dir() / f"{new_id}.pickle", "rb") as f:
+            rows.append(pickle.load(f))
+
+    if rows:
+        new_shard = pd.DataFrame(rows)
+        new_shard.index = new_shard["id"]
+        new_shard.index.name = "index"
+        meta = pd.concat([meta, new_shard])
+
+    if len(meta):
+        assert pd.api.types.is_dtype_equal(meta["start"].dtype, "datetime64[ns]"), (
+            meta["start"].dtype,
+            meta["start"].iloc[0],
+        )
+
+        meta.sort_values("start", inplace=True)
+
+    meta.to_parquet(activities_file())
+
+
 class ActivityRepository:
     def __init__(self) -> None:
-        if activities_path().exists():
-            self.meta = pd.read_parquet(activities_path())
-            self.meta.index = self.meta["id"]
-            self.meta.index.name = "index"
-            if not pd.api.types.is_dtype_equal(
-                self.meta["start"].dtype, "datetime64[ns]"
-            ):
-                self.meta["start"] = convert_to_datetime_ns(self.meta["start"])
-        else:
-            self.meta = pd.DataFrame()
-
-        self._loose_activities: list[ActivityMeta] = []
-        self._loose_activity_ids: set[int] = set()
+        self.meta = None
 
     def __len__(self) -> int:
         return len(self.meta)
 
-    def add_activity(self, activity_meta: ActivityMeta) -> None:
-        _extend_metadata_from_timeseries(activity_meta)
-        if activity_meta["id"] in self._loose_activity_ids:
-            logger.error(f"Activity with the same file already exists. New activity:")
-            print(activity_meta)
-            print("Existing activity:")
-            print(
-                [
-                    activity
-                    for activity in self._loose_activities
-                    if activity["id"] == activity_meta["id"]
-                ]
-            )
-            raise ValueError("Activity with the same file already exists.")
-        self._loose_activities.append(activity_meta)
-        self._loose_activity_ids.add(activity_meta["id"])
-
-    def commit(self) -> None:
-        if self._loose_activities:
-            logger.debug(
-                f"Adding {len(self._loose_activities)} activities to the repository …"
-            )
-            new_df = pd.DataFrame(self._loose_activities)
-            if len(self.meta):
-                new_ids_set = set(new_df["id"])
-                is_kept = [
-                    activity_id not in new_ids_set for activity_id in self.meta["id"]
-                ]
-                old_df = self.meta.loc[is_kept]
-            else:
-                old_df = self.meta
-
-            self.meta = pd.concat([old_df, new_df])
-            assert pd.api.types.is_dtype_equal(
-                self.meta["start"].dtype, "datetime64[ns]"
-            ), (self.meta["start"].dtype, self.meta["start"].iloc[0])
-            self.save()
-            self._loose_activities = []
-
-    def save(self) -> None:
-        self.meta.index = self.meta["id"]
-        self.meta.index.name = "index"
-        self.meta.sort_values("start", inplace=True)
-        self.meta.to_parquet(activities_path())
+    def reload(self) -> None:
+        self.meta = pd.read_parquet(activities_file())
 
     def has_activity(self, activity_id: int) -> bool:
         if len(self.meta):
@@ -143,7 +149,7 @@ class ActivityRepository:
 
     @functools.lru_cache(maxsize=3000)
     def get_time_series(self, id: int) -> pd.DataFrame:
-        path = activity_timeseries_path(id)
+        path = activity_enriched_time_series_dir() / f"{id}.parquet"
         try:
             df = pd.read_parquet(path)
         except OSError as e:
@@ -154,79 +160,6 @@ class ActivityRepository:
         return df
 
 
-def embellish_time_series(repository: ActivityRepository) -> None:
-    work_tracker = WorkTracker("embellish-time-series")
-    activities_to_process = work_tracker.filter(repository.get_activity_ids())
-    for activity_id in tqdm(activities_to_process, desc="Embellish time series data"):
-        path = activity_timeseries_path(activity_id)
-        df = pd.read_parquet(path)
-        df.name = id
-        df, changed = embellish_single_time_series(
-            df, repository.get_activity_by_id(activity_id)["start"]
-        )
-        if changed:
-            df.to_parquet(path)
-        work_tracker.mark_done(activity_id)
-    work_tracker.close()
-
-
-def embellish_single_time_series(
-    timeseries: pd.DataFrame, start: Optional[datetime.datetime] = None
-) -> bool:
-    changed = False
-
-    if start is not None and pd.api.types.is_dtype_equal(
-        timeseries["time"].dtype, "int64"
-    ):
-        time = timeseries["time"]
-        del timeseries["time"]
-        timeseries["time"] = [
-            convert_to_datetime_ns(start + datetime.timedelta(seconds=t)) for t in time
-        ]
-        changed = True
-    assert pd.api.types.is_dtype_equal(timeseries["time"].dtype, "datetime64[ns]")
-
-    distances = get_distance(
-        timeseries["latitude"].shift(1),
-        timeseries["longitude"].shift(1),
-        timeseries["latitude"],
-        timeseries["longitude"],
-    ).fillna(0.0)
-    time_diff_threshold_seconds = 30
-    time_diff = (timeseries["time"] - timeseries["time"].shift(1)).dt.total_seconds()
-    jump_indices = (time_diff >= time_diff_threshold_seconds) & (distances > 100)
-    distances.loc[jump_indices] = 0.0
-
-    if not "distance_km" in timeseries.columns:
-        timeseries["distance_km"] = pd.Series(np.cumsum(distances)) / 1000
-        changed = True
-
-    if "speed" not in timeseries.columns:
-        timeseries["speed"] = (
-            timeseries["distance_km"].diff()
-            / (timeseries["time"].diff().dt.total_seconds() + 1e-3)
-            * 3600
-        )
-        changed = True
-
-    potential_jumps = (timeseries["speed"] > 40) & (timeseries["speed"].diff() > 10)
-    if np.any(potential_jumps):
-        timeseries = timeseries.loc[~potential_jumps].copy()
-        changed = True
-
-    if "segment_id" not in timeseries.columns:
-        timeseries["segment_id"] = np.cumsum(jump_indices)
-        changed = True
-
-    if "x" not in timeseries.columns:
-        x, y = compute_tile_float(timeseries["latitude"], timeseries["longitude"], 0)
-        timeseries["x"] = x
-        timeseries["y"] = y
-        changed = True
-
-    return timeseries, changed
-
-
 def make_geojson_from_time_series(time_series: pd.DataFrame) -> str:
     fc = geojson.FeatureCollection(
         features=[
@@ -319,14 +252,3 @@ def extract_heart_rate_zones(time_series: pd.DataFrame) -> Optional[pd.DataFrame
             duration_per_zone.loc[i] = 0.0
     result = duration_per_zone.reset_index()
     return result
-
-
-def _extend_metadata_from_timeseries(metadata: ActivityMeta) -> None:
-    timeseries = pd.read_parquet(
-        pathlib.Path("Cache/Activity Timeseries") / f"{metadata['id']}.parquet"
-    )
-
-    metadata["start_latitude"] = timeseries["latitude"].iloc[0]
-    metadata["end_latitude"] = timeseries["latitude"].iloc[-1]
-    metadata["start_longitude"] = timeseries["longitude"].iloc[0]
-    metadata["end_longitude"] = timeseries["longitude"].iloc[-1]

geo_activity_playground/core/enrichment.py

@@ -0,0 +1,164 @@
+import datetime
+import logging
+import pickle
+from typing import Any
+from typing import Optional
+
+import numpy as np
+import pandas as pd
+from tqdm import tqdm
+
+from geo_activity_playground.core.activities import ActivityMeta
+from geo_activity_playground.core.activities import make_activity_meta
+from geo_activity_playground.core.coordinates import get_distance
+from geo_activity_playground.core.paths import activity_enriched_meta_dir
+from geo_activity_playground.core.paths import activity_enriched_time_series_dir
+from geo_activity_playground.core.paths import activity_extracted_meta_dir
+from geo_activity_playground.core.paths import activity_extracted_time_series_dir
+from geo_activity_playground.core.tiles import compute_tile_float
+from geo_activity_playground.core.time_conversion import convert_to_datetime_ns
+
+logger = logging.getLogger(__name__)
+
+
+def enrich_activities(kind_defaults: dict[dict[str, Any]]) -> None:
+    # Delete removed activities.
+    for enriched_metadata_path in activity_enriched_meta_dir().glob("*.pickle"):
+        if not (activity_extracted_meta_dir() / enriched_metadata_path.name).exists():
+            logger.warning(f"Deleting {enriched_metadata_path}")
+            enriched_metadata_path.unlink()
+    for enriched_time_series_path in activity_enriched_time_series_dir().glob(
+        "*.parquet"
+    ):
+        if not (
+            activity_extracted_time_series_dir() / enriched_time_series_path.name
+        ).exists():
+            logger.warning(f"Deleting {enriched_time_series_path}")
+            enriched_time_series_path.unlink()
+
+    # Get new metadata paths.
+    new_extracted_metadata_paths = []
+    for extracted_metadata_path in activity_extracted_meta_dir().glob("*.pickle"):
+        enriched_metadata_path = (
+            activity_enriched_meta_dir() / extracted_metadata_path.name
+        )
+        if (
+            not enriched_metadata_path.exists()
+            or enriched_metadata_path.stat().st_mtime
+            < extracted_metadata_path.stat().st_mtime
+        ):
+            new_extracted_metadata_paths.append(extracted_metadata_path)
+
+    for extracted_metadata_path in tqdm(
+        new_extracted_metadata_paths, desc="Enrich new activity data"
+    ):
+        # Read extracted data.
+        activity_id = extracted_metadata_path.stem
+        extracted_time_series_path = (
+            activity_extracted_time_series_dir() / f"{activity_id}.parquet"
+        )
+        time_series = pd.read_parquet(extracted_time_series_path)
+        with open(extracted_metadata_path, "rb") as f:
+            extracted_metadata = pickle.load(f)
+
+        metadata = make_activity_meta()
+        metadata.update(extracted_metadata)
+
+        # Enrich time series.
+        metadata.update(kind_defaults.get(metadata["kind"], {}))
+        time_series = _embellish_single_time_series(
+            time_series, metadata.get("start", None)
+        )
+        metadata.update(_get_metadata_from_timeseries(time_series))
+
+        # Write enriched data.
+        enriched_metadata_path = activity_enriched_meta_dir() / f"{activity_id}.pickle"
+        enriched_time_series_path = (
+            activity_enriched_time_series_dir() / f"{activity_id}.parquet"
+        )
+        with open(enriched_metadata_path, "wb") as f:
+            pickle.dump(metadata, f)
+        time_series.to_parquet(enriched_time_series_path)
+
+
+def _get_metadata_from_timeseries(timeseries: pd.DataFrame) -> ActivityMeta:
+    metadata = ActivityMeta()
+
+    # Extract some meta data from the time series.
+    metadata["start"] = timeseries["time"].iloc[0]
+    metadata["elapsed_time"] = timeseries["time"].iloc[-1] - timeseries["time"].iloc[0]
+    metadata["distance_km"] = timeseries["distance_km"].iloc[-1]
+    if "calories" in timeseries.columns:
+        metadata["calories"] = timeseries["calories"].iloc[-1]
+    metadata["moving_time"] = _compute_moving_time(timeseries)
+
+    metadata["start_latitude"] = timeseries["latitude"].iloc[0]
+    metadata["end_latitude"] = timeseries["latitude"].iloc[-1]
+    metadata["start_longitude"] = timeseries["longitude"].iloc[0]
+    metadata["end_longitude"] = timeseries["longitude"].iloc[-1]
+
+    return metadata
+
+
+def _compute_moving_time(time_series: pd.DataFrame) -> datetime.timedelta:
+    def moving_time(group) -> datetime.timedelta:
+        selection = group["speed"] > 1.0
+        time_diff = group["time"].diff().loc[selection]
+        return time_diff.sum()
+
+    return (
+        time_series.groupby("segment_id").apply(moving_time, include_groups=False).sum()
+    )
+
+
+def _embellish_single_time_series(
+    timeseries: pd.DataFrame, start: Optional[datetime.datetime] = None
+) -> pd.DataFrame:
+    if start is not None and pd.api.types.is_dtype_equal(
+        timeseries["time"].dtype, "int64"
+    ):
+        time = timeseries["time"]
+        del timeseries["time"]
+        timeseries["time"] = [
+            convert_to_datetime_ns(start + datetime.timedelta(seconds=t)) for t in time
+        ]
+    timeseries["time"] = convert_to_datetime_ns(timeseries["time"])
+    assert pd.api.types.is_dtype_equal(timeseries["time"].dtype, "datetime64[ns]"), (
+        timeseries["time"].dtype,
+        timeseries["time"].iloc[0],
+    )
+
+    distances = get_distance(
+        timeseries["latitude"].shift(1),
+        timeseries["longitude"].shift(1),
+        timeseries["latitude"],
+        timeseries["longitude"],
+    ).fillna(0.0)
+    time_diff_threshold_seconds = 30
+    time_diff = (timeseries["time"] - timeseries["time"].shift(1)).dt.total_seconds()
+    jump_indices = time_diff >= time_diff_threshold_seconds
+    distances.loc[jump_indices] = 0.0
+
+    if "distance_km" not in timeseries.columns:
+        timeseries["distance_km"] = pd.Series(np.cumsum(distances)) / 1000
+
+    if "speed" not in timeseries.columns:
+        timeseries["speed"] = (
+            timeseries["distance_km"].diff()
+            / (timeseries["time"].diff().dt.total_seconds() + 1e-3)
+            * 3600
+        )
+
+    potential_jumps = (timeseries["speed"] > 40) & (timeseries["speed"].diff() > 10)
+    if np.any(potential_jumps):
+        timeseries = timeseries.loc[~potential_jumps].copy()
+
+    if "segment_id" not in timeseries.columns:
+        timeseries["segment_id"] = np.cumsum(jump_indices)
+
+    if "x" not in timeseries.columns:
+        x, y = compute_tile_float(timeseries["latitude"], timeseries["longitude"], 0)
+        timeseries["x"] = x
+        timeseries["y"] = y
+
+    return timeseries

geo_activity_playground/core/paths.py

@@ -6,32 +6,51 @@ import pathlib
 import typing
 
 
-def dir_wrapper(
-    dir_func: typing.Callable[[], pathlib.Path]
-) -> typing.Callable[[], pathlib.Path]:
-    @functools.wraps(dir_func)
+def dir_wrapper(path: pathlib.Path) -> typing.Callable[[], pathlib.Path]:
     @functools.cache
     def wrapper() -> pathlib.Path:
-        path = dir_func()
         path.mkdir(exist_ok=True, parents=True)
         return path
 
     return wrapper
 
 
-@dir_wrapper
-def cache_dir() -> pathlib.Path:
-    return pathlib.Path("Cache")
+def file_wrapper(path: pathlib.Path) -> typing.Callable[[], pathlib.Path]:
+    @functools.cache
+    def wrapper() -> pathlib.Path:
+        path.parent.mkdir(exist_ok=True, parents=True)
+        return path
+
+    return wrapper
+
+
+_cache_dir = pathlib.Path("Cache")
+
+_activity_dir = _cache_dir / "Activity"
+_activity_extracted_dir = _activity_dir / "Extracted"
+_activity_extracted_meta_dir = _activity_extracted_dir / "Meta"
+_activity_extracted_time_series_dir = _activity_extracted_dir / "Time Series"
+
+_activity_enriched_dir = _activity_dir / "Enriched"
+_activity_enriched_meta_dir = _activity_enriched_dir / "Meta"
+_activity_enriched_time_series_dir = _activity_enriched_dir / "Time Series"
+_activities_file = _activity_dir / "activities.parquet"
 
+_tiles_per_time_series = _cache_dir / "Tiles" / "Tiles Per Time Series"
 
-@dir_wrapper
-def activity_timeseries_dir() -> pathlib.Path:
-    return cache_dir() / "Activity Timeseries"
+_strava_api_dir = pathlib.Path("Strava API")
+_strava_dynamic_config_path = _strava_api_dir / "strava-client-id.json"
 
 
-def activities_path() -> pathlib.Path:
-    return cache_dir() / "activities.parquet"
+cache_dir = dir_wrapper(_cache_dir)
 
+activity_extracted_dir = dir_wrapper(_activity_extracted_dir)
+activity_extracted_meta_dir = dir_wrapper(_activity_extracted_meta_dir)
+activity_extracted_time_series_dir = dir_wrapper(_activity_extracted_time_series_dir)
+activity_enriched_meta_dir = dir_wrapper(_activity_enriched_meta_dir)
+activity_enriched_time_series_dir = dir_wrapper(_activity_enriched_time_series_dir)
+tiles_per_time_series = dir_wrapper(_tiles_per_time_series)
+strava_api_dir = dir_wrapper(_strava_api_dir)
 
-def activity_timeseries_path(activity_id: int) -> pathlib.Path:
-    return activity_timeseries_dir() / f"{activity_id}.parquet"
+activities_file = file_wrapper(_activities_file)
+strava_dynamic_config_path = file_wrapper(_strava_dynamic_config_path)

geo_activity_playground/core/tasks.py

@@ -50,8 +50,8 @@ def work_tracker(path: pathlib.Path):
 
 
 class WorkTracker:
-    def __init__(self, name: str) -> None:
-        self._path = work_tracker_path(name)
+    def __init__(self, path: pathlib.Path) -> None:
+        self._path = path
 
         if self._path.exists():
             with open(self._path, "rb") as f:
@@ -59,12 +59,15 @@ class WorkTracker:
         else:
             self._done = set()
 
-    def filter(self, ids: Iterable[int]) -> set[int]:
+    def filter(self, ids: Iterable) -> set:
         return set(ids) - self._done
 
     def mark_done(self, id: int) -> None:
         self._done.add(id)
 
+    def discard(self, id) -> None:
+        self._done.discard(id)
+
     def close(self) -> None:
         with open(self._path, "wb") as f:
             pickle.dump(self._done, f)
@@ -77,3 +80,23 @@ def try_load_pickle(path: pathlib.Path) -> Any:
                 return pickle.load(f)
         except ModuleNotFoundError:
             pass
+
+
+class TransformVersion:
+    def __init__(self, path: pathlib.Path, code_version: int) -> None:
+        self._path = path
+        self._code_version = code_version
+
+        with open(path) as f:
+            self._actual_version = json.load(f)
+
+        assert (
+            self._actual_version <= self._code_version
+        ), "You attempt to use a more modern playground with an older code version, that is not supported."
+
+    def outdated(self) -> bool:
+        return self._actual_version < self._code_version
+
+    def write(self) -> None:
+        with open(self._path, "w") as f:
+            json.dump(self._code_version, f)