geo-activity-playground 0.26.3__py3-none-any.whl → 0.27.0__py3-none-any.whl

geo_activity_playground/__main__.py

@@ -2,16 +2,17 @@ import argparse
 import logging
 import os
 import pathlib
-import sys
 
 import coloredlogs
 
 from .importers.strava_checkout import convert_strava_checkout
 from geo_activity_playground.core.activities import ActivityRepository
-from geo_activity_playground.core.config import get_config
+from geo_activity_playground.core.config import ConfigAccessor
+from geo_activity_playground.core.config import import_old_config
+from geo_activity_playground.core.config import import_old_strava_config
 from geo_activity_playground.explorer.tile_visits import TileVisitAccessor
 from geo_activity_playground.explorer.video import explorer_video_main
-from geo_activity_playground.webui.app import webui_main
+from geo_activity_playground.webui.app import web_ui_main
 from geo_activity_playground.webui.upload.controller import scan_for_activities
 
 logger = logging.getLogger(__name__)
@@ -62,8 +63,8 @@ def main() -> None:
 
     subparser = subparsers.add_parser("serve", help="Launch webserver")
     subparser.set_defaults(
-        func=lambda options: webui_main(
-            *make_activity_repository(options.basedir, options.skip_strava),
+        func=lambda options: web_ui_main(
+            *make_activity_repository(options.basedir, options.skip_reload),
             host=options.host,
             port=options.port,
         )
@@ -74,12 +75,10 @@ def main() -> None:
     subparser.add_argument(
         "--port", default=5000, type=int, help="the port to run listen on"
     )
-    subparser.add_argument("--skip-strava", action=argparse.BooleanOptionalAction)
+    subparser.add_argument("--skip-reload", action=argparse.BooleanOptionalAction)
 
     subparser = subparsers.add_parser("cache", help="Cache stuff")
-    subparser.set_defaults(
-        func=lambda options: make_activity_repository(options.basedir, False)
-    )
+    subparser.set_defaults(func=lambda options: main_cache(options.basedir))
 
     options = parser.parse_args()
     coloredlogs.install(
@@ -93,23 +92,27 @@ def main() -> None:
 
 
 def make_activity_repository(
-    basedir: pathlib.Path, skip_strava: bool
-) -> tuple[ActivityRepository, TileVisitAccessor, dict]:
+    basedir: pathlib.Path, skip_reload: bool
+) -> tuple[ActivityRepository, TileVisitAccessor, ConfigAccessor]:
     os.chdir(basedir)
-    config = get_config()
-
-    if not config.get("prefer_metadata_from_file", True):
-        logger.error(
-            "The config option `prefer_metadata_from_file` is deprecated. If you want to prefer extract metadata from the activity file paths, please use the new `metadata_extraction_regexes` as explained at https://martin-ueding.github.io/geo-activity-playground/getting-started/using-activity-files/#directory-structure."
-        )
-        sys.exit(1)
 
     repository = ActivityRepository()
     tile_visit_accessor = TileVisitAccessor()
+    config_accessor = ConfigAccessor()
+    import_old_config(config_accessor)
+    import_old_strava_config(config_accessor)
 
-    scan_for_activities(repository, tile_visit_accessor, config, skip_strava)
+    if not skip_reload:
+        scan_for_activities(repository, tile_visit_accessor, config_accessor())
 
-    return repository, tile_visit_accessor, config
+    return repository, tile_visit_accessor, config_accessor
+
+
+def main_cache(basedir: pathlib.Path) -> None:
+    repository, tile_visit_accessor, config_accessor = make_activity_repository(
+        basedir, False
+    )
+    scan_for_activities(repository, tile_visit_accessor, config_accessor())
 
 
 if __name__ == "__main__":

geo_activity_playground/core/activities.py

@@ -12,7 +12,6 @@ import numpy as np
 import pandas as pd
 from tqdm import tqdm
 
-from geo_activity_playground.core.config import get_config
 from geo_activity_playground.core.paths import activities_file
 from geo_activity_playground.core.paths import activity_enriched_meta_dir
 from geo_activity_playground.core.paths import activity_enriched_time_series_dir
@@ -36,7 +35,7 @@ class ActivityMeta(TypedDict):
     path: str
     start_latitude: float
     start_longitude: float
-    start: datetime.datetime
+    start: np.datetime64
     steps: int
 
 
@@ -210,45 +209,3 @@ def make_speed_color_bar(time_series: pd.DataFrame) -> dict[str, str]:
         for speed in np.linspace(low, high, 10)
     ]
     return {"low": low, "high": high, "colors": colors}
-
-
-def extract_heart_rate_zones(time_series: pd.DataFrame) -> Optional[pd.DataFrame]:
-    if "heartrate" not in time_series:
-        return None
-    config = get_config()
-    try:
-        heart_config = config["heart"]
-    except KeyError:
-        logger.warning(
-            "Missing config entry `heart`, cannot determine heart rate zones."
-        )
-        return None
-
-    birthyear = heart_config.get("birthyear", None)
-    maximum = heart_config.get("maximum", None)
-    resting = heart_config.get("resting", None)
-
-    if not maximum and birthyear:
-        age = time_series["time"].iloc[0].year - birthyear
-        maximum = 220 - age
-    if not resting:
-        resting = 0
-    if not maximum:
-        logger.warning(
-            "Missing config entry `heart.maximum` or `heart.birthyear`, cannot determine heart rate zones."
-        )
-        return None
-
-    zones: pd.Series = (time_series["heartrate"] - resting) * 10 // (
-        maximum - resting
-    ) - 4
-    zones.loc[zones < 0] = 0
-    zones.loc[zones > 5] = 5
-    df = pd.DataFrame({"heartzone": zones, "step": time_series["time"].diff()}).dropna()
-    duration_per_zone = df.groupby("heartzone").sum()["step"].dt.total_seconds() / 60
-    duration_per_zone.name = "minutes"
-    for i in range(6):
-        if i not in duration_per_zone:
-            duration_per_zone.loc[i] = 0.0
-    result = duration_per_zone.reset_index()
-    return result

geo_activity_playground/core/config.py

@@ -1,6 +1,12 @@
+import dataclasses
 import functools
+import json
 import logging
 import pathlib
+from typing import Optional
+
+from geo_activity_playground.core.paths import new_config_file
+from geo_activity_playground.core.paths import strava_dynamic_config_path
 
 
 try:
@@ -12,6 +18,49 @@ except ModuleNotFoundError:
 logger = logging.getLogger(__name__)
 
 
+@dataclasses.dataclass
+class Config:
+    birth_year: Optional[int] = None
+    equipment_offsets: dict[str, float] = dataclasses.field(default_factory=dict)
+    explorer_zoom_levels: list[int] = dataclasses.field(
+        default_factory=lambda: [14, 17]
+    )
+    heart_rate_resting: int = 0
+    heart_rate_maximum: Optional[int] = None
+    kinds_without_achievements: list[str] = dataclasses.field(default_factory=list)
+    metadata_extraction_regexes: list[str] = dataclasses.field(default_factory=list)
+    num_processes: Optional[int] = None
+    privacy_zones: dict[str, list[list[float]]] = dataclasses.field(
+        default_factory=dict
+    )
+    strava_client_id: int = 131693
+    strava_client_secret: str = "0ccc0100a2c218512a7ef0cea3b0e322fb4b4365"
+    strava_client_code: Optional[str] = None
+    upload_password: Optional[str] = None
+
+
+class ConfigAccessor:
+    def __init__(self) -> None:
+        if new_config_file().exists():
+            with open(new_config_file()) as f:
+                self._config = Config(**json.load(f))
+        else:
+            self._config = Config()
+
+    def __call__(self) -> Config:
+        return self._config
+
+    def save(self) -> None:
+        with open(new_config_file(), "w") as f:
+            json.dump(
+                dataclasses.asdict(self._config),
+                f,
+                ensure_ascii=False,
+                indent=2,
+                sort_keys=True,
+            )
+
+
 @functools.cache
 def get_config() -> dict:
     config_path = pathlib.Path("config.toml")
@@ -22,3 +71,65 @@ def get_config() -> dict:
         config = tomllib.load(f)
 
     return config
+
+
+def import_old_config(config_accessor: ConfigAccessor) -> None:
+    old_config_path = pathlib.Path("config.toml")
+    if not old_config_path.exists():
+        return
+
+    if new_config_file().exists():
+        logger.warning(
+            "You have an old 'config.toml' which is now superseded by the 'config.json'. You can check the contents of the new 'config.json' and then delete the old 'config.toml'."
+        )
+        return
+
+    old_config = get_config()
+    config = config_accessor()
+
+    if "metadata_extraction_regexes" in old_config:
+        config.metadata_extraction_regexes = old_config["metadata_extraction_regexes"]
+
+    if "heart" in old_config:
+        if "birthyear" in old_config["heart"]:
+            config.birth_year = old_config["heart"]["birthyear"]
+        if "resting" in old_config["heart"]:
+            config.heart_rate_resting = old_config["heart"]["resting"]
+        if "maximum" in old_config["heart"]:
+            config.heart_rate_maximum = old_config["heart"]["maximum"]
+
+    if "strava" in old_config:
+        if "client_id" in old_config["strava"]:
+            config.strava_client_id = old_config["strava"]["client_id"]
+        if "client_secret" in old_config["strava"]:
+            config.strava_client_secret = old_config["strava"]["client_secret"]
+        if "code" in old_config["strava"]:
+            config.strava_client_code = old_config["strava"]["code"]
+
+    if "offsets" in old_config:
+        config.equipment_offsets = old_config["offsets"]
+
+    if "upload" in old_config:
+        if "password" in old_config["upload"]:
+            config.upload_password = old_config["upload"]["password"]
+
+    if "privacy_zones" in old_config:
+        config.privacy_zones = old_config["privacy_zones"]
+
+    config_accessor.save()
+
+
+def import_old_strava_config(config_accessor: ConfigAccessor) -> None:
+    if not strava_dynamic_config_path().exists():
+        return
+
+    with open(strava_dynamic_config_path()) as f:
+        strava_dynamic_config = json.load(f)
+
+    config = config_accessor()
+    config.strava_client_id = strava_dynamic_config["client_id"]
+    config.strava_client_secret = strava_dynamic_config["client_secret"]
+    config.strava_client_code = strava_dynamic_config["code"]
+
+    config_accessor.save()
+    strava_dynamic_config_path().unlink()

geo_activity_playground/core/enrichment.py

@@ -10,6 +10,7 @@ from tqdm import tqdm
 
 from geo_activity_playground.core.activities import ActivityMeta
 from geo_activity_playground.core.activities import make_activity_meta
+from geo_activity_playground.core.config import Config
 from geo_activity_playground.core.coordinates import get_distance
 from geo_activity_playground.core.paths import activity_enriched_meta_dir
 from geo_activity_playground.core.paths import activity_enriched_time_series_dir
@@ -21,7 +22,7 @@ from geo_activity_playground.core.time_conversion import convert_to_datetime_ns
 logger = logging.getLogger(__name__)
 
 
-def enrich_activities(kind_defaults: dict[dict[str, Any]]) -> None:
+def enrich_activities(config: Config) -> None:
     # Delete removed activities.
     for enriched_metadata_path in activity_enriched_meta_dir().glob("*.pickle"):
         if not (activity_extracted_meta_dir() / enriched_metadata_path.name).exists():
@@ -74,8 +75,16 @@ def enrich_activities(kind_defaults: dict[dict[str, Any]]) -> None:
         metadata = make_activity_meta()
         metadata.update(extracted_metadata)
 
+        # Skip activities that don't have geo information attached to them. This shouldn't happen, though.
+        if "latitude" not in time_series.columns:
+            logger.warning(
+                f"Activity {metadata} doesn't have latitude/longitude information. Ignoring this one."
+            )
+            continue
+
         # Enrich time series.
-        metadata.update(kind_defaults.get(metadata["kind"], {}))
+        if metadata["kind"] in config.kinds_without_achievements:
+            metadata["consider_for_achievements"] = False
         time_series = _embellish_single_time_series(
             time_series, metadata.get("start", None)
         )

geo_activity_playground/core/heart_rate.py

@@ -0,0 +1,49 @@
+import datetime
+import math
+from typing import Optional
+
+import pandas as pd
+
+from geo_activity_playground.core.config import Config
+
+
+class HeartRateZoneComputer:
+    def __init__(
+        self,
+        config: Config,
+    ) -> None:
+        self._config = config
+
+    def compute_zones(self, frequencies: pd.Series, year: int) -> pd.Series:
+        maximum = self._get_maximum(year)
+        zones: pd.Series = (frequencies - self._config.heart_rate_resting) * 10 // (
+            maximum - self._config.heart_rate_resting
+        ) - 4
+        zones.loc[zones < 0] = 0
+        zones.loc[zones > 5] = 5
+        return zones
+
+    def zone_boundaries(self) -> list[tuple[int, int]]:
+        maximum = self._get_maximum(datetime.date.today().year)
+        result = []
+        for zone in [1, 2, 3, 4, 5]:
+            lower = math.ceil(
+                (zone + 4) / 10 * (maximum - self._config.heart_rate_resting)
+                + self._config.heart_rate_resting
+            )
+            upper = math.floor(
+                (zone + 5) / 10 * (maximum - self._config.heart_rate_resting)
+                + self._config.heart_rate_resting
+            )
+            result.append((lower, upper))
+        return result
+
+    def _get_maximum(self, year: int) -> int:
+        if self._config.heart_rate_maximum:
+            return self._config.heart_rate_maximum
+        elif self._config.birth_year:
+            return 220 - year + self._config.birth_year
+        else:
+            raise RuntimeError(
+                "Cannot compute heart rate maximum from the given configuration items."
+            )

geo_activity_playground/core/paths.py

@@ -41,6 +41,10 @@ _tiles_per_time_series = _cache_dir / "Tiles" / "Tiles Per Time Series"
 _strava_api_dir = pathlib.Path("Strava API")
 _strava_dynamic_config_path = _strava_api_dir / "strava-client-id.json"
 
+_strava_last_activity_date_path = _cache_dir / "strava-last-activity-date.json"
+
+_new_config_file = pathlib.Path("config.json")
+
 
 cache_dir = dir_wrapper(_cache_dir)
 
@@ -54,3 +58,5 @@ strava_api_dir = dir_wrapper(_strava_api_dir)
 
 activities_file = file_wrapper(_activities_file)
 strava_dynamic_config_path = file_wrapper(_strava_dynamic_config_path)
+strava_last_activity_date_path = file_wrapper(_strava_last_activity_date_path)
+new_config_file = file_wrapper(_new_config_file)

geo_activity_playground/core/tasks.py

@@ -100,3 +100,17 @@ class TransformVersion:
     def write(self) -> None:
         with open(self._path, "w") as f:
             json.dump(self._code_version, f)
+
+
+def get_state(path: pathlib.Path, default: Any) -> Any:
+    if path.exists():
+        with open(path) as f:
+            return json.load(f)
+    else:
+        return default
+
+
+def set_state(path: pathlib.Path, state: Any) -> None:
+    path.parent.mkdir(exist_ok=True, parents=True)
+    with open(path, "w") as f:
+        json.dump(state, f, indent=2, sort_keys=True, ensure_ascii=False)

geo_activity_playground/core/tiles.py

@@ -95,7 +95,7 @@ def interpolate_missing_tile(
         return None
 
     # Some people have large jumps in their tracks. We don't want to interpolate when there is more than tile in between.
-    if abs(x1 - x2) > 1 or abs(y1 - y2) > 1:
+    if abs(int(x1) - int(x2)) > 1 or abs(int(y1) - int(y2)) > 1:
         return None
 
     x_hat = int(max(x1, x2))

geo_activity_playground/explorer/tile_visits.py

@@ -13,6 +13,7 @@ import pandas as pd
 from tqdm import tqdm
 
 from geo_activity_playground.core.activities import ActivityRepository
+from geo_activity_playground.core.config import Config
 from geo_activity_playground.core.paths import tiles_per_time_series
 from geo_activity_playground.core.tasks import try_load_pickle
 from geo_activity_playground.core.tasks import work_tracker_path
@@ -195,29 +196,36 @@ class TileEvolutionState:
         self.square_y: Optional[int] = None
 
 
-def compute_tile_evolution(tile_visits_accessor: TileVisitAccessor) -> None:
-    zoom_levels = list(reversed(list(range(20))))
-
-    for zoom in tqdm(zoom_levels, desc="Compute explorer cluster evolution"):
+def compute_tile_evolution(
+    tile_visits_accessor: TileVisitAccessor, config: Config
+) -> None:
+    for zoom in config.explorer_zoom_levels:
         _compute_cluster_evolution(
-            tile_visits_accessor.histories[zoom], tile_visits_accessor.states[zoom]
+            tile_visits_accessor.histories[zoom],
+            tile_visits_accessor.states[zoom],
+            zoom,
         )
-    for zoom in tqdm(zoom_levels, desc="Compute explorer square evolution"):
         _compute_square_history(
-            tile_visits_accessor.histories[zoom], tile_visits_accessor.states[zoom]
+            tile_visits_accessor.histories[zoom],
+            tile_visits_accessor.states[zoom],
+            zoom,
         )
 
     tile_visits_accessor.save()
 
 
-def _compute_cluster_evolution(tiles: pd.DataFrame, s: TileEvolutionState) -> None:
+def _compute_cluster_evolution(
+    tiles: pd.DataFrame, s: TileEvolutionState, zoom: int
+) -> None:
     if len(s.cluster_evolution) > 0:
         max_cluster_so_far = s.cluster_evolution["max_cluster_size"].iloc[-1]
     else:
         max_cluster_so_far = 0
 
     rows = []
-    for index, row in tiles.iloc[s.cluster_start :].iterrows():
+    for index, row in tqdm(
+        tiles.iloc[s.cluster_start :].iterrows(), desc=f"Cluster evolution for {zoom=}"
+    ):
         new_clusters = False
         # Current tile.
         tile = (row["tile_x"], row["tile_y"])
@@ -288,9 +296,13 @@ def _compute_cluster_evolution(tiles: pd.DataFrame, s: TileEvolutionState) -> No
     s.cluster_start = len(tiles)
 
 
-def _compute_square_history(tiles: pd.DataFrame, s: TileEvolutionState) -> None:
+def _compute_square_history(
+    tiles: pd.DataFrame, s: TileEvolutionState, zoom: int
+) -> None:
     rows = []
-    for index, row in tiles.iloc[s.square_start :].iterrows():
+    for index, row in tqdm(
+        tiles.iloc[s.square_start :].iterrows(), desc=f"Square evolution for {zoom=}"
+    ):
         tile = (row["tile_x"], row["tile_y"])
         x, y = tile
         s.visited_tiles.add(tile)

geo_activity_playground/importers/csv_parser.py

@@ -0,0 +1,73 @@
+"""
+CSV parser that can handle newlines in cells.
+
+In the Strava export there is a file `activities.csv`. With CSV being a horrible format, there are of course issues with it. One is that the activity description can have newlines in it, they are in the CSV file in verbatim. Therefore we need to have a CSV parser that can handle it. `pandas.read_csv` cannot do it.
+
+The grammar that we have looks like this:
+
+    document ::= line [line ...]
+
+    line ::= cell [ "," cell ...] "\n"
+
+    cell ::= '"' text_with_comma '"' | text_without_comma
+
+    text_with_comma ::= (token | '\\n' | ',') ...
+    text_without_comma ::= token ...
+
+This module implements a "recursive descent parser" that parses this grammar.
+"""
+
+
+def parse_csv(text: str) -> list[list]:
+    text = text.strip() + "\n"
+    result = {}
+    index = 0
+    result = []
+    while index < len(text):
+        line, index = _parse_line(text, index)
+        result.append(line)
+        assert len(line) == len(
+            result[0]
+        ), f"Expected {len(result[0])} columns at {index=}, got {len(line)} columns"
+
+    return result
+
+
+def _parse_line(text: str, start: int) -> tuple[list, int]:
+    index = start
+    result = []
+    while index < len(text) and text[index] != "\n":
+        cell, index = _parse_cell(text, index)
+        result.append(cell)
+        if text[index] == "\n":
+            return result, index + 1
+        else:
+            assert text[index] == ",", f"Expected ',' at {index=}, got {text[index]}"
+            index += 1
+    return result, index
+
+
+def _parse_cell(text: str, start: int) -> tuple[str, int]:
+    characters = []
+    escape = False
+    within_quotes = False
+    i = start
+    for i in range(start, len(text) + 1):
+        if i == len(text):
+            break
+
+        c = text[i]
+
+        if c == '"' and not escape:
+            within_quotes = not within_quotes
+            continue
+        elif c == "\\":
+            escape = True
+            continue
+        elif (c == "," or c == "\n") and not within_quotes:
+            break
+        else:
+            characters.append(c)
+            escape = False
+
+    return "".join(characters), i

geo_activity_playground/importers/directory.py

@@ -23,7 +23,9 @@ logger = logging.getLogger(__name__)
 ACTIVITY_DIR = pathlib.Path("Activities")
 
 
-def import_from_directory(metadata_extraction_regexes: list[str] = []) -> None:
+def import_from_directory(
+    metadata_extraction_regexes: list[str], num_processes: Optional[int]
+) -> None:
 
     activity_paths = [
         path
@@ -57,13 +59,20 @@ def import_from_directory(metadata_extraction_regexes: list[str] = []) -> None:
             del file_hashes[deleted_file]
             work_tracker.discard(deleted_file)
 
-    with multiprocessing.Pool() as pool:
-        paths_with_errors = tqdm(
-            pool.imap(_cache_single_file, new_activity_paths),
-            desc="Parse activity metadata",
-            total=len(new_activity_paths),
-        )
-        paths_with_errors = [error for error in paths_with_errors if error]
+    if num_processes == 1:
+        paths_with_errors = []
+        for path in tqdm(new_activity_paths, desc="Parse activity metadata"):
+            errors = _cache_single_file(path)
+            if errors:
+                paths_with_errors.append(errors)
+    else:
+        with multiprocessing.Pool(num_processes) as pool:
+            paths_with_errors = tqdm(
+                pool.imap(_cache_single_file, new_activity_paths),
+                desc="Parse activity metadata (concurrently)",
+                total=len(new_activity_paths),
+            )
+            paths_with_errors = [error for error in paths_with_errors if error]
 
     for path in tqdm(new_activity_paths, desc="Collate activity metadata"):
         activity_id = get_file_hash(path)