geo-activity-playground 0.17.5__tar.gz → 0.18.0__tar.gz

{geo_activity_playground-0.17.5 → geo_activity_playground-0.18.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: geo-activity-playground
-Version: 0.17.5
+Version: 0.18.0
 Summary: Analysis of geo data activities like rides, runs or hikes.
 License: MIT
 Author: Martin Ueding

{geo_activity_playground-0.17.5 → geo_activity_playground-0.18.0}/geo_activity_playground/__main__.py

@@ -7,6 +7,7 @@ import sys
 import coloredlogs
 
 from .importers.strava_checkout import convert_strava_checkout
+from .importers.strava_checkout import import_from_strava_checkout
 from geo_activity_playground.core.activities import ActivityRepository
 from geo_activity_playground.core.activities import embellish_time_series
 from geo_activity_playground.core.cache_migrations import apply_cache_migrations
@@ -99,17 +100,16 @@ def make_activity_repository(basedir: pathlib.Path) -> ActivityRepository:
     os.chdir(basedir)
     apply_cache_migrations()
     config = get_config()
-    if pathlib.Path("Activities").exists():
-        import_from_directory()
-    elif config:
-        if "strava" in config:
-            import_from_strava_api()
-    else:
-        logger.error(
-            "You need to either have (1) an “Activities” directory with GPX/FIT/TCX/KML files in there or (2) a “config.toml” with information for the Strava API (see https://martin-ueding.github.io/geo-activity-playground/getting-started/using-strava-api/)."
-        )
-        sys.exit(1)
+
     repository = ActivityRepository()
+
+    if pathlib.Path("Activities").exists():
+        import_from_directory(repository)
+    if pathlib.Path("Strava Export").exists():
+        import_from_strava_checkout(repository)
+    if "strava" in config:
+        import_from_strava_api(repository)
+
     embellish_time_series(repository)
     compute_tile_visits(repository)
     compute_tile_evolution()

{geo_activity_playground-0.17.5 → geo_activity_playground-0.18.0}/geo_activity_playground/core/activities.py

@@ -33,14 +33,52 @@ class ActivityMeta(TypedDict):
     start: datetime.datetime
 
 
+activity_path = pathlib.Path("Cache/activities.parquet")
+
+
 class ActivityRepository:
     def __init__(self) -> None:
-        self.meta = pd.read_parquet("Cache/activities.parquet")
-        self.meta.index = self.meta["id"]
-        self.meta.index.name = "index"
-        self.meta["distance"] /= 1000
-        self.meta["kind"].fillna("Unknown", inplace=True)
-        self.meta["equipment"].fillna("Unknown", inplace=True)
+        if activity_path.exists():
+            self.meta = pd.read_parquet(activity_path)
+        else:
+            self.meta = pd.DataFrame()
+
+        self._loose_activities: list[ActivityMeta] = []
+
+    def add_activity(self, activity_meta: ActivityMeta) -> None:
+        self._loose_activities.append(activity_meta)
+
+    def commit(self) -> None:
+        if self._loose_activities:
+            logger.debug(
+                f"Adding {len(self._loose_activities)} activities to the repository …"
+            )
+            new_df = pd.DataFrame(self._loose_activities)
+            self.meta = pd.concat([self.meta, new_df])
+            assert pd.api.types.is_dtype_equal(
+                self.meta["start"].dtype, "datetime64[ns, UTC]"
+            ), self.meta["start"].dtype
+            self.meta.index = self.meta["id"]
+            self.meta.index.name = "index"
+            self.meta.sort_values("start", inplace=True)
+            activity_path.parent.mkdir(exist_ok=True, parents=True)
+            self.meta.to_parquet(activity_path)
+            self._loose_activities = []
+
+    def has_activity(self, activity_id: int) -> bool:
+        if len(self.meta):
+            if activity_id in self.meta["id"]:
+                return True
+
+        for activity_meta in self._loose_activities:
+            if activity_meta["id"] == activity_id:
+                return True
+
+        return False
+
+    def last_activity_date(self) -> Optional[datetime.datetime]:
+        if len(self.meta):
+            return self.meta.iloc[-1]["start"]
 
     @property
     def activity_ids(self) -> set[int]:
@@ -64,43 +102,6 @@ class ActivityRepository:
             logger.error(f"Error while reading {path}, deleting cache file …")
             path.unlink(missing_ok=True)
             raise
-        changed = False
-        if pd.api.types.is_dtype_equal(df["time"].dtype, "int64"):
-            start = self.get_activity_by_id(id)["start"]
-            time = df["time"]
-            del df["time"]
-            df["time"] = [start + datetime.timedelta(seconds=t) for t in time]
-            changed = True
-        assert pd.api.types.is_dtype_equal(df["time"].dtype, "datetime64[ns, UTC]")
-
-        if "distance" in df.columns:
-            if "distance/km" not in df.columns:
-                df["distance/km"] = df["distance"] / 1000
-                changed = True
-
-            if "speed" not in df.columns:
-                df["speed"] = (
-                    df["distance"].diff()
-                    / (df["time"].diff().dt.total_seconds() + 1e-3)
-                    * 3.6
-                )
-                changed = True
-
-        if "latitude" in df.columns and "x" not in df.columns:
-            x, y = compute_tile_float(df["latitude"], df["longitude"], 0)
-            df["x"] = x
-            df["y"] = y
-            changed = True
-
-        if "segment_id" not in df.columns:
-            time_diff = (df["time"] - df["time"].shift(1)).dt.total_seconds()
-            jump_indices = time_diff >= 30
-            df["segment_id"] = np.cumsum(jump_indices)
-            changed = True
-
-        if changed:
-            logger.info(f"Updating activity time series for {id = } …")
-            df.to_parquet(path)
 
         return df
 
@@ -169,29 +170,25 @@ def make_geojson_from_time_series(time_series: pd.DataFrame) -> str:
 
 def make_geojson_color_line(time_series: pd.DataFrame) -> str:
     cmap = matplotlib.colormaps["viridis"]
-    return geojson.dumps(
-        geojson.FeatureCollection(
-            features=[
-                geojson.Feature(
-                    geometry=geojson.LineString(
-                        coordinates=[
-                            [row["longitude"], row["latitude"]],
-                            [next["longitude"], next["latitude"]],
-                        ]
-                    ),
-                    properties={
-                        "speed": next["speed"],
-                        "color": matplotlib.colors.to_hex(
-                            cmap(min(next["speed"] / 35, 1.0))
-                        ),
-                    },
-                )
-                for (_, row), (_, next) in zip(
-                    time_series.iterrows(), time_series.iloc[1:].iterrows()
-                )
-            ]
+    features = [
+        geojson.Feature(
+            geometry=geojson.LineString(
+                coordinates=[
+                    [row["longitude"], row["latitude"]],
+                    [next["longitude"], next["latitude"]],
+                ]
+            ),
+            properties={
+                "speed": next["speed"] if np.isfinite(next["speed"]) else 0.0,
+                "color": matplotlib.colors.to_hex(cmap(min(next["speed"] / 35, 1.0))),
+            },
         )
-    )
+        for (_, row), (_, next) in zip(
+            time_series.iterrows(), time_series.iloc[1:].iterrows()
+        )
+    ]
+    feature_collection = geojson.FeatureCollection(features)
+    return geojson.dumps(feature_collection)
 
 
 def extract_heart_rate_zones(time_series: pd.DataFrame) -> Optional[pd.DataFrame]:

{geo_activity_playground-0.17.5 → geo_activity_playground-0.18.0}/geo_activity_playground/core/cache_migrations.py

@@ -64,6 +64,7 @@ def apply_cache_migrations() -> None:
         delete_tile_visits,
         delete_heatmap_cache,
         delete_activity_metadata,
+        delete_activity_metadata,
     ]
 
     for migration in migrations[cache_status["num_applied_migrations"] :]:

{geo_activity_playground-0.17.5 → geo_activity_playground-0.18.0}/geo_activity_playground/importers/directory.py

@@ -8,6 +8,7 @@ import pandas as pd
 from tqdm import tqdm
 
 from geo_activity_playground.core.activities import ActivityMeta
+from geo_activity_playground.core.activities import ActivityRepository
 from geo_activity_playground.core.activity_parsers import ActivityParseError
 from geo_activity_playground.core.activity_parsers import read_activity
 from geo_activity_playground.core.tasks import WorkTracker
@@ -15,26 +16,19 @@ from geo_activity_playground.core.tasks import WorkTracker
 logger = logging.getLogger(__name__)
 
 
-def import_from_directory() -> None:
-    meta_file = pathlib.Path("Cache") / "activities.parquet"
-    if meta_file.exists():
-        meta = pd.read_parquet(meta_file)
-    else:
-        meta = None
-
+def import_from_directory(repository: ActivityRepository) -> None:
     paths_with_errors = []
     work_tracker = WorkTracker("parse-activity-files")
 
     activity_paths = {
         int(hashlib.sha3_224(str(path).encode()).hexdigest(), 16) % 2**62: path
         for path in pathlib.Path("Activities").rglob("*.*")
-        if path.is_file() and path.suffixes
+        if path.is_file() and path.suffixes and not path.stem.startswith(".")
     }
     activities_ids_to_parse = work_tracker.filter(activity_paths.keys())
 
     activity_stream_dir = pathlib.Path("Cache/Activity Timeseries")
     activity_stream_dir.mkdir(exist_ok=True, parents=True)
-    new_rows: list[dict] = []
     for activity_id in tqdm(activities_ids_to_parse, desc="Parse activity files"):
         path = activity_paths[activity_id]
         try:
@@ -62,6 +56,8 @@ def import_from_directory() -> None:
             # https://stackoverflow.com/a/74718395/653152
             name=path.name.removesuffix("".join(path.suffixes)),
             path=str(path),
+            kind="Unknown",
+            equipment="Unknown",
         )
         if len(path.parts) >= 3 and path.parts[1] != "Commute":
             activity_meta["kind"] = path.parts[1]
@@ -69,7 +65,7 @@ def import_from_directory() -> None:
             activity_meta["equipment"] = path.parts[2]
 
         activity_meta.update(activity_meta_from_file)
-        new_rows.append(activity_meta)
+        repository.add_activity(activity_meta)
 
     if paths_with_errors:
         logger.warning(
@@ -78,18 +74,6 @@ def import_from_directory() -> None:
         for path, error in paths_with_errors:
             logger.error(f"{path}: {error}")
 
-    new_df = pd.DataFrame(new_rows)
-    merged = pd.concat([meta, new_df])
-
-    if len(merged) == 0:
-        activities_dir = pathlib.Path("Activities").resolve()
-        logger.error(
-            f"You seemingly want to use activity files as a data source, but you have not copied any GPX/FIT/TCX/KML files."
-            f"Please copy at least one such file into {activities_dir}."
-        )
-        sys.exit(1)
+    repository.commit()
 
-    merged.sort_values("start", inplace=True)
-    meta_file.parent.mkdir(exist_ok=True, parents=True)
-    merged.to_parquet(meta_file)
     work_tracker.close()

{geo_activity_playground-0.17.5 → geo_activity_playground-0.18.0}/geo_activity_playground/importers/strava_api.py

@@ -14,6 +14,7 @@ from stravalib.exc import ObjectNotFound
 from stravalib.exc import RateLimitExceeded
 from tqdm import tqdm
 
+from geo_activity_playground.core.activities import ActivityRepository
 from geo_activity_playground.core.config import get_config
 
 
@@ -91,8 +92,8 @@ def round_to_next_quarter_hour(date: datetime.datetime) -> datetime.datetime:
     return next_quarter
 
 
-def import_from_strava_api() -> None:
-    while try_import_strava():
+def import_from_strava_api(repository: ActivityRepository) -> None:
+    while try_import_strava(repository):
         now = datetime.datetime.now()
         next_quarter = round_to_next_quarter_hour(now)
         seconds_to_wait = (next_quarter - now).total_seconds() + 10
@@ -102,22 +103,17 @@ def import_from_strava_api() -> None:
         time.sleep(seconds_to_wait)
 
 
-def try_import_strava() -> None:
-    meta_file = pathlib.Path("Cache") / "activities.parquet"
-    if meta_file.exists():
-        logger.info("Loading metadata file …")
-        meta = pd.read_parquet(meta_file)
-        get_after = meta.iloc[-1]["start"].isoformat().replace("+00:00", "Z")
-    else:
-        logger.info("Didn't find a metadata file.")
-        meta = None
-        get_after = "2000-01-01T00:00:00Z"
+def try_import_strava(repository: ActivityRepository) -> None:
+    get_after = (
+        repository.last_activity_date().isoformat().replace("+00:00", "Z")
+        if repository.last_activity_date() is not None
+        else "2000-01-01T00:00:00Z"
+    )
 
     gear_names = {None: "None"}
 
     client = Client(access_token=get_current_access_token())
 
-    new_rows: list[dict] = []
     try:
         for activity in tqdm(
             client.get_activities(after=get_after), desc="Downloading Strava activities"
@@ -155,7 +151,7 @@ def try_import_strava() -> None:
                 time_series.to_parquet(time_series_path)
 
             if len(time_series) > 0 and "latitude" in time_series.columns:
-                new_rows.append(
+                repository.add_activity(
                     {
                         "id": activity.id,
                         "commute": activity.commute,
@@ -172,11 +168,7 @@ def try_import_strava() -> None:
     except RateLimitExceeded:
         limit_exceeded = True
 
-    new_df = pd.DataFrame(new_rows)
-    merged: pd.DataFrame = pd.concat([meta, new_df])
-    merged.sort_values("start", inplace=True)
-    meta_file.parent.mkdir(exist_ok=True, parents=True)
-    merged.to_parquet(meta_file)
+    repository.commit()
 
     return limit_exceeded
 

geo_activity_playground-0.18.0/geo_activity_playground/importers/strava_checkout.py

@@ -0,0 +1,124 @@
+import datetime
+import logging
+import pathlib
+import shutil
+import traceback
+
+import dateutil.parser
+import numpy as np
+import pandas as pd
+from tqdm import tqdm
+
+from geo_activity_playground.core.activities import ActivityRepository
+from geo_activity_playground.core.activity_parsers import ActivityParseError
+from geo_activity_playground.core.activity_parsers import read_activity
+from geo_activity_playground.core.tasks import WorkTracker
+
+
+logger = logging.getLogger(__name__)
+
+
+def nan_as_none(elem):
+    if isinstance(elem, float) and np.isnan(elem):
+        return None
+    else:
+        return elem
+
+
+def import_from_strava_checkout(repository: ActivityRepository) -> None:
+    checkout_path = pathlib.Path("Strava Export")
+    activities = pd.read_csv(checkout_path / "activities.csv")
+    activities.index = activities["Activity ID"]
+    work_tracker = WorkTracker("import-strava-checkout-activities")
+    activities_ids_to_parse = work_tracker.filter(activities["Activity ID"])
+
+    activity_stream_dir = pathlib.Path("Cache/Activity Timeseries")
+    activity_stream_dir.mkdir(exist_ok=True, parents=True)
+
+    for activity_id in tqdm(activities_ids_to_parse, desc="Import from Strava export"):
+        row = activities.loc[activity_id]
+        activity_file = checkout_path / row["Filename"]
+        table_activity_meta = {
+            "calories": row["Calories"],
+            "commute": row["Commute"] == "true",
+            "distance": row["Distance"],
+            "elapsed_time": datetime.timedelta(seconds=int(row["Elapsed Time"])),
+            "equipment": str(
+                nan_as_none(row["Activity Gear"])
+                or nan_as_none(row["Bike"])
+                or nan_as_none(row["Gear"])
+                or ""
+            ),
+            "kind": row["Activity Type"],
+            "id": activity_id,
+            "name": row["Activity Name"],
+            "path": str(activity_file),
+            "start": dateutil.parser.parse(row["Activity Date"]).astimezone(
+                datetime.timezone.utc
+            ),
+        }
+
+        time_series_path = activity_stream_dir / f"{activity_id}.parquet"
+        if not time_series_path.exists():
+            try:
+                file_activity_meta, time_series = read_activity(activity_file)
+            except ActivityParseError as e:
+                logger.error(f"Error while parsing file {activity_file}:")
+                traceback.print_exc()
+                continue
+            except:
+                logger.error(
+                    f"Encountered a problem with {activity_file=}, see details below."
+                )
+                raise
+
+            if not len(time_series):
+                continue
+
+            time_series.to_parquet(time_series_path)
+
+        work_tracker.mark_done(activity_id)
+        repository.add_activity(table_activity_meta)
+
+    repository.commit()
+    work_tracker.close()
+
+
+def convert_strava_checkout(
+    checkout_path: pathlib.Path, playground_path: pathlib.Path
+) -> None:
+    activities = pd.read_csv(checkout_path / "activities.csv")
+    print(activities)
+
+    for _, row in tqdm(activities.iterrows(), desc="Import activity files"):
+        activity_date = dateutil.parser.parse(row["Activity Date"])
+        activity_name = row["Activity Name"]
+        activity_kind = row["Activity Type"]
+        is_commute = row["Commute"] == "true"
+        equipment = (
+            nan_as_none(row["Activity Gear"])
+            or nan_as_none(row["Bike"])
+            or nan_as_none(row["Gear"])
+            or ""
+        )
+        activity_file = checkout_path / row["Filename"]
+
+        activity_target = playground_path / "Activities" / str(activity_kind)
+        if equipment:
+            activity_target /= str(equipment)
+        if is_commute:
+            activity_target /= "Commute"
+
+        activity_target /= "".join(
+            [
+                f"{activity_date.year:04d}-{activity_date.month:02d}-{activity_date.day:02d}",
+                " ",
+                f"{activity_date.hour:02d}-{activity_date.minute:02d}-{activity_date.second:02d}",
+                " ",
+                activity_name,
+            ]
+            + activity_file.suffixes
+        )
+
+        activity_target.parent.mkdir(exist_ok=True, parents=True)
+        shutil.copy(activity_file, activity_target)

{geo_activity_playground-0.17.5 → geo_activity_playground-0.18.0}/geo_activity_playground/webui/calendar_controller.py

@@ -19,7 +19,7 @@ class CalendarController:
         meta["month"] = meta["start"].dt.month
 
         monthly_distance = meta.groupby(["year", "month"]).apply(
-            lambda group: sum(group["distance"])
+            lambda group: sum(group["distance"]) / 1000
         )
         monthly_distance.name = "total_distance"
         monthly_pivot = (
@@ -28,9 +28,19 @@ class CalendarController:
             .fillna(0.0)
         )
 
+        yearly_distance = meta.groupby(["year"]).apply(
+            lambda group: sum(group["distance"]) / 1000
+        )
+        yearly_distance.name = "total_distance"
+        yearly_distances = {
+            row["year"]: row["total_distance"]
+            for index, row in yearly_distance.reset_index().iterrows()
+        }
+
         return {
             "num_activities": len(self._repository.meta),
             "monthly_distances": monthly_pivot,
+            "yearly_distances": yearly_distances,
         }
 
     @functools.cache

{geo_activity_playground-0.17.5 → geo_activity_playground-0.18.0}/geo_activity_playground/webui/equipment_controller.py

@@ -19,7 +19,7 @@ class EquipmentController:
                 lambda group: pd.DataFrame(
                     {
                         "time": group["start"],
-                        "total_distance": group["distance"].cumsum(),
+                        "total_distance": group["distance"].cumsum() / 1000,
                     }
                 )
             )
@@ -50,7 +50,7 @@ class EquipmentController:
             .apply(
                 lambda group: pd.DataFrame(
                     {
-                        "total_distance": group["distance"].sum(),
+                        "total_distance": group["distance"].sum() / 1000,
                         "first_use": group["start"].iloc[0],
                         "last_use": group["start"].iloc[-1],
                     },

{geo_activity_playground-0.17.5 → geo_activity_playground-0.18.0}/geo_activity_playground/webui/templates/activity.html.j2

@@ -17,7 +17,7 @@
             <dt>Commute</dt>
             <dd>{{ activity.commute }}</dd>
             <dt>Distance</dt>
-            <dd>{{ activity.distance|round(1) }} km</dd>
+            <dd>{{ (activity.distance / 1000)|round(1) }} km</dd>
             <dt>Elapsed time</dt>
             <dd>{{ activity.elapsed_time }}</dd>
             <dt>Start time</dt>
@@ -28,6 +28,8 @@
             <dd>{{ activity.equipment }}</dd>
             <dt>ID</dt>
             <dd>{{ activity.id }}</dd>
+            <dt>Source path</dt>
+            <dd>{{ activity.path }}</dd>
         </dl>
     </div>
     <div class="col-8">

{geo_activity_playground-0.17.5 → geo_activity_playground-0.18.0}/geo_activity_playground/webui/templates/calendar.html.j2

@@ -16,6 +16,7 @@
                     {% for i in range(1, 13) %}
                     <th style="text-align: right;">{{ i }}</th>
                     {% endfor %}
+                    <th style="text-align: right;">Total</th>
                 </tr>
             </thead>
             <tbody>
@@ -32,6 +33,7 @@
                         {% endif %}
                     </td>
                     {% endfor %}
+                    <td align="right">{{ yearly_distances[year]|int() }} km</td>
                 </tr>
                 {% endfor %}
             </tbody>

{geo_activity_playground-0.17.5 → geo_activity_playground-0.18.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "geo-activity-playground"
-version = "0.17.5"
+version = "0.18.0"
 description = "Analysis of geo data activities like rides, runs or hikes."
 authors = ["Martin Ueding <mu@martin-ueding.de>"]
 license = "MIT"

geo_activity_playground-0.17.5/geo_activity_playground/importers/strava_checkout.py

@@ -1,54 +0,0 @@
-import pathlib
-import shutil
-
-import dateutil.parser
-import numpy as np
-import pandas as pd
-from tqdm import tqdm
-
-
-def nan_as_none(elem):
-    if isinstance(elem, float) and np.isnan(elem):
-        return None
-    else:
-        return elem
-
-
-def convert_strava_checkout(
-    checkout_path: pathlib.Path, playground_path: pathlib.Path
-) -> None:
-    activities = pd.read_csv(checkout_path / "activities.csv")
-    print(activities)
-
-    for _, row in tqdm(activities.iterrows(), desc="Import activity files"):
-        activity_date = dateutil.parser.parse(row["Activity Date"])
-        activity_name = row["Activity Name"]
-        activity_kind = row["Activity Type"]
-        is_commute = row["Commute"] == "true"
-        equipment = (
-            nan_as_none(row["Activity Gear"])
-            or nan_as_none(row["Bike"])
-            or nan_as_none(row["Gear"])
-            or ""
-        )
-        activity_file = checkout_path / row["Filename"]
-
-        activity_target = playground_path / "Activities" / str(activity_kind)
-        if equipment:
-            activity_target /= str(equipment)
-        if is_commute:
-            activity_target /= "Commute"
-
-        activity_target /= "".join(
-            [
-                f"{activity_date.year:04d}-{activity_date.month:02d}-{activity_date.day:02d}",
-                " ",
-                f"{activity_date.hour:02d}-{activity_date.minute:02d}-{activity_date.second:02d}",
-                " ",
-                activity_name,
-            ]
-            + activity_file.suffixes
-        )
-
-        activity_target.parent.mkdir(exist_ok=True, parents=True)
-        shutil.copy(activity_file, activity_target)