geo-activity-playground 0.23.0__tar.gz → 0.24.1__tar.gz

{geo_activity_playground-0.23.0 → geo_activity_playground-0.24.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: geo-activity-playground
-Version: 0.23.0
+Version: 0.24.1
 Summary: Analysis of geo data activities like rides, runs or hikes.
 License: MIT
 Author: Martin Ueding
@@ -29,6 +29,7 @@ Requires-Dist: pyarrow (>=16.1.0,<17.0.0)
 Requires-Dist: python-dateutil (>=2.8.2,<3.0.0)
 Requires-Dist: requests (>=2.28.1,<3.0.0)
 Requires-Dist: scipy (>=1.8.1,<2.0.0)
+Requires-Dist: shapely (>=2.0.5,<3.0.0)
 Requires-Dist: stravalib (>=1.3.3,<2.0.0)
 Requires-Dist: tcxreader (>=0.4.5,<0.5.0)
 Requires-Dist: tomli (>=2.0.1,<3.0.0) ; python_version < "3.11"

{geo_activity_playground-0.23.0 → geo_activity_playground-0.24.1}/geo_activity_playground/__main__.py

@@ -13,7 +13,7 @@ from geo_activity_playground.core.config import get_config
 from geo_activity_playground.explorer.tile_visits import TileVisitAccessor
 from geo_activity_playground.explorer.video import explorer_video_main
 from geo_activity_playground.webui.app import webui_main
-from geo_activity_playground.webui.upload_controller import scan_for_activities
+from geo_activity_playground.webui.upload.controller import scan_for_activities
 
 logger = logging.getLogger(__name__)
 

{geo_activity_playground-0.23.0 → geo_activity_playground-0.24.1}/geo_activity_playground/core/activities.py

@@ -18,6 +18,7 @@ from geo_activity_playground.core.paths import activities_path
 from geo_activity_playground.core.paths import activity_timeseries_path
 from geo_activity_playground.core.tasks import WorkTracker
 from geo_activity_playground.core.tiles import compute_tile_float
+from geo_activity_playground.core.time_conversion import convert_to_datetime_ns
 
 logger = logging.getLogger(__name__)
 
@@ -25,6 +26,7 @@ logger = logging.getLogger(__name__)
 class ActivityMeta(TypedDict):
     calories: float
     commute: bool
+    consider_for_achievements: bool
     distance_km: float
     elapsed_time: datetime.timedelta
     end_latitude: float
@@ -46,6 +48,10 @@ class ActivityRepository:
             self.meta = pd.read_parquet(activities_path())
             self.meta.index = self.meta["id"]
             self.meta.index.name = "index"
+            if not pd.api.types.is_dtype_equal(
+                self.meta["start"].dtype, "datetime64[ns]"
+            ):
+                self.meta["start"] = convert_to_datetime_ns(self.meta["start"])
         else:
             self.meta = pd.DataFrame()
 
@@ -78,11 +84,6 @@ class ActivityRepository:
                 f"Adding {len(self._loose_activities)} activities to the repository …"
             )
             new_df = pd.DataFrame(self._loose_activities)
-            if not pd.api.types.is_dtype_equal(
-                new_df["start"].dtype, "datetime64[ns, UTC]"
-            ):
-                new_df["start"] = new_df["start"].dt.tz_localize("UTC")
-                new_df["start"] = new_df["start"].dt.tz_convert("UTC")
             if len(self.meta):
                 new_ids_set = set(new_df["id"])
                 is_kept = [
@@ -91,9 +92,10 @@ class ActivityRepository:
                 old_df = self.meta.loc[is_kept]
             else:
                 old_df = self.meta
+
             self.meta = pd.concat([old_df, new_df])
             assert pd.api.types.is_dtype_equal(
-                self.meta["start"].dtype, "datetime64[ns, UTC]"
+                self.meta["start"].dtype, "datetime64[ns]"
             ), (self.meta["start"].dtype, self.meta["start"].iloc[0])
             self.save()
             self._loose_activities = []
@@ -121,9 +123,11 @@ class ActivityRepository:
         else:
             return None
 
-    @property
-    def activity_ids(self) -> set[int]:
-        return set(self.meta.index)
+    def get_activity_ids(self, only_achievements: bool = False) -> set[int]:
+        if only_achievements:
+            return set(self.meta.loc[self.meta["consider_for_achievements"]].index)
+        else:
+            return set(self.meta.index)
 
     def iter_activities(self, new_to_old=True, dropna=False) -> Iterator[ActivityMeta]:
         direction = -1 if new_to_old else 1
@@ -152,7 +156,7 @@ class ActivityRepository:
 
 def embellish_time_series(repository: ActivityRepository) -> None:
     work_tracker = WorkTracker("embellish-time-series")
-    activities_to_process = work_tracker.filter(repository.activity_ids)
+    activities_to_process = work_tracker.filter(repository.get_activity_ids())
     for activity_id in tqdm(activities_to_process, desc="Embellish time series data"):
         path = activity_timeseries_path(activity_id)
         df = pd.read_parquet(path)
@@ -176,9 +180,11 @@ def embellish_single_time_series(
     ):
         time = timeseries["time"]
         del timeseries["time"]
-        timeseries["time"] = [start + datetime.timedelta(seconds=t) for t in time]
+        timeseries["time"] = [
+            convert_to_datetime_ns(start + datetime.timedelta(seconds=t)) for t in time
+        ]
         changed = True
-    assert pd.api.types.is_dtype_equal(timeseries["time"].dtype, "datetime64[ns, UTC]")
+    assert pd.api.types.is_dtype_equal(timeseries["time"].dtype, "datetime64[ns]")
 
     distances = get_distance(
         timeseries["latitude"].shift(1),

{geo_activity_playground-0.23.0 → geo_activity_playground-0.24.1}/geo_activity_playground/core/activity_parsers.py

@@ -8,15 +8,13 @@ import charset_normalizer
 import dateutil.parser
 import fitdecode
 import gpxpy
-import numpy as np
 import pandas as pd
 import tcxreader.tcxreader
 import xmltodict
-from pandas._libs import NaTType
 
 from geo_activity_playground.core.activities import ActivityMeta
 from geo_activity_playground.core.activities import embellish_single_time_series
-from geo_activity_playground.core.coordinates import get_distance
+from geo_activity_playground.core.time_conversion import convert_to_datetime_ns
 
 logger = logging.getLogger(__name__)
 
@@ -58,24 +56,6 @@ def read_activity(path: pathlib.Path) -> tuple[ActivityMeta, pd.DataFrame]:
         raise ActivityParseError(f"Unsupported file format: {file_type}")
 
     if len(timeseries):
-        # Unify time zones to UTC.
-        try:
-            if timeseries["time"].dt.tz is not None:
-                timeseries["time"] = timeseries["time"].dt.tz_localize(None)
-            timeseries["time"] = timeseries["time"].dt.tz_localize("UTC")
-        except AttributeError as e:
-            print(timeseries)
-            print(timeseries.dtypes)
-            types = {}
-            for elem in timeseries["time"]:
-                t = str(type(elem))
-                if t not in types:
-                    types[t] = elem
-            print(types)
-            raise ActivityParseError(
-                "It looks like the date parsing has gone wrong."
-            ) from e
-
         timeseries, changed = embellish_single_time_series(timeseries)
 
         # Extract some meta data from the time series.
@@ -128,11 +108,12 @@ def read_fit_activity(path: pathlib.Path, open) -> tuple[ActivityMeta, pd.DataFr
                     ):
                         time = values["timestamp"]
                         if isinstance(time, datetime.datetime):
-                            time = time.astimezone(datetime.timezone.utc)
+                            pass
                         elif time is None or isinstance(time, int):
                             time = pd.NaT
                         else:
                             raise RuntimeError(f"Cannot parse time: {time} in {path}.")
+                        time = convert_to_datetime_ns(time)
                         row = {
                             "time": time,
                             "latitude": values["position_lat"] / ((2**32) / 360),
@@ -207,13 +188,11 @@ def read_gpx_activity(path: pathlib.Path, open) -> pd.DataFrame:
             for point in segment.points:
                 if isinstance(point.time, datetime.datetime):
                     time = point.time
-                    time = time.astimezone(datetime.timezone.utc)
                 elif isinstance(point.time, str):
                     time = dateutil.parser.parse(str(point.time))
-                    time = time.astimezone(datetime.timezone.utc)
                 else:
                     time = pd.NaT
-                    time.tz_localize("UTC")
+                time = convert_to_datetime_ns(time)
                 points.append((time, point.latitude, point.longitude, point.elevation))
 
     df = pd.DataFrame(points, columns=["time", "latitude", "longitude", "altitude"])
@@ -251,7 +230,7 @@ def read_tcx_activity(path: pathlib.Path, opener) -> pd.DataFrame:
         if trackpoint.latitude and trackpoint.longitude:
             time = trackpoint.time
             assert isinstance(time, datetime.datetime)
-            time = time.astimezone(datetime.timezone.utc)
+            time = convert_to_datetime_ns(time)
             row = {
                 "time": time,
                 "latitude": trackpoint.latitude,
@@ -279,7 +258,8 @@ def read_kml_activity(path: pathlib.Path, opener) -> pd.DataFrame:
     track = placemark["gx:Track"]
     rows = []
     for when, where in zip(track["when"], track["gx:coord"]):
-        time = dateutil.parser.parse(when).astimezone(datetime.timezone.utc)
+        time = dateutil.parser.parse(when)
+        time = convert_to_datetime_ns(time)
         parts = where.split(" ")
         if len(parts) == 2:
             lon, lat = parts
@@ -298,11 +278,7 @@ def read_simra_activity(path: pathlib.Path, opener) -> pd.DataFrame:
     data["time"] = data["timeStamp"].apply(
         lambda d: datetime.datetime.fromtimestamp(d / 1000)
     )
-    tz = (
-        datetime.datetime.now(datetime.timezone.utc).astimezone().tzinfo
-    )  # get local timezone
-    data["time"] = data["time"].dt.tz_localize(tz)
-    data["time"] = data["time"].dt.tz_convert("UTC")
+    data["time"] = convert_to_datetime_ns(data["time"])
     data = data.rename(columns={"lat": "latitude", "lon": "longitude"})
     return data.dropna(subset=["latitude"], ignore_index=True)[
         ["time", "latitude", "longitude"]

{geo_activity_playground-0.23.0 → geo_activity_playground-0.24.1}/geo_activity_playground/core/cache_migrations.py

@@ -34,6 +34,7 @@ def reset_time_series_embellishment() -> None:
 
 def delete_tile_visits() -> None:
     paths = [
+        pathlib.Path("Cache/activities-per-tile.pickle"),
         pathlib.Path("Cache/tile-evolution-state.pickle"),
         pathlib.Path("Cache/tile-history.pickle"),
         pathlib.Path("Cache/tile-visits.pickle"),
@@ -78,6 +79,24 @@ def convert_distances_to_km() -> None:
         time_series.to_parquet(time_series_path)
 
 
+def add_consider_for_achievements() -> None:
+    activities_path = pathlib.Path("Cache/activities.parquet")
+    if activities_path.exists():
+        df = pd.read_parquet(activities_path)
+        if "consider_for_achievements" not in df.columns:
+            df["consider_for_achievements"] = True
+        else:
+            df.loc[
+                df["consider_for_achievements"].isna(), "consider_for_achievements"
+            ] = True
+        df.to_parquet("Cache/activities.parquet")
+
+
+def delete_everything() -> None:
+    if pathlib.Path("Cache").exists():
+        shutil.rmtree("Cache")
+
+
 def apply_cache_migrations() -> None:
     logger.info("Apply cache migration if needed …")
     cache_status_file = pathlib.Path("Cache/status.json")
@@ -98,6 +117,11 @@ def apply_cache_migrations() -> None:
         delete_activity_metadata,
         delete_tile_visits,
         delete_heatmap_cache,
+        add_consider_for_achievements,
+        delete_tile_visits,
+        delete_heatmap_cache,
+        delete_tile_visits,
+        delete_everything,
     ]
 
     for migration in migrations[cache_status["num_applied_migrations"] :]:

{geo_activity_playground-0.23.0 → geo_activity_playground-0.24.1}/geo_activity_playground/core/heatmap.py

@@ -29,7 +29,7 @@ def get_bounds(lat_lon_data: np.ndarray) -> GeoBounds:
 def add_margin(lower: float, upper: float) -> tuple[float, float]:
     spread = upper - lower
     margin = spread / 20
-    return max(0, lower - margin), upper + margin
+    return max(0.0, lower - margin), upper + margin
 
 
 def add_margin_to_geo_bounds(bounds: GeoBounds) -> GeoBounds:
@@ -51,11 +51,28 @@ class TileBounds:
     y_tile_min: int
     y_tile_max: int
 
+
+@dataclasses.dataclass
+class PixelBounds:
+    x_min: int
+    x_max: int
+    y_min: int
+    y_max: int
+
+    @classmethod
+    def from_tile_bounds(cls, tile_bounds: TileBounds) -> "PixelBounds":
+        return cls(
+            int(tile_bounds.x_tile_min) * OSM_TILE_SIZE,
+            int(tile_bounds.x_tile_max) * OSM_TILE_SIZE,
+            int(tile_bounds.y_tile_min) * OSM_TILE_SIZE,
+            int(tile_bounds.y_tile_max) * OSM_TILE_SIZE,
+        )
+
     @property
     def shape(self) -> tuple[int, int]:
         return (
-            (self.y_tile_max - self.y_tile_min) * OSM_TILE_SIZE,
-            (self.x_tile_max - self.x_tile_min) * OSM_TILE_SIZE,
+            self.y_max - self.y_min,
+            self.x_max - self.x_min,
         )
 
 
@@ -107,7 +124,7 @@ def get_sensible_zoom_level(
 
 
 def build_map_from_tiles(tile_bounds: TileBounds) -> np.ndarray:
-    background = np.zeros((*tile_bounds.shape, 3))
+    background = np.zeros((*PixelBounds.from_tile_bounds(tile_bounds).shape, 3))
 
     for x in range(tile_bounds.x_tile_min, tile_bounds.x_tile_max):
         for y in range(tile_bounds.y_tile_min, tile_bounds.y_tile_max):
@@ -129,20 +146,3 @@ def convert_to_grayscale(image: np.ndarray) -> np.ndarray:
     image = np.sum(image * [0.2126, 0.7152, 0.0722], axis=2)
     image = np.dstack((image, image, image))
     return image
-
-
-def crop_image_to_bounds(
-    image: np.ndarray, geo_bounds: GeoBounds, tile_bounds: TileBounds
-) -> np.ndarray:
-    min_x, min_y = compute_tile_float(
-        geo_bounds.lat_max, geo_bounds.lon_min, tile_bounds.zoom
-    )
-    max_x, max_y = compute_tile_float(
-        geo_bounds.lat_min, geo_bounds.lon_max, tile_bounds.zoom
-    )
-    min_x = int((min_x - tile_bounds.x_tile_min) * OSM_TILE_SIZE)
-    min_y = int((min_y - tile_bounds.y_tile_min) * OSM_TILE_SIZE)
-    max_x = int((max_x - tile_bounds.x_tile_min) * OSM_TILE_SIZE)
-    max_y = int((max_y - tile_bounds.y_tile_min) * OSM_TILE_SIZE)
-    image = image[min_y:max_y, min_x:max_x, :]
-    return image

geo_activity_playground-0.24.1/geo_activity_playground/core/privacy_zones.py

@@ -0,0 +1,16 @@
+import pandas as pd
+import shapely
+
+
+class PrivacyZone:
+    def __init__(self, points: list[list[float]]) -> None:
+        self.points = points
+        self._polygon = shapely.Polygon(points)
+        shapely.prepare(self._polygon)
+
+    def filter_time_series(self, time_series: pd.DataFrame) -> pd.DataFrame:
+        mask = [
+            not shapely.contains_xy(self._polygon, row["longitude"], row["latitude"])
+            for index, row in time_series.iterrows()
+        ]
+        return time_series.loc[mask]

{geo_activity_playground-0.23.0 → geo_activity_playground-0.24.1}/geo_activity_playground/core/similarity.py

@@ -29,7 +29,7 @@ def precompute_activity_distances(repository: ActivityRepository) -> None:
     with stored_object(fingerprint_path, {}) as fingerprints, stored_object(
         distances_path, {}
     ) as distances:
-        activity_ids = repository.activity_ids
+        activity_ids = repository.get_activity_ids()
 
         activity_ids_without_fingerprint = [
             activity_id

geo_activity_playground-0.24.1/geo_activity_playground/core/test_time_conversion.py

@@ -0,0 +1,37 @@
+import datetime
+
+import numpy as np
+import pandas as pd
+
+from .time_conversion import convert_to_datetime_ns
+
+target = np.datetime64(datetime.datetime(2000, 1, 2, 3, 4, 5))
+
+
+def test_convert_to_datetime_ns() -> None:
+    dt_local = datetime.datetime(2000, 1, 2, 3, 4, 5)
+    dt_tz = datetime.datetime(
+        2000, 1, 2, 3, 4, 5, tzinfo=datetime.timezone(datetime.timedelta(hours=3))
+    )
+    dt_utc = datetime.datetime(2000, 1, 2, 3, 4, 5, tzinfo=datetime.timezone.utc)
+
+    inputs = [
+        dt_local,
+        dt_tz,
+        dt_utc,
+        pd.Timestamp(dt_local),
+        pd.Timestamp(dt_tz),
+        pd.Timestamp(dt_utc),
+    ]
+
+    for d in inputs:
+        actual = convert_to_datetime_ns(d)
+        # assert pd.api.types.is_dtype_equal(actual.dtype, "datetime64[ns]")
+        assert actual == target
+
+        actual = convert_to_datetime_ns(pd.Series([d]))
+        assert actual.iloc[0] == target
+
+
+def test_NaT() -> None:
+    assert pd.isna(convert_to_datetime_ns(pd.NaT))

geo_activity_playground-0.24.1/geo_activity_playground/core/time_conversion.py

@@ -0,0 +1,14 @@
+import numpy as np
+import pandas as pd
+
+
+def convert_to_datetime_ns(date) -> np.datetime64:
+    if isinstance(date, pd.Series):
+        ts = pd.to_datetime(date)
+        ts = ts.dt.tz_localize(None)
+        return ts
+    else:
+        ts = pd.to_datetime(date)
+        if ts.tzinfo is not None:
+            ts = ts.tz_localize(None)
+        return ts.to_datetime64()

{geo_activity_playground-0.23.0 → geo_activity_playground-0.24.1}/geo_activity_playground/explorer/tile_visits.py

@@ -25,11 +25,13 @@ class TileVisitAccessor:
     TILE_EVOLUTION_STATES_PATH = pathlib.Path("Cache/tile-evolution-state.pickle")
     TILE_HISTORIES_PATH = pathlib.Path(f"Cache/tile-history.pickle")
     TILE_VISITS_PATH = pathlib.Path(f"Cache/tile-visits.pickle")
+    ACTIVITIES_PER_TILE_PATH = pathlib.Path(f"Cache/activities-per-tile.pickle")
 
     def __init__(self) -> None:
         self.visits: dict[int, dict[tuple[int, int], dict[str, Any]]] = try_load_pickle(
             self.TILE_VISITS_PATH
         ) or collections.defaultdict(dict)
+        "zoom → (tile_x, tile_y) → tile_info"
 
         self.histories: dict[int, pd.DataFrame] = try_load_pickle(
             self.TILE_HISTORIES_PATH
@@ -39,6 +41,12 @@ class TileVisitAccessor:
             self.TILE_EVOLUTION_STATES_PATH
         ) or collections.defaultdict(TileEvolutionState)
 
+        self.activities_per_tile: dict[
+            int, dict[tuple[int, int], set[int]]
+        ] = try_load_pickle(self.ACTIVITIES_PER_TILE_PATH) or collections.defaultdict(
+            dict
+        )
+
     def save(self) -> None:
         with open(self.TILE_VISITS_PATH, "wb") as f:
             pickle.dump(self.visits, f)
@@ -49,13 +57,16 @@ class TileVisitAccessor:
         with open(self.TILE_EVOLUTION_STATES_PATH, "wb") as f:
             pickle.dump(self.states, f)
 
+        with open(self.ACTIVITIES_PER_TILE_PATH, "wb") as f:
+            pickle.dump(self.activities_per_tile, f)
+
 
 def compute_tile_visits(
     repository: ActivityRepository, tile_visits_accessor: TileVisitAccessor
 ) -> None:
 
     work_tracker = WorkTracker("tile-visits")
-    activity_ids_to_process = work_tracker.filter(repository.activity_ids)
+    activity_ids_to_process = work_tracker.filter(repository.get_activity_ids())
     new_tile_history_rows = collections.defaultdict(list)
     for activity_id in tqdm(
         activity_ids_to_process, desc="Extract explorer tile visits"
@@ -64,41 +75,42 @@ def compute_tile_visits(
         for zoom in range(20):
             for time, tile_x, tile_y in _tiles_from_points(time_series, zoom):
                 tile = (tile_x, tile_y)
-                if tile in tile_visits_accessor.visits[zoom]:
-                    d = tile_visits_accessor.visits[zoom][tile]
-                    if d["first_time"] > time:
-                        d["first_time"] = time
-                        d["first_id"] = activity_id
-                    if d["last_time"] < time:
-                        d["last_time"] = time
-                        d["last_id"] = activity_id
-                    d["activity_ids"].add(activity_id)
-                else:
-                    tile_visits_accessor.visits[zoom][tile] = {
-                        "first_time": time,
-                        "first_id": activity_id,
-                        "last_time": time,
-                        "last_id": activity_id,
-                        "activity_ids": {activity_id},
-                    }
-                    new_tile_history_rows[zoom].append(
-                        {
-                            "activity_id": activity_id,
-                            "time": time,
-                            "tile_x": tile_x,
-                            "tile_y": tile_y,
+                if not tile in tile_visits_accessor.activities_per_tile[zoom]:
+                    tile_visits_accessor.activities_per_tile[zoom][tile] = set()
+                tile_visits_accessor.activities_per_tile[zoom][tile].add(activity_id)
+
+                activity = repository.get_activity_by_id(activity_id)
+                if activity["consider_for_achievements"]:
+                    if tile in tile_visits_accessor.visits[zoom]:
+                        d = tile_visits_accessor.visits[zoom][tile]
+                        if d["first_time"] > time:
+                            d["first_time"] = time
+                            d["first_id"] = activity_id
+                        if d["last_time"] < time:
+                            d["last_time"] = time
+                            d["last_id"] = activity_id
+                        d["activity_ids"].add(activity_id)
+                    else:
+                        tile_visits_accessor.visits[zoom][tile] = {
+                            "first_time": time,
+                            "first_id": activity_id,
+                            "last_time": time,
+                            "last_id": activity_id,
+                            "activity_ids": {activity_id},
                         }
-                    )
+                        new_tile_history_rows[zoom].append(
+                            {
+                                "activity_id": activity_id,
+                                "time": time,
+                                "tile_x": tile_x,
+                                "tile_y": tile_y,
+                            }
+                        )
         work_tracker.mark_done(activity_id)
 
-    if activity_ids_to_process:
+    if new_tile_history_rows:
         for zoom, new_rows in new_tile_history_rows.items():
             new_df = pd.DataFrame(new_rows)
-            if not pd.api.types.is_dtype_equal(
-                new_df["time"].dtype, "datetime64[ns, UTC]"
-            ):
-                new_df["time"] = new_df["time"].dt.tz_localize("UTC")
-                new_df["time"] = new_df["time"].dt.tz_convert("UTC")
             new_df.sort_values("time", inplace=True)
             tile_visits_accessor.histories[zoom] = pd.concat(
                 [tile_visits_accessor.histories[zoom], new_df]
@@ -112,7 +124,7 @@ def compute_tile_visits(
 def _tiles_from_points(
     time_series: pd.DataFrame, zoom: int
 ) -> Iterator[tuple[datetime.datetime, int, int]]:
-    assert pd.api.types.is_dtype_equal(time_series["time"].dtype, "datetime64[ns, UTC]")
+    assert pd.api.types.is_dtype_equal(time_series["time"].dtype, "datetime64[ns]")
     xf = time_series["x"] * 2**zoom
     yf = time_series["y"] * 2**zoom
     for t1, x1, y1, x2, y2, s1, s2 in zip(

{geo_activity_playground-0.23.0 → geo_activity_playground-0.24.1}/geo_activity_playground/importers/directory.py

@@ -6,6 +6,7 @@ import pickle
 import re
 import sys
 import traceback
+from typing import Any
 from typing import Optional
 
 import pandas as pd
@@ -23,7 +24,9 @@ ACTIVITY_DIR = pathlib.Path("Activities")
 
 
 def import_from_directory(
-    repository: ActivityRepository, metadata_extraction_regexes: list[str] = []
+    repository: ActivityRepository,
+    kind_defaults: dict[str, Any] = {},
+    metadata_extraction_regexes: list[str] = [],
 ) -> None:
     paths_with_errors = []
     work_tracker = WorkTracker("parse-activity-files")
@@ -66,9 +69,11 @@ def import_from_directory(
             path=str(path),
             kind="Unknown",
             equipment="Unknown",
+            consider_for_achievements=True,
         )
         activity_meta.update(activity_meta_from_file)
         activity_meta.update(_get_metadata_from_path(path, metadata_extraction_regexes))
+        activity_meta.update(kind_defaults.get(activity_meta["kind"], {}))
         repository.add_activity(activity_meta)
 
     if paths_with_errors:
@@ -97,7 +102,7 @@ def _cache_single_file(path: pathlib.Path) -> Optional[tuple[pathlib.Path, str]]
         except ActivityParseError as e:
             logger.error(f"Error while parsing file {path}:")
             traceback.print_exc()
-            return (path, str(e))
+            return path, str(e)
         except:
             logger.error(f"Encountered a problem with {path=}, see details below.")
             raise

{geo_activity_playground-0.23.0 → geo_activity_playground-0.24.1}/geo_activity_playground/importers/strava_api.py

@@ -9,6 +9,7 @@ from typing import Any
 
 import pandas as pd
 from stravalib import Client
+from stravalib.exc import Fault
 from stravalib.exc import ObjectNotFound
 from stravalib.exc import RateLimitExceeded
 from tqdm import tqdm
@@ -16,6 +17,7 @@ from tqdm import tqdm
 from geo_activity_playground.core.activities import ActivityRepository
 from geo_activity_playground.core.config import get_config
 from geo_activity_playground.core.paths import cache_dir
+from geo_activity_playground.core.time_conversion import convert_to_datetime_ns
 
 
 logger = logging.getLogger(__name__)
@@ -164,7 +166,7 @@ def try_import_strava(repository: ActivityRepository) -> bool:
                         "distance_km": activity.distance.magnitude / 1000,
                         "name": activity.name,
                         "kind": str(activity.type),
-                        "start": activity.start_date,
+                        "start": convert_to_datetime_ns(activity.start_date),
                         "elapsed_time": activity.elapsed_time,
                         "equipment": gear_names[activity.gear_id],
                         "calories": detailed_activity.calories,
@@ -173,6 +175,11 @@ def try_import_strava(repository: ActivityRepository) -> bool:
         limit_exceeded = False
     except RateLimitExceeded:
         limit_exceeded = True
+    except Fault as e:
+        if "Too Many Requests" in str(e):
+            limit_exceeded = True
+        else:
+            raise
 
     repository.commit()
 

{geo_activity_playground-0.23.0 → geo_activity_playground-0.24.1}/geo_activity_playground/importers/strava_checkout.py

@@ -16,6 +16,7 @@ from geo_activity_playground.core.activities import ActivityRepository
 from geo_activity_playground.core.activity_parsers import ActivityParseError
 from geo_activity_playground.core.activity_parsers import read_activity
 from geo_activity_playground.core.tasks import WorkTracker
+from geo_activity_playground.core.time_conversion import convert_to_datetime_ns
 
 
 logger = logging.getLogger(__name__)
@@ -174,9 +175,9 @@ def import_from_strava_checkout(repository: ActivityRepository) -> None:
             "id": activity_id,
             "name": row["Activity Name"],
             "path": str(activity_file),
-            "start": dateutil.parser.parse(
-                row["Activity Date"], dayfirst=dayfirst
-            ).astimezone(datetime.timezone.utc),
+            "start": convert_to_datetime_ns(
+                dateutil.parser.parse(row["Activity Date"], dayfirst=dayfirst)
+            ),
         }
 
         time_series_path = activity_stream_dir / f"{activity_id}.parquet"