geo-activity-playground 0.19.0__tar.gz → 0.20.0__tar.gz

{geo_activity_playground-0.19.0 → geo_activity_playground-0.20.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: geo-activity-playground
-Version: 0.19.0
+Version: 0.20.0
 Summary: Analysis of geo data activities like rides, runs or hikes.
 License: MIT
 Author: Martin Ueding
@@ -20,6 +20,7 @@ Requires-Dist: fitdecode (>=0.10.0,<0.11.0)
 Requires-Dist: flask (>=3.0.0,<4.0.0)
 Requires-Dist: geojson (>=3.0.1,<4.0.0)
 Requires-Dist: gpxpy (>=1.5.0,<2.0.0)
+Requires-Dist: imagehash (>=4.3.1,<5.0.0)
 Requires-Dist: jinja2 (>=3.1.2,<4.0.0)
 Requires-Dist: matplotlib (>=3.6.3,<4.0.0)
 Requires-Dist: numpy (>=1.22.4,<2.0.0)
@@ -32,7 +33,7 @@ Requires-Dist: stravalib (>=1.3.3,<2.0.0)
 Requires-Dist: tcxreader (>=0.4.5,<0.5.0)
 Requires-Dist: tomli (>=2.0.1,<3.0.0) ; python_version < "3.11"
 Requires-Dist: tqdm (>=4.64.0,<5.0.0)
-Requires-Dist: vegafusion (>=1.4.3,<2.0.0)
 Requires-Dist: vegafusion-python-embed (>=1.4.3,<2.0.0)
+Requires-Dist: vegafusion[embed] (>=1.4.3,<2.0.0)
 Requires-Dist: vl-convert-python (>=1.0.1,<2.0.0)
 Requires-Dist: xmltodict (>=0.13.0,<0.14.0)

{geo_activity_playground-0.19.0 → geo_activity_playground-0.20.0}/geo_activity_playground/__main__.py

@@ -6,6 +6,7 @@ import sys
 
 import coloredlogs
 
+from .core.similarity import precompute_activity_distances
 from .importers.strava_checkout import convert_strava_checkout
 from .importers.strava_checkout import import_from_strava_checkout
 from geo_activity_playground.core.activities import ActivityRepository
@@ -114,6 +115,7 @@ def make_activity_repository(
         import_from_strava_api(repository)
 
     embellish_time_series(repository)
+    precompute_activity_distances(repository)
     compute_tile_visits(repository)
     compute_tile_evolution()
     return repository

{geo_activity_playground-0.19.0 → geo_activity_playground-0.20.0}/geo_activity_playground/core/activities.py

@@ -1,6 +1,7 @@
 import datetime
 import functools
 import logging
+import pathlib
 from typing import Iterator
 from typing import Optional
 from typing import TypedDict
@@ -12,6 +13,7 @@ import pandas as pd
 from tqdm import tqdm
 
 from geo_activity_playground.core.config import get_config
+from geo_activity_playground.core.coordinates import get_distance
 from geo_activity_playground.core.paths import activities_path
 from geo_activity_playground.core.paths import activity_timeseries_path
 from geo_activity_playground.core.tasks import WorkTracker
@@ -25,11 +27,15 @@ class ActivityMeta(TypedDict):
     commute: bool
     distance_km: float
     elapsed_time: datetime.timedelta
+    end_latitude: float
+    end_longitude: float
     equipment: str
     id: int
     kind: str
     name: str
     path: str
+    start_latitude: float
+    start_longitude: float
     start: datetime.datetime
 
 
@@ -43,15 +49,27 @@ class ActivityRepository:
             self.meta = pd.DataFrame()
 
         self._loose_activities: list[ActivityMeta] = []
+        self._loose_activity_ids: set[int] = set()
 
     def __len__(self) -> int:
         return len(self.meta)
 
     def add_activity(self, activity_meta: ActivityMeta) -> None:
-        assert not self.has_activity(
-            activity_meta["id"]
-        ), f"Trying to add the following activity which already exists: {activity_meta}"
+        _extend_metadata_from_timeseries(activity_meta)
+        if activity_meta["id"] in self._loose_activity_ids:
+            logger.error(f"Activity with the same file already exists. New activity:")
+            print(activity_meta)
+            print("Existing activity:")
+            print(
+                [
+                    activity
+                    for activity in self._loose_activities
+                    if activity["id"] == activity_meta["id"]
+                ]
+            )
+            raise ValueError("Activity with the same file already exists.")
         self._loose_activities.append(activity_meta)
+        self._loose_activity_ids.add(activity_meta["id"])
 
     def commit(self) -> None:
         if self._loose_activities:
@@ -59,7 +77,15 @@ class ActivityRepository:
                 f"Adding {len(self._loose_activities)} activities to the repository …"
             )
             new_df = pd.DataFrame(self._loose_activities)
-            self.meta = pd.concat([self.meta, new_df])
+            if len(self.meta):
+                new_ids_set = set(new_df["id"])
+                is_kept = [
+                    activity_id not in new_ids_set for activity_id in self.meta["id"]
+                ]
+                old_df = self.meta.loc[is_kept]
+            else:
+                old_df = self.meta
+            self.meta = pd.concat([old_df, new_df])
             assert pd.api.types.is_dtype_equal(
                 self.meta["start"].dtype, "datetime64[ns, UTC]"
             ), self.meta["start"].dtype
@@ -86,6 +112,8 @@ class ActivityRepository:
     def last_activity_date(self) -> Optional[datetime.datetime]:
         if len(self.meta):
             return self.meta.iloc[-1]["start"]
+        else:
+            return None
 
     @property
     def activity_ids(self) -> set[int]:
@@ -122,55 +150,81 @@ def embellish_time_series(repository: ActivityRepository) -> None:
         path = activity_timeseries_path(activity_id)
         df = pd.read_parquet(path)
         df.name = id
-        changed = False
-        if pd.api.types.is_dtype_equal(df["time"].dtype, "int64"):
-            start = repository.get_activity_by_id(activity_id)["start"]
-            time = df["time"]
-            del df["time"]
-            df["time"] = [start + datetime.timedelta(seconds=t) for t in time]
-            changed = True
-        assert pd.api.types.is_dtype_equal(df["time"].dtype, "datetime64[ns, UTC]")
-
-        if "distance_km" in df.columns:
-            if "speed" not in df.columns:
-                df["speed"] = (
-                    df["distance_km"].diff()
-                    / (df["time"].diff().dt.total_seconds() + 1e-3)
-                    * 3600
-                )
-                changed = True
-
-            potential_jumps = (df["speed"] > 40) & (df["speed"].diff() > 10)
-            if np.any(potential_jumps):
-                df = df.loc[~potential_jumps]
-                changed = True
-
-        if "x" not in df.columns:
-            x, y = compute_tile_float(df["latitude"], df["longitude"], 0)
-            df["x"] = x
-            df["y"] = y
-            changed = True
-
-        if "segment_id" not in df.columns:
-            time_diff = (df["time"] - df["time"].shift(1)).dt.total_seconds()
-            jump_indices = time_diff >= 30
-            df["segment_id"] = np.cumsum(jump_indices)
-            changed = True
-
+        df, changed = embellish_single_time_series(
+            df, repository.get_activity_by_id(activity_id)["start"]
+        )
         if changed:
             df.to_parquet(path)
         work_tracker.mark_done(activity_id)
     work_tracker.close()
 
 
+def embellish_single_time_series(
+    timeseries: pd.DataFrame, start: Optional[datetime.datetime] = None
+) -> bool:
+    changed = False
+    time_diff_threshold_seconds = 30
+    time_diff = (timeseries["time"] - timeseries["time"].shift(1)).dt.total_seconds()
+    jump_indices = time_diff >= time_diff_threshold_seconds
+
+    if start is not None and pd.api.types.is_dtype_equal(
+        timeseries["time"].dtype, "int64"
+    ):
+        time = timeseries["time"]
+        del timeseries["time"]
+        timeseries["time"] = [start + datetime.timedelta(seconds=t) for t in time]
+        changed = True
+    assert pd.api.types.is_dtype_equal(timeseries["time"].dtype, "datetime64[ns, UTC]")
+
+    # Add distance column if missing.
+    if "distance_km" not in timeseries.columns:
+        distances = get_distance(
+            timeseries["latitude"].shift(1),
+            timeseries["longitude"].shift(1),
+            timeseries["latitude"],
+            timeseries["longitude"],
+        ).fillna(0.0)
+        distances.loc[jump_indices] = 0.0
+        timeseries["distance_km"] = pd.Series(np.cumsum(distances)) / 1000
+        changed = True
+
+    if "distance_km" in timeseries.columns:
+        if "speed" not in timeseries.columns:
+            timeseries["speed"] = (
+                timeseries["distance_km"].diff()
+                / (timeseries["time"].diff().dt.total_seconds() + 1e-3)
+                * 3600
+            )
+            changed = True
+
+        potential_jumps = (timeseries["speed"] > 40) & (timeseries["speed"].diff() > 10)
+        if np.any(potential_jumps):
+            timeseries = timeseries.loc[~potential_jumps]
+            changed = True
+
+    if "x" not in timeseries.columns:
+        x, y = compute_tile_float(timeseries["latitude"], timeseries["longitude"], 0)
+        timeseries["x"] = x
+        timeseries["y"] = y
+        changed = True
+
+    if "segment_id" not in timeseries.columns:
+        timeseries["segment_id"] = np.cumsum(jump_indices)
+        changed = True
+
+    return timeseries, changed
+
+
 def make_geojson_from_time_series(time_series: pd.DataFrame) -> str:
-    line = geojson.LineString(
-        [
-            (lon, lat)
-            for lat, lon in zip(time_series["latitude"], time_series["longitude"])
+    fc = geojson.FeatureCollection(
+        features=[
+            geojson.LineString(
+                [(lon, lat) for lat, lon in zip(group["latitude"], group["longitude"])]
+            )
+            for _, group in time_series.groupby("segment_id")
         ]
     )
-    return geojson.dumps(line)
+    return geojson.dumps(fc)
 
 
 def make_geojson_color_line(time_series: pd.DataFrame) -> str:
@@ -188,9 +242,8 @@ def make_geojson_color_line(time_series: pd.DataFrame) -> str:
                 "color": matplotlib.colors.to_hex(cmap(min(next["speed"] / 35, 1.0))),
             },
         )
-        for (_, row), (_, next) in zip(
-            time_series.iterrows(), time_series.iloc[1:].iterrows()
-        )
+        for _, group in time_series.groupby("segment_id")
+        for (_, row), (_, next) in zip(group.iterrows(), group.iloc[1:].iterrows())
     ]
     feature_collection = geojson.FeatureCollection(features)
     return geojson.dumps(feature_collection)
@@ -236,3 +289,14 @@ def extract_heart_rate_zones(time_series: pd.DataFrame) -> Optional[pd.DataFrame
             duration_per_zone.loc[i] = 0.0
     result = duration_per_zone.reset_index()
     return result
+
+
+def _extend_metadata_from_timeseries(metadata: ActivityMeta) -> None:
+    timeseries = pd.read_parquet(
+        pathlib.Path("Cache/Activity Timeseries") / f"{metadata['id']}.parquet"
+    )
+
+    metadata["start_latitude"] = timeseries["latitude"].iloc[0]
+    metadata["end_latitude"] = timeseries["latitude"].iloc[-1]
+    metadata["start_longitude"] = timeseries["longitude"].iloc[0]
+    metadata["end_longitude"] = timeseries["longitude"].iloc[-1]

{geo_activity_playground-0.19.0 → geo_activity_playground-0.20.0}/geo_activity_playground/core/activity_parsers.py

@@ -14,6 +14,7 @@ import tcxreader.tcxreader
 import xmltodict
 
 from geo_activity_playground.core.activities import ActivityMeta
+from geo_activity_playground.core.activities import embellish_single_time_series
 from geo_activity_playground.core.coordinates import get_distance
 
 logger = logging.getLogger(__name__)
@@ -51,7 +52,7 @@ def read_activity(path: pathlib.Path) -> tuple[ActivityMeta, pd.DataFrame]:
     elif file_type in [".kml", ".kmz"]:
         timeseries = read_kml_activity(path, opener)
     elif file_type == ".csv":  # Simra csv export
-        timeseries = read_simra_activity(path)
+        timeseries = read_simra_activity(path, opener)
     else:
         raise ActivityParseError(f"Unsupported file format: {file_type}")
 
@@ -74,18 +75,7 @@ def read_activity(path: pathlib.Path) -> tuple[ActivityMeta, pd.DataFrame]:
                 "It looks like the date parsing has gone wrong."
             ) from e
 
-        # Add distance column if missing.
-        if "distance_km" not in timeseries.columns:
-            distances = [0] + [
-                get_distance(lat_1, lon_1, lat_2, lon_2)
-                for lat_1, lon_1, lat_2, lon_2 in zip(
-                    timeseries["latitude"],
-                    timeseries["longitude"],
-                    timeseries["latitude"].iloc[1:],
-                    timeseries["longitude"].iloc[1:],
-                )
-            ]
-            timeseries["distance_km"] = pd.Series(np.cumsum(distances)) / 1000
+        timeseries, changed = embellish_single_time_series(timeseries)
 
         # Extract some meta data from the time series.
         metadata["start"] = timeseries["time"].iloc[0]
@@ -212,9 +202,13 @@ def read_gpx_activity(path: pathlib.Path, open) -> pd.DataFrame:
                     time = dateutil.parser.parse(str(point.time))
                 assert isinstance(time, datetime.datetime)
                 time = time.astimezone(datetime.timezone.utc)
-                points.append((time, point.latitude, point.longitude))
+                points.append((time, point.latitude, point.longitude, point.elevation))
 
-    return pd.DataFrame(points, columns=["time", "latitude", "longitude"])
+    df = pd.DataFrame(points, columns=["time", "latitude", "longitude", "altitude"])
+    # Some files don't have altitude information. In these cases we remove the column.
+    if not df["altitude"].any():
+        del df["altitude"]
+    return df
 
 
 def read_tcx_activity(path: pathlib.Path, opener) -> pd.DataFrame:

{geo_activity_playground-0.19.0 → geo_activity_playground-0.20.0}/geo_activity_playground/core/cache_migrations.py

@@ -95,6 +95,9 @@ def apply_cache_migrations() -> None:
         delete_activity_metadata,
         delete_activity_metadata,
         convert_distances_to_km,
+        delete_activity_metadata,
+        delete_tile_visits,
+        delete_heatmap_cache,
     ]
 
     for migration in migrations[cache_status["num_applied_migrations"] :]:

{geo_activity_playground-0.19.0 → geo_activity_playground-0.20.0}/geo_activity_playground/core/config.py

@@ -21,7 +21,4 @@ def get_config() -> dict:
     with open(config_path, "rb") as f:
         config = tomllib.load(f)
 
-    # Filter out empty config groups.
-    config = {key: value for key, value in config.items() if value}
-
     return config

{geo_activity_playground-0.19.0 → geo_activity_playground-0.20.0}/geo_activity_playground/core/heatmap.py

@@ -4,7 +4,6 @@ This code is based on https://github.com/remisalmon/Strava-local-heatmap.
 import dataclasses
 import logging
 
-import matplotlib.pyplot as pl
 import numpy as np
 
 from geo_activity_playground.core.tiles import compute_tile_float
@@ -147,32 +146,3 @@ def crop_image_to_bounds(
     max_y = int((max_y - tile_bounds.y_tile_min) * OSM_TILE_SIZE)
     image = image[min_y:max_y, min_x:max_x, :]
     return image
-
-
-def gaussian_filter(image, sigma):
-    # returns image filtered with a gaussian function of variance sigma**2
-    #
-    # input: image = numpy.ndarray
-    #        sigma = float
-    # output: image = numpy.ndarray
-
-    i, j = np.meshgrid(
-        np.arange(image.shape[0]), np.arange(image.shape[1]), indexing="ij"
-    )
-
-    mu = (int(image.shape[0] / 2.0), int(image.shape[1] / 2.0))
-
-    gaussian = (
-        1.0
-        / (2.0 * np.pi * sigma * sigma)
-        * np.exp(-0.5 * (((i - mu[0]) / sigma) ** 2 + ((j - mu[1]) / sigma) ** 2))
-    )
-
-    gaussian = np.roll(gaussian, (-mu[0], -mu[1]), axis=(0, 1))
-
-    image_fft = np.fft.rfft2(image)
-    gaussian_fft = np.fft.rfft2(gaussian)
-
-    image = np.fft.irfft2(image_fft * gaussian_fft)
-
-    return image

geo_activity_playground-0.20.0/geo_activity_playground/core/similarity.py

@@ -0,0 +1,93 @@
+import pathlib
+import pickle
+
+import imagehash
+import numpy as np
+import pandas as pd
+from PIL import Image
+from PIL import ImageDraw
+from tqdm import tqdm
+
+from .activities import ActivityRepository
+from .coordinates import get_distance
+from geo_activity_playground.core.tasks import stored_object
+
+
+fingerprint_path = pathlib.Path("Cache/activity_fingerprints.pickle")
+distances_path = pathlib.Path("Cache/activity_distances.pickle")
+
+
+def add_distance(distances, this, other, distance) -> None:
+    if this not in distances:
+        distances[this] = {}
+    if distance not in distances[this]:
+        distances[this][distance] = set()
+    distances[this][distance].add(other)
+
+
+def precompute_activity_distances(repository: ActivityRepository) -> None:
+    with stored_object(fingerprint_path, {}) as fingerprints, stored_object(
+        distances_path, {}
+    ) as distances:
+        activity_ids = repository.activity_ids
+
+        activity_ids_without_fingerprint = [
+            activity_id
+            for activity_id in activity_ids
+            if activity_id not in fingerprints
+        ]
+        for activity_id in tqdm(
+            activity_ids_without_fingerprint, desc="Compute activity fingerprints"
+        ):
+            ts = repository.get_time_series(activity_id)
+            ts_hash = _compute_image_hash(ts)
+            fingerprints[activity_id] = ts_hash
+
+        for this in tqdm(
+            activity_ids_without_fingerprint, desc="Compute activity distances"
+        ):
+            for other in activity_ids:
+                distance = _hamming_distance(fingerprints[this], fingerprints[other])
+                add_distance(distances, this, other, distance)
+                add_distance(distances, other, this, distance)
+
+
+def asymmetric_activity_overlap(
+    activity: pd.DataFrame, reference: pd.DataFrame
+) -> float:
+    sample = activity.iloc[np.linspace(0, len(activity) - 1, 50, dtype=np.int64)]
+    min_distances = [
+        _get_min_distance(latitude, longitude, reference)
+        for (latitude, longitude) in zip(sample["latitude"], sample["longitude"])
+    ]
+    return sum(distance < 25 for distance in min_distances) / len(min_distances)
+
+
+def _get_min_distance(latitude: float, longitude: float, other: pd.DataFrame) -> float:
+    distances = get_distance(latitude, longitude, other["latitude"], other["longitude"])
+    return np.min(distances)
+
+
+def _compute_image_hash(time_series) -> int:
+    z = 12 + 8
+    x = time_series["x"] * 2**z
+    y = time_series["y"] * 2**z
+    xy_pixels = np.array([x - x.min(), y - y.min()]).T
+    dim = xy_pixels.max(axis=0)
+    # Some activities have bogus data in them which makes them require a huge image. We just skip those outright and return a dummy hash value.
+    if max(dim) > 6000:
+        return 0
+    im = Image.new("L", tuple(map(int, dim)))
+    draw = ImageDraw.Draw(im)
+    pixels = list(map(int, xy_pixels.flatten()))
+    draw.line(pixels, fill=255, width=5)
+    return int(str(imagehash.dhash(im)), 16)
+
+
+def _hamming_distance(a: int, b: int) -> int:
+    diff = a ^ b
+    result = 0
+    while diff:
+        result += diff % 2
+        diff //= 2
+    return result

{geo_activity_playground-0.19.0 → geo_activity_playground-0.20.0}/geo_activity_playground/core/tasks.py

@@ -2,11 +2,35 @@ import contextlib
 import json
 import pathlib
 import pickle
+from collections.abc import Iterable
 from typing import Any
+from typing import Generic
+from typing import Sequence
+from typing import TypeVar
 
 from geo_activity_playground.core.paths import cache_dir
 
 
+T = TypeVar("T")
+
+
+@contextlib.contextmanager
+def stored_object(path: pathlib.Path, default):
+    if path.exists():
+        with open(path, "rb") as f:
+            payload = pickle.load(f)
+    else:
+        payload = default
+
+    yield payload
+
+    temp_location = path.with_suffix(".tmp")
+    with open(temp_location, "wb") as f:
+        pickle.dump(payload, f)
+    path.unlink(missing_ok=True)
+    temp_location.rename(path)
+
+
 def work_tracker_path(name: str) -> pathlib.Path:
     return cache_dir() / f"work-tracker-{name}.pickle"
 
@@ -35,7 +59,7 @@ class WorkTracker:
         else:
             self._done = set()
 
-    def filter(self, ids: list[int]) -> set[int]:
+    def filter(self, ids: Iterable[int]) -> set[int]:
         return set(ids) - self._done
 
     def mark_done(self, id: int) -> None:

{geo_activity_playground-0.19.0 → geo_activity_playground-0.20.0}/geo_activity_playground/explorer/grid_file.py

@@ -1,6 +1,6 @@
 import json
 import logging
-from typing import Iterator
+from collections.abc import Iterable
 from typing import Optional
 
 import geojson
@@ -8,7 +8,6 @@ import gpxpy
 import pandas as pd
 
 from geo_activity_playground.core.coordinates import Bounds
-from geo_activity_playground.core.tiles import adjacent_to
 from geo_activity_playground.core.tiles import get_tile_upper_left_lat_lon
 
 
@@ -66,10 +65,10 @@ def make_explorer_rectangle(
 
 
 def make_grid_points(
-    tile_iterator: Iterator[tuple[int, int]], zoom: int
+    tiles: Iterable[tuple[int, int]], zoom: int
 ) -> list[list[list[float]]]:
     result = []
-    for tile_x, tile_y in tile_iterator:
+    for tile_x, tile_y in tiles:
         tile = [
             get_tile_upper_left_lat_lon(tile_x, tile_y, zoom),
             get_tile_upper_left_lat_lon(tile_x + 1, tile_y, zoom),

{geo_activity_playground-0.19.0 → geo_activity_playground-0.20.0}/geo_activity_playground/explorer/tile_visits.py

@@ -6,6 +6,7 @@ import pathlib
 import pickle
 from typing import Any
 from typing import Iterator
+from typing import Optional
 
 import pandas as pd
 from tqdm import tqdm
@@ -116,10 +117,10 @@ class TileEvolutionState:
         self.square_start = 0
         self.cluster_start = 0
         self.max_square_size = 0
-        self.visited_tiles = set()
+        self.visited_tiles: set[tuple[int, int]] = set()
         self.square_evolution = pd.DataFrame()
-        self.square_x = None
-        self.square_y = None
+        self.square_x: Optional[int] = None
+        self.square_y: Optional[int] = None
 
 
 def compute_tile_evolution() -> None:

{geo_activity_playground-0.19.0 → geo_activity_playground-0.20.0}/geo_activity_playground/importers/directory.py

@@ -1,6 +1,7 @@
 import hashlib
 import logging
 import pathlib
+import pickle
 import sys
 import traceback
 
@@ -22,35 +23,45 @@ def import_from_directory(
     paths_with_errors = []
     work_tracker = WorkTracker("parse-activity-files")
 
-    activity_paths = {
-        int(hashlib.sha3_224(str(path).encode()).hexdigest(), 16) % 2**62: path
+    activity_paths = [
+        path
         for path in pathlib.Path("Activities").rglob("*.*")
         if path.is_file() and path.suffixes and not path.stem.startswith(".")
-    }
-    activities_ids_to_parse = work_tracker.filter(activity_paths.keys())
+    ]
+    new_activity_paths = work_tracker.filter(activity_paths)
 
     activity_stream_dir = pathlib.Path("Cache/Activity Timeseries")
     activity_stream_dir.mkdir(exist_ok=True, parents=True)
-    for activity_id in tqdm(activities_ids_to_parse, desc="Parse activity files"):
-        path = activity_paths[activity_id]
-        try:
-            activity_meta_from_file, timeseries = read_activity(path)
-        except ActivityParseError as e:
-            logger.error(f"Error while parsing file {path}:")
-            traceback.print_exc()
-            paths_with_errors.append((path, str(e)))
-            continue
-        except:
-            logger.error(f"Encountered a problem with {path=}, see details below.")
-            raise
-
-        work_tracker.mark_done(activity_id)
-
-        if len(timeseries) == 0:
-            continue
+    file_metadata_dir = pathlib.Path("Cache/Activity Metadata")
+    file_metadata_dir.mkdir(exist_ok=True, parents=True)
 
+    for path in tqdm(new_activity_paths, desc="Parse activity files"):
+        activity_id = _get_file_hash(path)
         timeseries_path = activity_stream_dir / f"{activity_id}.parquet"
-        timeseries.to_parquet(timeseries_path)
+        file_metadata_path = file_metadata_dir / f"{activity_id}.pickle"
+        work_tracker.mark_done(path)
+
+        if not timeseries_path.exists():
+            try:
+                activity_meta_from_file, timeseries = read_activity(path)
+            except ActivityParseError as e:
+                logger.error(f"Error while parsing file {path}:")
+                traceback.print_exc()
+                paths_with_errors.append((path, str(e)))
+                continue
+            except:
+                logger.error(f"Encountered a problem with {path=}, see details below.")
+                raise
+
+            if len(timeseries) == 0:
+                continue
+
+            timeseries.to_parquet(timeseries_path)
+            with open(file_metadata_path, "wb") as f:
+                pickle.dump(activity_meta_from_file, f)
+        else:
+            with open(file_metadata_path, "rb") as f:
+                activity_meta_from_file = pickle.load(f)
 
         activity_meta = ActivityMeta(
             commute=path.parts[-2] == "Commute",
@@ -82,3 +93,11 @@ def import_from_directory(
     repository.commit()
 
     work_tracker.close()
+
+
+def _get_file_hash(path: pathlib.Path) -> int:
+    file_hash = hashlib.blake2s()
+    with open(path, "rb") as f:
+        while chunk := f.read(8192):
+            file_hash.update(chunk)
+    return int(file_hash.hexdigest(), 16) % 2**62

{geo_activity_playground-0.19.0 → geo_activity_playground-0.20.0}/geo_activity_playground/importers/strava_api.py

@@ -191,7 +191,7 @@ def download_strava_time_series(activity_id: int, client: Client) -> pd.DataFram
         if name in streams:
             columns[name] = streams[name].data
     if "distance" in streams:
-        columns["distance_km"] = streams["distance"].data / 1000
+        columns["distance_km"] = pd.Series(streams["distance"].data) / 1000
 
     df = pd.DataFrame(columns)
     return df