geo-activity-playground 1.2.0__py3-none-any.whl → 1.3.0__py3-none-any.whl

geo_activity_playground/core/time_conversion.py

@@ -1,14 +1,42 @@
-import numpy as np
-import pandas as pd
-
-
-def convert_to_datetime_ns(date) -> np.datetime64 | pd.Series:
-    if isinstance(date, pd.Series):
-        ts = pd.to_datetime(date)
-        ts = ts.dt.tz_localize(None)
-        return ts
-    else:
-        ts = pd.to_datetime(date)
-        if ts.tzinfo is not None:
-            ts = ts.tz_localize(None)
-        return ts.to_datetime64()
+import datetime
+import json
+import logging
+import zoneinfo
+
+import requests
+
+from .paths import USER_CACHE_DIR
+
+logger = logging.getLogger(__name__)
+
+
+def sanitize_datetime(
+    dt: datetime.datetime, fallback_from: str, fallback_to: str
+) -> datetime.datetime:
+    if dt.tzinfo is None:
+        dt = dt.replace(tzinfo=zoneinfo.ZoneInfo(fallback_from))
+    return dt.astimezone(zoneinfo.ZoneInfo(fallback_to))
+
+
+def get_country_timezone(latitude: float, longitude: float) -> tuple[str, str]:
+    cache_file = USER_CACHE_DIR / "geotimezone" / f"{latitude:.5f}-{longitude:.5f}.json"
+    data = {}
+    if cache_file.exists():
+        try:
+            with open(cache_file) as f:
+                data = json.load(f)
+        except json.decoder.JSONDecodeError as e:
+            logger.warning(
+                f"'{cache_file}' could not be parsed ('{e}'). Deleting and trying again."
+            )
+            cache_file.unlink()
+
+    if not cache_file.exists():
+        url = f"https://api.geotimezone.com/public/timezone?latitude={latitude}&longitude={longitude}"
+        r = requests.get(url)
+        r.raise_for_status()
+        data = r.json()
+        cache_file.parent.mkdir(exist_ok=True, parents=True)
+        with open(cache_file, "w") as f:
+            json.dump(data, f)
+    return data["location"], data["iana_timezone"]

geo_activity_playground/explorer/tile_visits.py

@@ -143,7 +143,7 @@ def compute_tile_visits_new(
         work_tracker.reset()
 
     for activity_id in tqdm(
-        work_tracker.filter(repository.get_activity_ids()), desc="Tile visits"
+        work_tracker.filter(repository.get_activity_ids()), desc="Tile visits", delay=2
     ):
         _process_activity(repository, tile_visit_accessor.tile_state, activity_id)
         work_tracker.mark_done(activity_id)
@@ -213,7 +213,6 @@ def _process_activity(
 def _tiles_from_points(
     time_series: pd.DataFrame, zoom: int
 ) -> Iterator[tuple[datetime.datetime, int, int]]:
-    assert pd.api.types.is_dtype_equal(time_series["time"].dtype, "datetime64[ns]")
     xf = time_series["x"] * 2**zoom
     yf = time_series["y"] * 2**zoom
     for t1, x1, y1, x2, y2, s1, s2 in zip(
@@ -257,7 +256,9 @@ def _compute_cluster_evolution(
 
     rows = []
     for index, row in tqdm(
-        tiles.iloc[s.cluster_start :].iterrows(), desc=f"Cluster evolution for {zoom=}"
+        tiles.iloc[s.cluster_start :].iterrows(),
+        desc=f"Cluster evolution for {zoom=}",
+        delay=2,
     ):
         new_clusters = False
         # Current tile.
@@ -334,7 +335,9 @@ def _compute_square_history(
 ) -> None:
     rows = []
     for index, row in tqdm(
-        tiles.iloc[s.square_start :].iterrows(), desc=f"Square evolution for {zoom=}"
+        tiles.iloc[s.square_start :].iterrows(),
+        desc=f"Square evolution for {zoom=}",
+        delay=2,
     ):
         tile = (row["tile_x"], row["tile_y"])
         x, y = tile

geo_activity_playground/importers/activity_parsers.py

@@ -13,8 +13,8 @@ import pandas as pd
 import tcxreader.tcxreader
 import xmltodict
 
-from ..core.datamodel import ActivityMeta
-from ..core.time_conversion import convert_to_datetime_ns
+from ..core.datamodel import Activity
+from ..core.datamodel import get_or_make_kind
 
 logger = logging.getLogger(__name__)
 
@@ -23,9 +23,9 @@ class ActivityParseError(BaseException):
     pass
 
 
-def read_activity(path: pathlib.Path) -> tuple[ActivityMeta, pd.DataFrame]:
+def read_activity(path: pathlib.Path) -> tuple[Activity, pd.DataFrame]:
     suffixes = [s.lower() for s in path.suffixes]
-    metadata = ActivityMeta()
+    activity = Activity()
 
     if suffixes[-1] == ".gz":
         opener = gzip.open
@@ -43,7 +43,7 @@ def read_activity(path: pathlib.Path) -> tuple[ActivityMeta, pd.DataFrame]:
             raise ActivityParseError(f"Encoding issue") from e
     elif file_type == ".fit":
         try:
-            metadata, timeseries = read_fit_activity(path, opener)
+            activity, timeseries = read_fit_activity(path, opener)
         except fitdecode.exceptions.FitError as e:
             raise ActivityParseError(f"Error in FIT file") from e
         except KeyError as e:
@@ -60,10 +60,10 @@ def read_activity(path: pathlib.Path) -> tuple[ActivityMeta, pd.DataFrame]:
     else:
         raise ActivityParseError(f"Unsupported file format: {file_type}")
 
-    return metadata, timeseries
+    return activity, timeseries
 
 
-def read_fit_activity(path: pathlib.Path, open) -> tuple[ActivityMeta, pd.DataFrame]:
+def read_fit_activity(path: pathlib.Path, open) -> tuple[Activity, pd.DataFrame]:
     """
     {'timestamp': datetime.datetime(2023, 11, 11, 16, 29, 49, tzinfo=datetime.timezone.utc),
     'position_lat': <int>,
@@ -82,7 +82,7 @@ def read_fit_activity(path: pathlib.Path, open) -> tuple[ActivityMeta, pd.DataFr
     'ascent': 35,
     'descent': 11}
     """
-    metadata = ActivityMeta()
+    activity = Activity()
     rows = []
     with open(path, "rb") as f:
         with fitdecode.FitReader(f) as fit:
@@ -103,10 +103,9 @@ def read_fit_activity(path: pathlib.Path, open) -> tuple[ActivityMeta, pd.DataFr
                         if isinstance(time, datetime.datetime):
                             pass
                         elif time is None or isinstance(time, int):
-                            time = pd.NaT
+                            time = None
                         else:
                             raise RuntimeError(f"Cannot parse time: {time} in {path}.")
-                        time = convert_to_datetime_ns(time)
                         row = {
                             "time": time,
                             "latitude": values["position_lat"] / ((2**32) / 360),
@@ -114,7 +113,9 @@ def read_fit_activity(path: pathlib.Path, open) -> tuple[ActivityMeta, pd.DataFr
                         }
                         if "heart_rate" in fields:
                             row["heartrate"] = values["heart_rate"]
-                        if "calories" in fields:
+                        if "calories" in fields and isinstance(
+                            values["calories"], float
+                        ):
                             row["calories"] = values["calories"]
                         if "cadence" in fields:
                             row["cadence"] = values["cadence"]
@@ -142,17 +143,18 @@ def read_fit_activity(path: pathlib.Path, open) -> tuple[ActivityMeta, pd.DataFr
 
                     # Additional meta data fields as documented in https://developer.garmin.com/fit/file-types/workout/.
                     if "wkt_name" in fields:
-                        metadata["name"] = values["wkt_name"]
+                        activity.name = values["wkt_name"]
                     if "sport" in fields:
-                        metadata["kind"] = str(values["sport"])
+                        kind_name = str(values["sport"])
                         if "sub_sport" in values:
-                            metadata["kind"] += " " + str(values["sub_sport"])
+                            kind_name += " " + str(values["sub_sport"])
+                        activity.kind = get_or_make_kind(kind_name)
                     if "total_calories" in fields:
-                        metadata["calories"] = values["total_calories"]
+                        activity.calories = values["total_calories"]
                     if "total_strides" in fields:
-                        metadata["steps"] = 2 * int(values["total_strides"])
+                        activity.steps = 2 * int(values["total_strides"])
 
-    return metadata, pd.DataFrame(rows)
+    return activity, pd.DataFrame(rows)
 
 
 def _fit_speed_unit_factor(unit: str) -> float:
@@ -184,8 +186,7 @@ def read_gpx_activity(path: pathlib.Path, open) -> pd.DataFrame:
                 elif isinstance(point.time, str):
                     time = dateutil.parser.parse(str(point.time))
                 else:
-                    time = pd.NaT
-                time = convert_to_datetime_ns(time)
+                    time = None
                 points.append((time, point.latitude, point.longitude, point.elevation))
 
     df = pd.DataFrame(points, columns=["time", "latitude", "longitude", "elevation"])
@@ -223,7 +224,6 @@ def read_tcx_activity(path: pathlib.Path, opener) -> pd.DataFrame:
         if trackpoint.latitude and trackpoint.longitude:
             time = trackpoint.time
             assert isinstance(time, datetime.datetime)
-            time = convert_to_datetime_ns(time)
             row = {
                 "time": time,
                 "latitude": trackpoint.latitude,
@@ -252,7 +252,6 @@ def read_kml_activity(path: pathlib.Path, opener) -> pd.DataFrame:
             for track in _list_or_scalar(placemark.get("gx:Track", [])):
                 for when, where in zip(track["when"], track["gx:coord"]):
                     time = dateutil.parser.parse(when)
-                    time = convert_to_datetime_ns(time)
                     parts = where.split(" ")
                     if len(parts) == 2:
                         lon, lat = parts
@@ -282,7 +281,7 @@ def read_simra_activity(path: pathlib.Path, opener) -> pd.DataFrame:
     data["time"] = data["timeStamp"].apply(
         lambda d: datetime.datetime.fromtimestamp(d / 1000)
     )
-    data["time"] = convert_to_datetime_ns(data["time"])
+    data["time"] = data["time"]
     data = data.rename(columns={"lat": "latitude", "lon": "longitude"})
     return data.dropna(subset=["latitude"], ignore_index=True)[
         ["time", "latitude", "longitude"]

geo_activity_playground/importers/directory.py

@@ -1,21 +1,21 @@
-import hashlib
 import logging
 import pathlib
-import pickle
 import re
 import traceback
-from typing import Optional
 
-from tqdm import tqdm
+import sqlalchemy
 
+from ..core.activities import ActivityRepository
 from ..core.config import Config
-from ..core.datamodel import ActivityMeta
+from ..core.datamodel import Activity
+from ..core.datamodel import DB
 from ..core.datamodel import DEFAULT_UNKNOWN_NAME
-from ..core.paths import activity_extracted_dir
-from ..core.paths import activity_extracted_meta_dir
-from ..core.paths import activity_extracted_time_series_dir
-from ..core.tasks import stored_object
-from ..core.tasks import WorkTracker
+from ..core.datamodel import get_or_make_equipment
+from ..core.datamodel import get_or_make_kind
+from ..core.enrichment import update_and_commit
+from ..explorer.tile_visits import compute_tile_evolution
+from ..explorer.tile_visits import compute_tile_visits_new
+from ..explorer.tile_visits import TileVisitAccessor
 from .activity_parsers import ActivityParseError
 from .activity_parsers import read_activity
 
@@ -25,9 +25,10 @@ ACTIVITY_DIR = pathlib.Path("Activities")
 
 
 def import_from_directory(
-    metadata_extraction_regexes: list[str], config: Config
+    repository: ActivityRepository,
+    tile_visit_accessor: TileVisitAccessor,
+    config: Config,
 ) -> None:
-
     activity_paths = [
         path
         for path in ACTIVITY_DIR.rglob("*.*")
@@ -36,105 +37,58 @@ def import_from_directory(
         and not path.stem.startswith(".")
         and not path.suffix in config.ignore_suffixes
     ]
-    work_tracker = WorkTracker(activity_extracted_dir() / "work-tracker-extract.pickle")
-    new_activity_paths = work_tracker.filter(activity_paths)
-
-    with stored_object(
-        activity_extracted_dir() / "file-hashes.pickle", {}
-    ) as file_hashes:
-        for path in tqdm(new_activity_paths, desc="Detect deleted activities"):
-            file_hashes[path] = get_file_hash(path)
-
-        deleted_files = set(file_hashes.keys()) - set(activity_paths)
-        deleted_hashes = [file_hashes[path] for path in deleted_files]
-        for deleted_hash in deleted_hashes:
-            activity_extracted_meta_path = (
-                activity_extracted_meta_dir() / f"{deleted_hash}.pickle"
-            )
-            activity_extracted_time_series_path = (
-                activity_extracted_time_series_dir() / f"{deleted_hash}.parquet"
+
+    for activity_path in activity_paths:
+        with DB.session.no_autoflush:
+            activity = DB.session.scalar(
+                sqlalchemy.select(Activity).filter(Activity.path == str(activity_path))
             )
-            logger.warning(f"Deleting {activity_extracted_meta_path}")
-            logger.warning(f"Deleting {activity_extracted_time_series_path}")
-            activity_extracted_meta_path.unlink(missing_ok=True)
-            activity_extracted_time_series_path.unlink(missing_ok=True)
-        for deleted_file in deleted_files:
-            logger.warning(f"Deleting {deleted_file}")
-            del file_hashes[deleted_file]
-            work_tracker.discard(deleted_file)
-
-    paths_with_errors = []
-    for path in tqdm(new_activity_paths, desc="Parse activity metadata (serially)"):
-        errors = _cache_single_file(path)
-        if errors:
-            paths_with_errors.append(errors)
-
-    for path in tqdm(new_activity_paths, desc="Collate activity metadata"):
-        activity_id = get_file_hash(path)
-        file_metadata_path = activity_extracted_meta_dir() / f"{activity_id}.pickle"
-        work_tracker.mark_done(path)
-
-        if not file_metadata_path.exists():
-            continue
-
-        with open(file_metadata_path, "rb") as f:
-            activity_meta_from_file = pickle.load(f)
-
-        activity_meta = ActivityMeta(
-            id=activity_id,
-            # https://stackoverflow.com/a/74718395/653152
-            name=path.name.removesuffix("".join(path.suffixes)),
-            path=str(path),
-            kind=DEFAULT_UNKNOWN_NAME,
-            equipment=DEFAULT_UNKNOWN_NAME,
-            consider_for_achievements=True,
+            if activity is None:
+                import_from_file(activity_path, repository, tile_visit_accessor, config)
+
+
+def import_from_file(
+    path: pathlib.Path,
+    repository: ActivityRepository,
+    tile_visit_accessor: TileVisitAccessor,
+    config: Config,
+) -> None:
+    logger.info(f"Importing {path} …")
+    try:
+        activity, time_series = read_activity(path)
+    except ActivityParseError as e:
+        logger.error(f"Error while parsing file {path}:")
+        traceback.print_exc()
+        return
+    except:
+        logger.error(f"Encountered a problem with {path=}, see details below.")
+        raise
+
+    if len(time_series) == 0:
+        logger.warning(f"Activity with {path=} has no time series data, skipping.")
+        return
+
+    activity.path = str(path)
+    if activity.name is None:
+        activity.name = path.name.removesuffix("".join(path.suffixes))
+
+    meta_from_path = _get_metadata_from_path(path, config.metadata_extraction_regexes)
+    activity.name = meta_from_path.get("name", activity.name)
+    if activity.equipment is None:
+        activity.equipment = get_or_make_equipment(
+            meta_from_path.get("equipment", DEFAULT_UNKNOWN_NAME), config
         )
-        activity_meta.update(activity_meta_from_file)
-        activity_meta.update(_get_metadata_from_path(path, metadata_extraction_regexes))
-        with open(file_metadata_path, "wb") as f:
-            pickle.dump(activity_meta, f)
-
-    if paths_with_errors:
-        logger.warning(
-            "There were errors while parsing some of the files. These were skipped and tried again next time."
+    if activity.kind is None:
+        activity.kind = get_or_make_kind(
+            meta_from_path.get("kind", DEFAULT_UNKNOWN_NAME)
         )
-        for path, error in paths_with_errors:
-            logger.error(f"{path}: {error}")
-
-    work_tracker.close()
-
-
-def _cache_single_file(path: pathlib.Path) -> Optional[tuple[pathlib.Path, str]]:
-    activity_id = get_file_hash(path)
-    timeseries_path = activity_extracted_time_series_dir() / f"{activity_id}.parquet"
-    file_metadata_path = activity_extracted_meta_dir() / f"{activity_id}.pickle"
-
-    if not timeseries_path.exists():
-        try:
-            activity_meta_from_file, timeseries = read_activity(path)
-        except ActivityParseError as e:
-            logger.error(f"Error while parsing file {path}:")
-            traceback.print_exc()
-            return path, str(e)
-        except:
-            logger.error(f"Encountered a problem with {path=}, see details below.")
-            raise
-
-        if len(timeseries) == 0:
-            return None
-
-        timeseries.to_parquet(timeseries_path)
-        with open(file_metadata_path, "wb") as f:
-            pickle.dump(activity_meta_from_file, f)
-    return None
-
-
-def get_file_hash(path: pathlib.Path) -> int:
-    file_hash = hashlib.blake2s()
-    with open(path, "rb") as f:
-        while chunk := f.read(8192):
-            file_hash.update(chunk)
-    return int(file_hash.hexdigest(), 16) % 2**62
+
+    update_and_commit(activity, time_series, config)
+
+    if len(repository) > 0:
+        compute_tile_visits_new(repository, tile_visit_accessor)
+        compute_tile_evolution(tile_visit_accessor.tile_state, config)
+        tile_visit_accessor.save()
 
 
 def _get_metadata_from_path(

geo_activity_playground/importers/strava_api.py

@@ -3,6 +3,7 @@ import logging
 import pathlib
 import pickle
 import time
+import zoneinfo
 
 import pandas as pd
 from stravalib import Client
@@ -11,15 +12,20 @@ from stravalib.exc import ObjectNotFound
 from stravalib.exc import RateLimitExceeded
 from tqdm import tqdm
 
+from ..core.activities import ActivityRepository
 from ..core.config import Config
-from ..core.datamodel import ActivityMeta
-from ..core.paths import activity_extracted_meta_dir
+from ..core.datamodel import Activity
+from ..core.datamodel import DB
+from ..core.datamodel import get_or_make_equipment
+from ..core.datamodel import get_or_make_kind
+from ..core.enrichment import apply_enrichments
+from ..core.enrichment import update_and_commit
 from ..core.paths import activity_extracted_time_series_dir
 from ..core.paths import strava_api_dir
 from ..core.paths import strava_last_activity_date_path
 from ..core.tasks import get_state
 from ..core.tasks import set_state
-from ..core.time_conversion import convert_to_datetime_ns
+from ..explorer.tile_visits import TileVisitAccessor
 
 
 logger = logging.getLogger(__name__)
@@ -68,8 +74,12 @@ def round_to_next_quarter_hour(date: datetime.datetime) -> datetime.datetime:
     return next_quarter
 
 
-def import_from_strava_api(config: Config) -> None:
-    while try_import_strava(config):
+def import_from_strava_api(
+    config: Config,
+    repository: ActivityRepository,
+    tile_visit_accessor: TileVisitAccessor,
+) -> None:
+    while try_import_strava(config, repository, tile_visit_accessor):
         now = datetime.datetime.now()
         next_quarter = round_to_next_quarter_hour(now)
         seconds_to_wait = (next_quarter - now).total_seconds() + 10
@@ -79,7 +89,11 @@ def import_from_strava_api(config: Config) -> None:
         time.sleep(seconds_to_wait)
 
 
-def try_import_strava(config: Config) -> bool:
+def try_import_strava(
+    config: Config,
+    repository: ActivityRepository,
+    tile_visit_accessor: TileVisitAccessor,
+) -> bool:
     get_after = get_state(strava_last_activity_date_path(), "2000-01-01T00:00:00Z")
 
     gear_names = {None: "None"}
@@ -87,74 +101,77 @@ def try_import_strava(config: Config) -> bool:
     client = Client(access_token=get_current_access_token(config))
 
     try:
-        for activity in tqdm(
+        for strava_activity in tqdm(
             client.get_activities(after=get_after), desc="Downloading Strava activities"
         ):
             cache_file = (
                 pathlib.Path("Cache")
                 / "Strava Activity Metadata"
-                / f"{activity.id}.pickle"
+                / f"{strava_activity.id}.pickle"
             )
             # Sometimes we still get an activity here although it has already been imported from the Strava checkout.
             if cache_file.exists():
                 continue
             cache_file.parent.mkdir(exist_ok=True, parents=True)
             with open(cache_file, "wb") as f:
-                pickle.dump(activity, f)
-            if activity.gear_id not in gear_names:
-                gear = client.get_gear(activity.gear_id)
-                gear_names[activity.gear_id] = (
+                pickle.dump(strava_activity, f)
+            if strava_activity.gear_id not in gear_names:
+                gear = client.get_gear(strava_activity.gear_id)
+                gear_names[strava_activity.gear_id] = (
                     f"{gear.name}" or f"{gear.brand_name} {gear.model_name}"
                 )
 
             time_series_path = (
-                activity_extracted_time_series_dir() / f"{activity.id}.parquet"
+                activity_extracted_time_series_dir() / f"{strava_activity.id}.parquet"
             )
             if time_series_path.exists():
                 time_series = pd.read_parquet(time_series_path)
             else:
                 try:
-                    time_series = download_strava_time_series(activity.id, client)
+                    time_series = download_strava_time_series(
+                        strava_activity.id, client
+                    )
                 except ObjectNotFound as e:
                     logger.error(
-                        f"The activity {activity.id} with name “{activity.name}” cannot be found."
+                        f"The activity {strava_activity.id} with name “{strava_activity.name}” cannot be found."
                         f"Perhaps it is a manual activity without a time series. Ignoring. {e=}"
                     )
                     continue
-                time_series.name = activity.id
+                time_series.name = strava_activity.id
                 new_time = [
-                    activity.start_date + datetime.timedelta(seconds=time)
+                    strava_activity.start_date + datetime.timedelta(seconds=time)
                     for time in time_series["time"]
                 ]
                 del time_series["time"]
                 time_series["time"] = new_time
                 time_series.to_parquet(time_series_path)
 
-            detailed_activity = get_detailed_activity(activity.id, client)
+            detailed_activity = get_detailed_activity(strava_activity.id, client)
 
             if len(time_series) > 0 and "latitude" in time_series.columns:
-                activity_meta = ActivityMeta(
-                    **{
-                        "id": activity.id,
-                        "commute": activity.commute,
-                        "distance_km": activity.distance / 1000,
-                        "name": activity.name,
-                        "kind": str(activity.type.root),
-                        "start": convert_to_datetime_ns(activity.start_date),
-                        "elapsed_time": activity.elapsed_time,
-                        "equipment": gear_names[activity.gear_id],
-                        "calories": detailed_activity.calories,
-                        "moving_time": activity.moving_time,
-                    }
+                activity = Activity()
+                activity.upstream_id = str(strava_activity.id)
+                activity.distance_km = strava_activity.distance / 1000
+                activity.name = strava_activity.name
+                activity.kind = get_or_make_kind(str(strava_activity.type.root))
+                activity.start = strava_activity.start_date.astimezone(
+                    zoneinfo.ZoneInfo("UTC")
+                )
+                activity.elapsed_time = strava_activity.elapsed_time
+                activity.equipment = get_or_make_equipment(
+                    gear_names[strava_activity.gear_id], config
                 )
-                with open(
-                    activity_extracted_meta_dir() / f"{activity.id}.pickle", "wb"
-                ) as f:
-                    pickle.dump(activity_meta, f)
+                activity.calories = detailed_activity.calories
+                activity.moving_time = detailed_activity.moving_time
+
+                update_and_commit(activity, time_series, config)
+                compute_tile_visits_new(repository, tile_visit_accessor)
+                compute_tile_evolution(tile_visit_accessor.tile_state, config)
+                tile_visit_accessor.save()
 
             set_state(
                 strava_last_activity_date_path(),
-                activity.start_date.isoformat().replace("+00:00", "Z"),
+                strava_activity.start_date.isoformat().replace("+00:00", "Z"),
             )
 
         limit_exceeded = False