geo-activity-playground 1.2.0__py3-none-any.whl → 1.3.1__py3-none-any.whl

geo_activity_playground/core/enrichment.py

@@ -1,212 +1,277 @@
 import datetime
 import logging
-import pickle
-from typing import Optional
+import uuid
+import zoneinfo
+from typing import Callable
 
 import numpy as np
 import pandas as pd
-import sqlalchemy
-from tqdm import tqdm
 
 from .config import Config
 from .coordinates import get_distance
+from .copernicus_dem import get_elevation
 from .datamodel import Activity
-from .datamodel import ActivityMeta
 from .datamodel import DB
-from .datamodel import get_or_make_equipment
-from .datamodel import get_or_make_kind
 from .missing_values import some
-from .paths import activity_extracted_meta_dir
-from .paths import activity_extracted_time_series_dir
-from .paths import TIME_SERIES_DIR
 from .tiles import compute_tile_float
-from .time_conversion import convert_to_datetime_ns
+from .time_conversion import get_country_timezone
 
 logger = logging.getLogger(__name__)
 
 
-def populate_database_from_extracted(config: Config) -> None:
-    available_ids = {
-        int(path.stem) for path in activity_extracted_meta_dir().glob("*.pickle")
-    }
-    present_ids = {
-        int(elem)
-        for elem in DB.session.scalars(sqlalchemy.select(Activity.upstream_id)).all()
-        if elem
-    }
-    new_ids = available_ids - present_ids
-
-    for upstream_id in tqdm(new_ids, desc="Importing new activities into database"):
-        extracted_metadata_path = (
-            activity_extracted_meta_dir() / f"{upstream_id}.pickle"
-        )
-        with open(extracted_metadata_path, "rb") as f:
-            extracted_metadata: ActivityMeta = pickle.load(f)
+def enrichment_set_timezone(
+    activity: Activity, time_series: pd.DataFrame, config: Config
+) -> bool:
+    assert (
+        len(time_series) > 0
+    ), f"You cannot import an activity without points. {activity=}"
+    latitude, longitude = time_series[["latitude", "longitude"]].iloc[0].to_list()
+    if activity.iana_timezone is None or activity.start_country is None:
+        country, tz_str = get_country_timezone(latitude, longitude)
+        activity.iana_timezone = tz_str
+        activity.start_country = country
+        return True
+    else:
+        return False
 
-        extracted_time_series_path = (
-            activity_extracted_time_series_dir() / f"{upstream_id}.parquet"
-        )
-        time_series = pd.read_parquet(extracted_time_series_path)
-
-        # Skip activities that don't have geo information attached to them. This shouldn't happen, though.
-        if "latitude" not in time_series.columns:
-            logger.warning(
-                f"Activity {upstream_id} doesn't have latitude/longitude information. Ignoring this one."
-            )
-            continue
-
-        time_series = _embellish_single_time_series(
-            time_series,
-            extracted_metadata.get("start", None),
-            config.time_diff_threshold_seconds,
-        )
 
-        kind_name = extracted_metadata.get("kind", None)
-        if kind_name:
-            # Rename kinds if needed.
-            if kind_name in config.kind_renames:
-                kind_name = config.kind_renames[kind_name]
-            kind = get_or_make_kind(kind_name, config)
-        else:
-            kind = None
+def enrichment_normalize_time(
+    activity: Activity, time_series: pd.DataFrame, config: Config
+) -> bool:
+    # Routes (as opposed to tracks) don't have time information. We cannot do anything with time here.
+    if (
+        "time" in time_series.columns
+        and pd.isna(time_series["time"]).all()
+        and not pd.api.types.is_datetime64_any_dtype(time_series["time"].dtype)
+    ):
+        time_series["time"] = pd.NaT
+        return True
 
-        equipment_name = extracted_metadata.get("equipment", None)
-        if equipment_name:
-            equipment = get_or_make_equipment(equipment_name, config)
-        elif kind:
-            equipment = kind.default_equipment
-        else:
-            equipment = None
-
-        activity = Activity(
-            name=extracted_metadata.get("name", "Name Placeholder"),
-            distance_km=0,
-            equipment=equipment,
-            kind=kind,
-            calories=some(extracted_metadata.get("calories", None)),
-            elevation_gain=some(extracted_metadata.get("elevation_gain", None)),
-            steps=some(extracted_metadata.get("steps", None)),
-            path=extracted_metadata.get("path", None),
-            upstream_id=upstream_id,
-        )
+    changed = False
+    tz_utc = zoneinfo.ZoneInfo("UTC")
+    # If the time is naive, assume that it is UTC.
+    if time_series["time"].dt.tz is None:
+        time_series["time"] = time_series["time"].dt.tz_localize(tz_utc)
+        changed = True
+
+    if time_series["time"].dt.tz.utcoffset(None) != tz_utc.utcoffset(None):
+        time_series["time"] = time_series["time"].dt.tz_convert(tz_utc)
+        changed = True
 
-        update_via_time_series(activity, time_series)
+    if not pd.api.types.is_dtype_equal(
+        time_series["time"].dtype, "datetime64[ns, UTC]"
+    ):
+        time_series["time"] = time_series["time"].dt.tz_convert(tz_utc)
+        changed = True
 
-        DB.session.add(activity)
-        try:
-            DB.session.commit()
-        except sqlalchemy.exc.StatementError:
-            logger.error(
-                f"Could not insert the following activity into the database: {vars(activity)=}"
-            )
-            raise
+    assert pd.api.types.is_dtype_equal(
+        time_series["time"].dtype, "datetime64[ns, UTC]"
+    ), (
+        time_series["time"].dtype,
+        time_series["time"].iloc[0],
+    )
 
-        enriched_time_series_path = TIME_SERIES_DIR() / f"{activity.id}.parquet"
-        time_series.to_parquet(enriched_time_series_path)
+    new_start = some(time_series["time"].iloc[0])
+    if new_start != activity.start:
+        activity.start = new_start
+        changed = True
 
+    new_elapsed_time = some(time_series["time"].iloc[-1] - time_series["time"].iloc[0])
+    if new_elapsed_time != activity.elapsed_time:
+        activity.elapsed_time = new_elapsed_time
+        changed = True
 
-def update_via_time_series(
-    activity: Activity, time_series: pd.DataFrame
-) -> ActivityMeta:
-    activity.start = some(time_series["time"].iloc[0])
-    activity.elapsed_time = some(
-        time_series["time"].iloc[-1] - time_series["time"].iloc[0]
-    )
-    activity.distance_km = (
-        time_series["distance_km"].iloc[-1] - time_series["distance_km"].iloc[0]
-    )
-    if "calories" in time_series.columns:
-        activity.calories = (
-            time_series["calories"].iloc[-1] - time_series["calories"].iloc[0]
+    return changed
+
+
+def enrichment_rename_altitude(
+    activity: Activity, time_series: pd.DataFrame, config: Config
+) -> bool:
+    if "altitude" in time_series.columns:
+        time_series.rename(columns={"altitude": "elevation"}, inplace=True)
+        return True
+    else:
+        return False
+
+
+def enrichment_compute_tile_xy(
+    activity: Activity, time_series: pd.DataFrame, config: Config
+) -> bool:
+    if "x" not in time_series.columns:
+        x, y = compute_tile_float(time_series["latitude"], time_series["longitude"], 0)
+        time_series["x"] = x
+        time_series["y"] = y
+        return True
+    else:
+        return False
+
+
+def enrichment_copernicus_elevation(
+    activity: Activity, time_series: pd.DataFrame, config: Config
+) -> bool:
+    if "copernicus_elevation" not in time_series.columns:
+        time_series["copernicus_elevation"] = [
+            get_elevation(lat, lon)
+            for lat, lon in zip(time_series["latitude"], time_series["longitude"])
+        ]
+        return True
+    else:
+        return False
+
+
+def enrichment_elevation_gain(
+    activity: Activity, time_series: pd.DataFrame, config: Config
+) -> bool:
+    if (
+        "elevation" in time_series.columns
+        or "copernicus_elevation" in time_series.columns
+    ) and "elevation_gain_cum" not in time_series.columns:
+        elevation = (
+            time_series["elevation"]
+            if "elevation" in time_series.columns
+            else time_series["copernicus_elevation"]
         )
-    activity.moving_time = _compute_moving_time(time_series)
-
-    activity.start_latitude = time_series["latitude"].iloc[0]
-    activity.end_latitude = time_series["latitude"].iloc[-1]
-    activity.start_longitude = time_series["longitude"].iloc[0]
-    activity.end_longitude = time_series["longitude"].iloc[-1]
-    if "elevation_gain_cum" in time_series.columns:
-        elevation_gain_cum = time_series["elevation_gain_cum"].fillna(0)
+        elevation_diff = elevation.diff()
+        elevation_diff = elevation_diff.ewm(span=5, min_periods=5).mean()
+        elevation_diff.loc[elevation_diff.abs() > 30] = 0
+        elevation_diff.loc[elevation_diff < 0] = 0
+        time_series["elevation_gain_cum"] = elevation_diff.cumsum().fillna(0)
+
         activity.elevation_gain = (
-            elevation_gain_cum.iloc[-1] - elevation_gain_cum.iloc[0]
+            time_series["elevation_gain_cum"].iloc[-1]
+            - time_series["elevation_gain_cum"].iloc[0]
         )
+        return True
+    else:
+        return False
 
 
-def _compute_moving_time(time_series: pd.DataFrame) -> datetime.timedelta:
-    def moving_time(group) -> datetime.timedelta:
-        selection = group["speed"] > 1.0
-        time_diff = group["time"].diff().loc[selection]
-        return time_diff.sum()
-
-    return (
-        time_series.groupby("segment_id").apply(moving_time, include_groups=False).sum()
-    )
+def enrichment_add_calories(
+    activity: Activity, time_series: pd.DataFrame, config: Config
+) -> bool:
+    if activity.calories is None and "calories" in time_series.columns:
+        activity.calories = (
+            time_series["calories"].iloc[-1] - time_series["calories"].iloc[0]
+        )
+        return True
+    else:
+        return False
 
 
-def _embellish_single_time_series(
-    timeseries: pd.DataFrame,
-    start: Optional[datetime.datetime],
-    time_diff_threshold_seconds: int,
-) -> pd.DataFrame:
-    if start is not None and pd.api.types.is_dtype_equal(
-        timeseries["time"].dtype, "int64"
-    ):
-        time = timeseries["time"]
-        del timeseries["time"]
-        timeseries["time"] = [
-            convert_to_datetime_ns(start + datetime.timedelta(seconds=t)) for t in time
-        ]
-    timeseries["time"] = convert_to_datetime_ns(timeseries["time"])
-    assert pd.api.types.is_dtype_equal(timeseries["time"].dtype, "datetime64[ns]"), (
-        timeseries["time"].dtype,
-        timeseries["time"].iloc[0],
-    )
+def enrichment_distance(
+    activity: Activity, time_series: pd.DataFrame, config: Config
+) -> bool:
+    changed = False
 
     distances = get_distance(
-        timeseries["latitude"].shift(1),
-        timeseries["longitude"].shift(1),
-        timeseries["latitude"],
-        timeseries["longitude"],
+        time_series["latitude"].shift(1),
+        time_series["longitude"].shift(1),
+        time_series["latitude"],
+        time_series["longitude"],
     ).fillna(0.0)
-    if time_diff_threshold_seconds:
+
+    if config.time_diff_threshold_seconds:
         time_diff = (
-            timeseries["time"] - timeseries["time"].shift(1)
+            time_series["time"] - time_series["time"].shift(1)
         ).dt.total_seconds()
-        jump_indices = time_diff >= time_diff_threshold_seconds
+        jump_indices = time_diff >= config.time_diff_threshold_seconds
         distances.loc[jump_indices] = 0.0
 
-    if "distance_km" not in timeseries.columns:
-        timeseries["distance_km"] = pd.Series(np.cumsum(distances)) / 1000
+    if "distance_km" not in time_series.columns:
+        time_series["distance_km"] = pd.Series(np.cumsum(distances)) / 1000
+        changed = True
 
-    if "speed" not in timeseries.columns:
-        timeseries["speed"] = (
-            timeseries["distance_km"].diff()
-            / (timeseries["time"].diff().dt.total_seconds() + 1e-3)
+    if "speed" not in time_series.columns:
+        time_series["speed"] = (
+            time_series["distance_km"].diff()
+            / (time_series["time"].diff().dt.total_seconds() + 1e-3)
             * 3600
         )
+        changed = True
 
-    potential_jumps = (timeseries["speed"] > 40) & (timeseries["speed"].diff() > 10)
+    potential_jumps = (time_series["speed"] > 40) & (time_series["speed"].diff() > 10)
     if np.any(potential_jumps):
-        timeseries = timeseries.loc[~potential_jumps].copy()
+        time_series.replace(time_series.loc[~potential_jumps])
+        changed = True
 
-    if "segment_id" not in timeseries.columns:
-        if time_diff_threshold_seconds:
-            timeseries["segment_id"] = np.cumsum(jump_indices)
+    if "segment_id" not in time_series.columns:
+        if config.time_diff_threshold_seconds:
+            time_series["segment_id"] = np.cumsum(jump_indices)
         else:
-            timeseries["segment_id"] = 0
+            time_series["segment_id"] = 0
+        changed = True
+
+    new_distance_km = (
+        time_series["distance_km"].iloc[-1] - time_series["distance_km"].iloc[0]
+    )
+    if new_distance_km != activity.distance_km:
+        activity.distance_km = new_distance_km
+        changed = True
 
-    if "x" not in timeseries.columns:
-        x, y = compute_tile_float(timeseries["latitude"], timeseries["longitude"], 0)
-        timeseries["x"] = x
-        timeseries["y"] = y
+    return changed
+
+
+def enrichment_moving_time(
+    activity: Activity, time_series: pd.DataFrame, config: Config
+) -> bool:
+    def moving_time(group) -> datetime.timedelta:
+        selection = group["speed"] > 1.0
+        time_diff = group["time"].diff().loc[selection]
+        return time_diff.sum()
+
+    new_moving_time = (
+        time_series.groupby("segment_id").apply(moving_time, include_groups=False).sum()
+    )
+    if new_moving_time != activity.moving_time:
+        activity.moving_time = new_moving_time
+        return True
+    else:
+        return False
+
+
+def enrichment_copy_latlon(
+    activity: Activity, time_series: pd.DataFrame, config: Config
+) -> bool:
+    if activity.start_latitude is None:
+        activity.start_latitude = time_series["latitude"].iloc[0]
+        activity.end_latitude = time_series["latitude"].iloc[-1]
+        activity.start_longitude = time_series["longitude"].iloc[0]
+        activity.end_longitude = time_series["longitude"].iloc[-1]
+        return True
+    else:
+        return False
+
+
+enrichments: list[Callable[[Activity, pd.DataFrame, Config], bool]] = [
+    enrichment_set_timezone,
+    enrichment_normalize_time,
+    enrichment_rename_altitude,
+    enrichment_compute_tile_xy,
+    enrichment_copernicus_elevation,
+    enrichment_elevation_gain,
+    enrichment_add_calories,
+    enrichment_distance,
+    enrichment_moving_time,
+    enrichment_copy_latlon,
+]
+
+
+def apply_enrichments(
+    activity: Activity, time_series: pd.DataFrame, config: Config
+) -> bool:
+    was_changed = False
+    for enrichment in enrichments:
+        was_changed |= enrichment(activity, time_series, config)
+    return was_changed
 
-    if "altitude" in timeseries.columns:
-        timeseries.rename(columns={"altitude": "elevation"}, inplace=True)
-    if "elevation" in timeseries.columns:
-        elevation_diff = timeseries["elevation"].diff()
-        elevation_diff = elevation_diff.ewm(span=5, min_periods=5).mean()
-        elevation_diff.loc[elevation_diff.abs() > 30] = 0
-        elevation_diff.loc[elevation_diff < 0] = 0
-        timeseries["elevation_gain_cum"] = elevation_diff.cumsum()
 
-    return timeseries
+def update_and_commit(
+    activity: Activity, time_series: pd.DataFrame, config: Config
+) -> None:
+    changed = apply_enrichments(activity, time_series, config)
+    if not activity.time_series_uuid:
+        activity.time_series_uuid = str(uuid.uuid4())
+    if changed:
+        activity.replace_time_series(time_series)
+    DB.session.add(activity)
+    DB.session.commit()

geo_activity_playground/core/paths.py

@@ -3,10 +3,18 @@ import functools
 import pathlib
 import typing
 
+import appdirs
+
 """
 Paths within the playground and cache.
 """
 
+APPDIRS = appdirs.AppDirs(appname="Geo Activity Playground", appauthor="Martin Ueding")
+
+USER_CACHE_DIR = pathlib.Path(APPDIRS.user_cache_dir)
+USER_CONFIG_DIR = pathlib.Path(APPDIRS.user_config_dir)
+USER_DATA_DIR = pathlib.Path(APPDIRS.user_data_dir)
+
 
 def dir_wrapper(path: pathlib.Path) -> typing.Callable[[], pathlib.Path]:
     def wrapper() -> pathlib.Path:

geo_activity_playground/core/test_pandas_timezone.py

@@ -0,0 +1,36 @@
+import datetime
+import tempfile
+import zoneinfo
+
+import altair as alt
+import pandas as pd
+
+
+def test_dataframe_timezone() -> None:
+    df = pd.DataFrame(
+        {
+            "time": [
+                datetime.datetime(
+                    2025, 1, 1, 1, 1, 1, tzinfo=zoneinfo.ZoneInfo("Europe/Berlin")
+                )
+            ]
+        }
+    )
+
+    with tempfile.TemporaryFile() as f:
+        df.to_parquet(f)
+
+
+def test_altair_timezone() -> None:
+    df = pd.DataFrame(
+        {
+            "time": [
+                datetime.datetime(
+                    2025, 1, 1, 1, 1, 1, tzinfo=zoneinfo.ZoneInfo("Europe/Berlin")
+                )
+            ]
+        }
+    )
+
+    chart = alt.Chart(df).mark_tick().encode(alt.X("time"))
+    chart.to_json(format="vega")

geo_activity_playground/core/test_time_zone_from_location.py

@@ -0,0 +1,7 @@
+from .time_conversion import get_country_timezone
+
+
+def test_time_zone_from_location() -> None:
+    country, iana_timezone = get_country_timezone(50, 7)
+    assert country == "Germany"
+    assert iana_timezone == "Europe/Berlin"

geo_activity_playground/core/test_time_zone_import.py

@@ -0,0 +1,93 @@
+import datetime
+import pathlib
+import zoneinfo
+
+from ..importers.activity_parsers import read_activity
+from .time_conversion import sanitize_datetime
+
+
+def test_time_zone_from_string() -> None:
+    """
+    Understanding test for zoneinfo.
+
+    A user from Helsinki has recorded an activity. His device recorded 2025-06-21 14:41:06 in UTC. The local time was 17:41:06. How would we represent that properly? We need to import the time as UTC and then convert into the Helsinki time zone. At the end we drop the time zone information to make it a “naive” time but with the local time zone.
+    """
+    tz_helsinki = zoneinfo.ZoneInfo("Europe/Helsinki")
+    tz_utc = zoneinfo.ZoneInfo("UTC")
+    dt_utc = datetime.datetime(2025, 6, 21, 14, 41, 6, tzinfo=tz_utc)
+    dt_helsinki = dt_utc.astimezone(tz_helsinki)
+    assert dt_helsinki == datetime.datetime(2025, 6, 21, 17, 41, 6, tzinfo=tz_helsinki)
+    assert dt_helsinki.replace(tzinfo=None) == datetime.datetime(2025, 6, 21, 17, 41, 6)
+
+
+def test_utc_to_helsinki() -> None:
+    assert sanitize_datetime(
+        datetime.datetime(2025, 6, 21, 14, 41, 6, tzinfo=zoneinfo.ZoneInfo("UTC")),
+        fallback_from="UTC",
+        fallback_to="Europe/Helsinki",
+    ) == datetime.datetime(
+        2025, 6, 21, 17, 41, 6, tzinfo=zoneinfo.ZoneInfo("Europe/Helsinki")
+    )
+
+
+def test_0200_to_helsinki() -> None:
+    assert sanitize_datetime(
+        datetime.datetime(
+            2025,
+            6,
+            21,
+            16,
+            41,
+            6,
+            tzinfo=datetime.timezone(datetime.timedelta(hours=2)),
+        ),
+        fallback_from="UTC",
+        fallback_to="Europe/Helsinki",
+    ) == datetime.datetime(
+        2025, 6, 21, 17, 41, 6, tzinfo=zoneinfo.ZoneInfo("Europe/Helsinki")
+    )
+
+
+def test_naive_utc_to_helsinki() -> None:
+    assert sanitize_datetime(
+        datetime.datetime(2025, 6, 21, 14, 41, 6),
+        fallback_from="UTC",
+        fallback_to="Europe/Helsinki",
+    ) == datetime.datetime(
+        2025, 6, 21, 17, 41, 6, tzinfo=zoneinfo.ZoneInfo("Europe/Helsinki")
+    )
+
+
+def test_naive_helsinki_to_helsinki() -> None:
+    assert sanitize_datetime(
+        datetime.datetime(2025, 6, 21, 17, 41, 6),
+        fallback_from="Europe/Helsinki",
+        fallback_to="Europe/Helsinki",
+    ) == datetime.datetime(
+        2025, 6, 21, 17, 41, 6, tzinfo=zoneinfo.ZoneInfo("Europe/Helsinki")
+    )
+
+
+def test_time_zone_from_abvio() -> None:
+    """
+    Apply local time zone from Abvio generated files.
+
+    As reported in https://github.com/martin-ueding/geo-activity-playground/issues/303, the GPX files from Abvio contain the time data as UTC, but there is a field `abvio:startTimeZone` which contains it.
+
+    ```
+    <desc>Cyclemeter Row 21. Jun 2025 at 17.41.06</desc>
+    <time>2025-06-21T15:10:41Z</time>
+    <trkpt lat="…" lon="…"><ele>137.7</ele><time>2025-06-21T14:41:06Z</time></trkpt>
+    ...
+    <abvio:startTime>2025-06-21 14:41:06.537</abvio:startTime>
+    <abvio:startTimeZone>Europe/Helsinki</abvio:startTimeZone>
+    ```
+    """
+    path = pathlib.Path(
+        "/home/mu/Dokumente/Geo Activity Playground/Test-Suite/b1b9ec9b-016a-4223-9218-12b97d7019f2.gpx"
+    )
+    meta, ts = read_activity(path)
+
+    assert ts["time"].iloc[0] == datetime.datetime(
+        2025, 6, 21, 17, 41, 6, tzinfo=zoneinfo.ZoneInfo("Europe/Helsinki")
+    )

geo_activity_playground/core/test_timezone_sqlalchemy.py

@@ -0,0 +1,44 @@
+import datetime
+import zoneinfo
+
+import pytest
+import sqlalchemy as sa
+from sqlalchemy import Column
+from sqlalchemy import ForeignKey
+from sqlalchemy import String
+from sqlalchemy import Table
+from sqlalchemy.orm import DeclarativeBase
+from sqlalchemy.orm import Mapped
+from sqlalchemy.orm import mapped_column
+from sqlalchemy.orm import relationship
+from sqlalchemy.orm import Session
+
+
+class MyTestBase(DeclarativeBase):
+    pass
+
+
+class MyTestEvent(MyTestBase):
+    __tablename__ = "events"
+
+    # Housekeeping data:
+    id: Mapped[int] = mapped_column(primary_key=True)
+    time: Mapped[datetime.datetime] = mapped_column(
+        sa.DateTime(timezone=True), nullable=True
+    )
+
+
+@pytest.mark.xfail(reason="SQLite cannot store time zones.")
+def test_timezone_sqlalchemy() -> None:
+    engine = sa.create_engine("sqlite://", echo=False)
+    MyTestBase.metadata.create_all(engine)
+
+    dt_berlin = datetime.datetime(
+        2025, 7, 1, 14, 0, 0, tzinfo=zoneinfo.ZoneInfo("Europe/Berlin")
+    )
+
+    with Session(engine) as session:
+        event = MyTestEvent(time=dt_berlin)
+        session.add(event)
+        session.commit()
+        assert event.time == dt_berlin

geo_activity_playground/core/tiles.py

@@ -4,12 +4,15 @@ from typing import Iterator
 from typing import Optional
 
 import numpy as np
+import pandas as pd
 
 
 logger = logging.getLogger(__name__)
 
 
-def compute_tile_float(lat: float, lon: float, zoom: int) -> tuple[float, float]:
+def compute_tile_float(
+    lat: float | pd.Series, lon: float | pd.Series, zoom: int
+) -> tuple[float, float]:
     x = np.radians(lon)
     y = np.arcsinh(np.tan(np.radians(lat)))
     x = (1 + x / np.pi) / 2