geo-activity-playground 1.1.0__py3-none-any.whl → 1.3.0__py3-none-any.whl

geo_activity_playground/importers/strava_checkout.py

@@ -1,10 +1,9 @@
 import datetime
 import logging
 import pathlib
-import pickle
 import shutil
 import sys
-import traceback
+import zoneinfo
 from typing import Optional
 
 import dateutil.parser
@@ -12,17 +11,16 @@ import numpy as np
 import pandas as pd
 from tqdm import tqdm
 
-from ..core.datamodel import ActivityMeta
+from ..core.config import Config
 from ..core.datamodel import DEFAULT_UNKNOWN_NAME
+from ..core.datamodel import get_or_make_equipment
+from ..core.datamodel import get_or_make_kind
+from ..core.enrichment import update_and_commit
 from ..core.paths import activity_extracted_meta_dir
-from ..core.paths import activity_extracted_time_series_dir
-from ..core.paths import strava_last_activity_date_path
 from ..core.tasks import get_state
 from ..core.tasks import set_state
 from ..core.tasks import work_tracker_path
 from ..core.tasks import WorkTracker
-from ..core.time_conversion import convert_to_datetime_ns
-from .activity_parsers import ActivityParseError
 from .activity_parsers import read_activity
 from .csv_parser import parse_csv
 
@@ -144,7 +142,7 @@ def float_with_comma_or_period(x: str) -> Optional[float]:
     return float(x)
 
 
-def import_from_strava_checkout() -> None:
+def import_from_strava_checkout(config: Config) -> None:
     checkout_path = pathlib.Path("Strava Export")
     with open(checkout_path / "activities.csv", encoding="utf-8") as f:
         rows = parse_csv(f.read())
@@ -191,47 +189,13 @@ def import_from_strava_checkout() -> None:
         if not row["Filename"]:
             continue
 
-        start_datetime = dateutil.parser.parse(row["Activity Date"], dayfirst=dayfirst)
+        start_datetime = dateutil.parser.parse(
+            row["Activity Date"], dayfirst=dayfirst
+        ).replace(tzinfo=zoneinfo.ZoneInfo("utc"))
 
         activity_file = checkout_path / row["Filename"]
-        table_activity_meta: ActivityMeta = {
-            "calories": float_with_comma_or_period(row["Calories"]),
-            "commute": row["Commute"] == "true",
-            "distance_km": row["Distance"],
-            "elapsed_time": datetime.timedelta(
-                seconds=float_with_comma_or_period(row["Elapsed Time"])
-            ),
-            "equipment": str(
-                nan_as_none(row["Activity Gear"])
-                or nan_as_none(row["Bike"])
-                or nan_as_none(row["Gear"])
-                or ""
-            ),
-            "kind": row["Activity Type"],
-            "id": activity_id,
-            "name": row["Activity Name"],
-            "path": str(activity_file),
-            "start": convert_to_datetime_ns(start_datetime),
-            "steps": float_with_comma_or_period(row["Total Steps"]),
-        }
-
-        time_series_path = (
-            activity_extracted_time_series_dir() / f"{activity_id}.parquet"
-        )
-        if time_series_path.exists():
-            time_series = pd.read_parquet(time_series_path)
-        else:
-            try:
-                file_activity_meta, time_series = read_activity(activity_file)
-            except ActivityParseError as e:
-                logger.error(f"Error while parsing file {activity_file}:")
-                traceback.print_exc()
-                continue
-            except:
-                logger.error(
-                    f"Encountered a problem with {activity_file=}, see details below."
-                )
-                raise
+
+        activity, time_series = read_activity(activity_file)
 
         if not len(time_series):
             continue
@@ -239,16 +203,26 @@ def import_from_strava_checkout() -> None:
         if "latitude" not in time_series.columns:
             continue
 
-        meta_path = activity_extracted_meta_dir() / f"{activity_id}.pickle"
-        with open(meta_path, "wb") as f:
-            pickle.dump(table_activity_meta, f)
-        time_series.to_parquet(time_series_path)
-
-        start_str = (
-            pd.Timestamp(table_activity_meta["start"]).to_pydatetime().isoformat() + "Z"
+        activity.upstream_id = activity_id
+        activity.calories = float_with_comma_or_period(row["Calories"])
+        activity.distance_km = row["Distance"]
+        activity.elapsed_time = datetime.timedelta(
+            seconds=float_with_comma_or_period(row["Elapsed Time"])
+        )
+        activity.equipment = get_or_make_equipment(
+            nan_as_none(row["Activity Gear"])
+            or nan_as_none(row["Bike"])
+            or nan_as_none(row["Gear"])
+            or DEFAULT_UNKNOWN_NAME,
+            config,
         )
-        latest_start_str = get_state(strava_last_activity_date_path(), start_str)
-        set_state(strava_last_activity_date_path(), max(start_str, latest_start_str))
+        activity.kind = get_or_make_kind(row["Activity Type"])
+        activity.name = row["Activity Name"]
+        activity.path = str(activity_file)
+        activity.start = start_datetime
+        activity.steps = float_with_comma_or_period(row["Total Steps"])
+
+        update_and_commit(activity, time_series, config)
 
     work_tracker.close()
 

geo_activity_playground/webui/app.py

@@ -6,11 +6,14 @@ import os
 import pathlib
 import secrets
 import shutil
+import threading
 import urllib.parse
 import uuid
+import warnings
 
 import pandas as pd
 import sqlalchemy
+from flask import Config
 from flask import Flask
 from flask import request
 from flask_alembic import Alembic
@@ -19,12 +22,14 @@ from ..core.activities import ActivityRepository
 from ..core.config import ConfigAccessor
 from ..core.config import import_old_config
 from ..core.config import import_old_strava_config
+from ..core.datamodel import Activity
 from ..core.datamodel import DB
 from ..core.datamodel import Equipment
 from ..core.datamodel import Kind
 from ..core.datamodel import Photo
 from ..core.datamodel import Tag
 from ..core.heart_rate import HeartRateZoneComputer
+from ..core.paths import TIME_SERIES_DIR
 from ..core.raster_map import GrayscaleImageTransform
 from ..core.raster_map import IdentityImageTransform
 from ..core.raster_map import PastelImageTransform
@@ -49,6 +54,7 @@ from .blueprints.settings_blueprint import make_settings_blueprint
 from .blueprints.square_planner_blueprint import make_square_planner_blueprint
 from .blueprints.summary_blueprint import make_summary_blueprint
 from .blueprints.tile_blueprint import make_tile_blueprint
+from .blueprints.time_zone_fixer_blueprint import make_time_zone_fixer_blueprint
 from .blueprints.upload_blueprint import make_upload_blueprint
 from .blueprints.upload_blueprint import scan_for_activities
 from .flasher import FlaskFlasher
@@ -65,11 +71,22 @@ def get_secret_key():
             secret = json.load(f)
     else:
         secret = secrets.token_hex()
+        secret_file.parent.mkdir(exist_ok=True, parents=True)
         with open(secret_file, "w") as f:
             json.dump(secret, f)
     return secret
 
 
+def importer_thread(
+    app: Flask,
+    repository: ActivityRepository,
+    tile_visit_accessor: TileVisitAccessor,
+    config: Config,
+) -> None:
+    with app.app_context():
+        scan_for_activities(repository, tile_visit_accessor, config)
+
+
 def web_ui_main(
     basedir: pathlib.Path,
     skip_reload: bool,
@@ -78,6 +95,12 @@ def web_ui_main(
 ) -> None:
     os.chdir(basedir)
 
+    warnings.filterwarnings("ignore", "__array__ implementation doesn't")
+    warnings.filterwarnings("ignore", '\'field "native_field_num"')
+    warnings.filterwarnings("ignore", '\'field "units"')
+    warnings.filterwarnings(
+        "ignore", r"datetime.datetime.utcfromtimestamp\(\) is deprecated"
+    )
     app = Flask(__name__)
 
     database_path = pathlib.Path("database.sqlite")
@@ -100,9 +123,21 @@ def web_ui_main(
     import_old_config(config_accessor)
     import_old_strava_config(config_accessor)
 
+    with app.app_context():
+        for activity in DB.session.scalars(sqlalchemy.select(Activity)).all():
+            if not activity.time_series_uuid:
+                activity.time_series_uuid = str(uuid.uuid4())
+                DB.session.commit()
+            old_path = TIME_SERIES_DIR() / f"{activity.id}.parquet"
+            if old_path.exists() and not activity.time_series_path.exists():
+                old_path.rename(activity.time_series_path)
+
     if not skip_reload:
-        with app.app_context():
-            scan_for_activities(repository, tile_visit_accessor, config_accessor())
+        thread = threading.Thread(
+            target=importer_thread,
+            args=(app, repository, tile_visit_accessor, config_accessor()),
+        )
+        thread.start()
 
     app.config["UPLOAD_FOLDER"] = "Activities"
     app.secret_key = get_secret_key()
@@ -183,6 +218,9 @@ def web_ui_main(
         ),
         "/summary": make_summary_blueprint(repository, config, search_query_history),
         "/tile": make_tile_blueprint(image_transforms, tile_getter),
+        "/time-zone-fixer": make_time_zone_fixer_blueprint(
+            authenticator, config, tile_visit_accessor
+        ),
         "/upload": make_upload_blueprint(
             repository, tile_visit_accessor, config_accessor(), authenticator, flasher
         ),

geo_activity_playground/webui/blueprints/activity_blueprint.py

@@ -32,7 +32,7 @@ from ...core.datamodel import DB
 from ...core.datamodel import Equipment
 from ...core.datamodel import Kind
 from ...core.datamodel import Tag
-from ...core.enrichment import update_via_time_series
+from ...core.enrichment import update_and_commit
 from ...core.heart_rate import HeartRateZoneComputer
 from ...core.privacy_zones import PrivacyZone
 from ...core.raster_map import map_image_from_tile_bounds
@@ -44,8 +44,7 @@ from ...explorer.grid_file import make_grid_points
 from ...explorer.tile_visits import TileVisitAccessor
 from ..authenticator import Authenticator
 from ..authenticator import needs_authentication
-from ..columns import column_elevation
-from ..columns import column_speed
+from ..columns import TIME_SERIES_COLUMNS
 
 logger = logging.getLogger(__name__)
 
@@ -137,7 +136,7 @@ def make_activity_blueprint(
             new_tiles_per_zoom[zoom] = len(new_tiles)
 
         line_color_columns_avail = dict(
-            [(column.name, column) for column in [column_speed, column_elevation]]
+            [(column.name, column) for column in TIME_SERIES_COLUMNS]
         )
         line_color_column = (
             request.args.get("line_color_column")
@@ -165,8 +164,8 @@ def make_activity_blueprint(
                         time_series[line_color_column],
                         line_color_columns_avail[line_color_column].format,
                     ),
-                    "date": activity.start.date(),
-                    "time": activity.start.time(),
+                    "date": activity.start_local_tz.date(),
+                    "time": activity.start_local_tz.time(),
                     "line_color_column": line_color_column,
                     "line_color_columns_avail": line_color_columns_avail,
                 }
@@ -178,7 +177,7 @@ def make_activity_blueprint(
             )
         ) is not None:
             context["heart_zones_plot"] = heart_rate_zone_plot(heart_zones)
-        if "elevation" in time_series.columns:
+        if "copernicus_elevation" in time_series.columns:
             context["elevation_time_plot"] = elevation_time_plot(time_series)
         if "elevation_gain_cum" in time_series.columns:
             context["elevation_gain_cum_plot"] = elevation_gain_cum_plot(time_series)
@@ -380,9 +379,8 @@ def make_activity_blueprint(
             if form_end:
                 activity.index_end = int(form_end)
 
-            update_via_time_series(activity, activity.time_series)
-
-            DB.session.commit()
+            time_series = activity.time_series
+            update_and_commit(activity, time_series, config)
 
         cmap = matplotlib.colormaps["turbo"]
         num_points = len(activity.time_series)
@@ -509,7 +507,11 @@ def elevation_time_plot(time_series: pd.DataFrame) -> str:
         .mark_line()
         .encode(
             alt.X("time", title="Time"),
-            alt.Y("elevation", scale=alt.Scale(zero=False), title="Elevation / m"),
+            alt.Y(
+                "copernicus_elevation",
+                scale=alt.Scale(zero=False),
+                title="Elevation / m",
+            ),
             alt.Color("segment_id:N", title="Segment"),
         )
         .interactive(bind_y=False)

geo_activity_playground/webui/blueprints/entry_views.py

@@ -44,7 +44,7 @@ def register_entry_views(
                 .order_by(Activity.start.desc())
                 .limit(100)
             ):
-                context["latest_activities"][activity.start.date()].append(
+                context["latest_activities"][activity.start_local_tz.date()].append(
                     {
                         "activity": activity,
                         "line_geojson": make_geojson_from_time_series(

geo_activity_playground/webui/blueprints/explorer_blueprint.py

@@ -2,7 +2,6 @@ import abc
 import datetime
 import hashlib
 import io
-import itertools
 import logging
 from collections.abc import Iterable
 from typing import Union
@@ -13,10 +12,8 @@ import matplotlib
 import matplotlib.pyplot as pl
 import numpy as np
 import pandas as pd
-import sqlalchemy
 from flask import Blueprint
 from flask import flash
-from flask import json
 from flask import redirect
 from flask import render_template
 from flask import request
@@ -24,7 +21,6 @@ from flask import Response
 from flask import url_for
 from flask.typing import ResponseReturnValue
 
-from ...core.activities import ActivityRepository
 from ...core.config import ConfigAccessor
 from ...core.coordinates import Bounds
 from ...core.datamodel import Activity
@@ -34,8 +30,6 @@ from ...core.raster_map import TileGetter
 from ...core.tiles import compute_tile
 from ...core.tiles import get_tile_upper_left_lat_lon
 from ...explorer.grid_file import get_border_tiles
-from ...explorer.grid_file import make_explorer_rectangle
-from ...explorer.grid_file import make_explorer_tile
 from ...explorer.grid_file import make_grid_file_geojson
 from ...explorer.grid_file import make_grid_file_gpx
 from ...explorer.grid_file import make_grid_points
@@ -231,7 +225,7 @@ def make_explorer_blueprint(
         tile_evolution_state = tile_visit_accessor.tile_state["evolution_state"][zoom]
         tile_history = tile_visit_accessor.tile_state["tile_history"][zoom]
 
-        medians = tile_history.median()
+        medians = tile_history[["tile_x", "tile_y"]].median()
         median_lat, median_lon = get_tile_upper_left_lat_lon(
             medians["tile_x"], medians["tile_y"], zoom
         )

geo_activity_playground/webui/blueprints/heatmap_blueprint.py

@@ -51,7 +51,7 @@ def make_heatmap_blueprint(
 
         zoom = 14
         tiles = tile_histories[zoom]
-        medians = tiles.median(skipna=True)
+        medians = tiles[["tile_x", "tile_y"]].median(skipna=True)
         median_lat, median_lon = get_tile_upper_left_lat_lon(
             medians["tile_x"], medians["tile_y"], zoom
         )
@@ -162,7 +162,7 @@ def _get_counts(
                 )
                 tile_counts = np.zeros(tile_pixels, dtype=np.int32)
                 parsed_activities.clear()
-            for activity_id in activity_ids:
+            for activity_id in list(activity_ids):
                 if activity_id in parsed_activities:
                     continue
                 parsed_activities.add(activity_id)

geo_activity_playground/webui/blueprints/photo_blueprint.py

@@ -12,6 +12,7 @@ from flask import render_template
 from flask import request
 from flask import Response
 from flask import url_for
+from flask.typing import ResponseReturnValue
 from PIL import Image
 from PIL import ImageOps
 
@@ -123,7 +124,7 @@ def make_photo_blueprint(
 
     @blueprint.route("/new", methods=["GET", "POST"])
     @needs_authentication(authenticator)
-    def new() -> Response:
+    def new() -> ResponseReturnValue:
         if request.method == "POST":
             # check if the post request has the file part
             if "file" not in request.files:
@@ -132,66 +133,74 @@ def make_photo_blueprint(
                 )
                 return redirect(url_for(".new"))
 
-            file = request.files["file"]
-            # If the user does not select a file, the browser submits an
-            # empty file without a filename.
-            if file.filename == "":
-                flasher.flash_message("No selected file.", FlashTypes.WARNING)
-                return redirect(url_for(".new"))
-            if not file:
-                flasher.flash_message("Empty file uploaded.", FlashTypes.WARNING)
-                return redirect(url_for(".new"))
+            new_photos: list[Photo] = []
+            for file in request.files.getlist("file"):
+                # If the user does not select a file, the browser submits an
+                # empty file without a filename.
+                if file.filename == "":
+                    flasher.flash_message("No selected file.", FlashTypes.WARNING)
+                    return redirect(url_for(".new"))
+                if not file:
+                    flasher.flash_message("Empty file uploaded.", FlashTypes.WARNING)
+                    return redirect(url_for(".new"))
+
+                filename = str(uuid.uuid4()) + pathlib.Path(file.filename).suffix
+                path = PHOTOS_DIR() / "original" / filename
+                path.parent.mkdir(exist_ok=True)
+                file.save(path)
+                metadata = get_metadata_from_image(path)
+
+                if "time" not in metadata:
+                    flasher.flash_message(
+                        f"Your image '{file.filename}' doesn't have the EXIF attribute 'EXIF DateTimeOriginal' and hence cannot be dated.",
+                        FlashTypes.DANGER,
+                    )
+                    continue
+                time: datetime.datetime = metadata["time"]
+
+                activity = DB.session.scalar(
+                    sqlalchemy.select(Activity)
+                    .where(
+                        Activity.start.is_not(None),
+                        Activity.elapsed_time.is_not(None),
+                        Activity.start <= time,
+                    )
+                    .order_by(Activity.start.desc())
+                    .limit(1)
+                )
+                if activity is None or activity.start + activity.elapsed_time < time:
+                    flasher.flash_message(
+                        f"Your image '{file.filename}' is from {time} but no activity could be found. Please first upload an activity or fix the time in the photo.",
+                        FlashTypes.DANGER,
+                    )
+                    continue
+
+                if "latitude" not in metadata:
+                    time_series = activity.time_series
+                    print(time_series)
+                    row = time_series.loc[time_series["time"] >= time].iloc[0]
+                    metadata["latitude"] = row["latitude"]
+                    metadata["longitude"] = row["longitude"]
+
+                photo = Photo(
+                    filename=filename,
+                    time=time,
+                    latitude=metadata["latitude"],
+                    longitude=metadata["longitude"],
+                    activity=activity,
+                )
 
-            filename = str(uuid.uuid4()) + pathlib.Path(file.filename).suffix
-            path = PHOTOS_DIR() / "original" / filename
-            path.parent.mkdir(exist_ok=True)
-            file.save(path)
-            metadata = get_metadata_from_image(path)
+                DB.session.add(photo)
+                DB.session.commit()
+                new_photos.append(photo)
 
-            if "time" not in metadata:
+            if new_photos:
                 flasher.flash_message(
-                    "Your image doesn't have the EXIF attribute 'EXIF DateTimeOriginal' and hence cannot be dated.",
-                    FlashTypes.DANGER,
+                    f"Added {len(new_photos)} new photos.", FlashTypes.SUCCESS
                 )
+                return redirect(f"/activity/{new_photos[-1].activity.id}")
+            else:
                 return redirect(url_for(".new"))
-            time: datetime.datetime = metadata["time"]
-
-            activity = DB.session.scalar(
-                sqlalchemy.select(Activity)
-                .where(
-                    Activity.start.is_not(None),
-                    Activity.elapsed_time.is_not(None),
-                    Activity.start <= time,
-                )
-                .order_by(Activity.start.desc())
-                .limit(1)
-            )
-            if activity is None or activity.start + activity.elapsed_time < time:
-                flasher.flash_message(
-                    f"Your image is from {time} but no activity could be found. Please first upload an activity or fix the time in the photo",
-                    FlashTypes.DANGER,
-                )
-            print(activity)
-
-            if "latitude" not in metadata:
-                time_series = activity.time_series
-                print(time_series)
-                row = time_series.loc[time_series["time"] >= time].iloc[0]
-                metadata["latitude"] = row["latitude"]
-                metadata["longitude"] = row["longitude"]
-
-            photo = Photo(
-                filename=filename,
-                time=time,
-                latitude=metadata["latitude"],
-                longitude=metadata["longitude"],
-                activity=activity,
-            )
-
-            DB.session.add(photo)
-            DB.session.commit()
-
-            return redirect(f"/activity/{activity.id}")
         else:
             return render_template("photo/new.html.j2")
 

geo_activity_playground/webui/blueprints/settings_blueprint.py

@@ -1,6 +1,5 @@
 import json
 import re
-import shutil
 import urllib.parse
 from typing import Any
 from typing import Optional
@@ -21,11 +20,8 @@ from ...core.datamodel import DB
 from ...core.datamodel import Equipment
 from ...core.datamodel import Kind
 from ...core.datamodel import Tag
-from ...core.enrichment import _embellish_single_time_series
-from ...core.enrichment import update_via_time_series
+from ...core.enrichment import update_and_commit
 from ...core.heart_rate import HeartRateZoneComputer
-from ...core.paths import _activity_enriched_dir
-from ...core.paths import TIME_SERIES_DIR
 from ..authenticator import Authenticator
 from ..authenticator import needs_authentication
 from ..flasher import Flasher
@@ -357,15 +353,8 @@ def make_settings_blueprint(
                 DB.session.scalars(sqlalchemy.select(Activity)).all(),
                 desc="Recomputing segments",
             ):
-                time_series = _embellish_single_time_series(
-                    activity.raw_time_series,
-                    None,
-                    threshold,
-                )
-                update_via_time_series(activity, time_series)
-                enriched_time_series_path = TIME_SERIES_DIR() / f"{activity.id}.parquet"
-                time_series.to_parquet(enriched_time_series_path)
-                DB.session.commit()
+                time_series = activity.time_series
+                update_and_commit(activity, time_series, config_accessor())
         return render_template(
             "settings/segmentation.html.j2",
             threshold=config_accessor().time_diff_threshold_seconds,
@@ -447,6 +436,23 @@ def make_settings_blueprint(
         else:
             return render_template("settings/tags-edit.html.j2", tag=tag)
 
+    @blueprint.route("/tile-source", methods=["GET", "POST"])
+    @needs_authentication(authenticator)
+    def tile_source() -> str:
+        if request.method == "POST":
+            config_accessor().map_tile_url = request.form["map_tile_url"]
+            config_accessor().map_tile_attribution = request.form[
+                "map_tile_attribution"
+            ]
+            config_accessor.save()
+            flasher.flash_message("Tile source updated.", FlashTypes.SUCCESS)
+        return render_template(
+            "settings/tile-source.html.j2",
+            map_tile_url=config_accessor().map_tile_url,
+            map_tile_attribution=config_accessor().map_tile_attribution,
+            test_url=config_accessor().map_tile_url.format(zoom=14, x=8514, y=5504),
+        )
+
     return blueprint
 
 

geo_activity_playground/webui/blueprints/summary_blueprint.py

@@ -178,10 +178,10 @@ def make_summary_blueprint(
     def index():
         query = search_query_from_form(request.args)
         search_query_history.register_query(query)
-        activities = apply_search_query(repository.meta, query)
+        df = apply_search_query(repository.meta, query)
 
         kind_scale = make_kind_scale(repository.meta, config)
-        df = activities
+        df_without_nan = df.loc[~pd.isna(df["start"])]
 
         return render_template(
             "summary/index.html.j2",
@@ -191,19 +191,19 @@ def make_summary_blueprint(
                 for spec in DB.session.scalars(sqlalchemy.select(PlotSpec)).all()
             ],
             plot_per_year_per_kind={
-                column.display_name: plot_per_year_per_kind(df, column)
+                column.display_name: plot_per_year_per_kind(df_without_nan, column)
                 for column in META_COLUMNS
             },
             plot_per_year_cumulative={
-                column.display_name: plot_year_cumulative(df, column)
+                column.display_name: plot_year_cumulative(df_without_nan, column)
                 for column in META_COLUMNS
             },
             plot_per_iso_week={
-                column.display_name: plot_per_iso_week(df, column)
+                column.display_name: plot_per_iso_week(df_without_nan, column)
                 for column in META_COLUMNS
             },
             heatmap_per_day={
-                column.display_name: heatmap_per_day(df, column)
+                column.display_name: heatmap_per_day(df_without_nan, column)
                 for column in META_COLUMNS
             },
         )