geo-activity-playground 0.38.2__py3-none-any.whl → 0.39.0__py3-none-any.whl

geo_activity_playground/__main__.py

@@ -1,20 +1,13 @@
 import argparse
 import logging
-import os
 import pathlib
 
 import coloredlogs
 
+from .explorer.video import explorer_video_main
+from .heatmap_video import main_heatmap_video
 from .importers.strava_checkout import convert_strava_checkout
-from geo_activity_playground.core.activities import ActivityRepository
-from geo_activity_playground.core.config import ConfigAccessor
-from geo_activity_playground.core.config import import_old_config
-from geo_activity_playground.core.config import import_old_strava_config
-from geo_activity_playground.explorer.tile_visits import TileVisitAccessor
-from geo_activity_playground.explorer.video import explorer_video_main
-from geo_activity_playground.heatmap_video import main_heatmap_video
-from geo_activity_playground.webui.app import web_ui_main
-from geo_activity_playground.webui.upload_blueprint import scan_for_activities
+from .webui.app import web_ui_main
 
 logger = logging.getLogger(__name__)
 
@@ -35,16 +28,6 @@ def main() -> None:
         description="The tools are organized in subcommands.", metavar="Command"
     )
 
-    # subparser = subparsers.add_parser(
-    #     "explorer",
-    #     help="Generate GeoJSON/GPX files with explored and missing explorer tiles.",
-    # )
-    # subparser.set_defaults(
-    #     func=lambda options: main_explorer(
-    #         make_time_series_source(options.basedir)
-    #     )
-    # )
-
     subparser = subparsers.add_parser(
         "explorer-video", help="Generate video with explorer timeline."
     )
@@ -65,7 +48,8 @@ def main() -> None:
     subparser = subparsers.add_parser("serve", help="Launch webserver")
     subparser.set_defaults(
         func=lambda options: web_ui_main(
-            *make_activity_repository(options.basedir, options.skip_reload),
+            options.basedir,
+            options.skip_reload,
             host=options.host,
             port=options.port,
         )
@@ -78,9 +62,6 @@ def main() -> None:
     )
     subparser.add_argument("--skip-reload", action=argparse.BooleanOptionalAction)
 
-    subparser = subparsers.add_parser("cache", help="Cache stuff")
-    subparser.set_defaults(func=lambda options: main_cache(options.basedir))
-
     subparser = subparsers.add_parser(
         "heatmap-video", help="Create a video with the evolution of the heatmap"
     )
@@ -103,28 +84,5 @@ def main() -> None:
     options.func(options)
 
 
-def make_activity_repository(
-    basedir: pathlib.Path, skip_reload: bool
-) -> tuple[ActivityRepository, TileVisitAccessor, ConfigAccessor]:
-    os.chdir(basedir)
-
-    repository = ActivityRepository()
-    tile_visit_accessor = TileVisitAccessor()
-    config_accessor = ConfigAccessor()
-    import_old_config(config_accessor)
-    import_old_strava_config(config_accessor)
-
-    if not skip_reload:
-        scan_for_activities(repository, tile_visit_accessor, config_accessor())
-
-    return repository, tile_visit_accessor, config_accessor
-
-
-def main_cache(basedir: pathlib.Path) -> None:
-    (repository, tile_visit_accessor, config_accessor) = make_activity_repository(
-        basedir, False
-    )
-
-
 if __name__ == "__main__":
     main()

geo_activity_playground/alembic/README

@@ -0,0 +1 @@
+Generic single-database configuration.

geo_activity_playground/alembic/env.py

@@ -0,0 +1,76 @@
+from logging.config import fileConfig
+
+from alembic import context
+from sqlalchemy import engine_from_config
+from sqlalchemy import pool
+
+from geo_activity_playground.core.datamodel import Base
+
+# this is the Alembic Config object, which provides
+# access to the values within the .ini file in use.
+config = context.config
+
+# Interpret the config file for Python logging.
+# This line sets up loggers basically.
+if config.config_file_name is not None:
+    fileConfig(config.config_file_name)
+
+target_metadata = Base.metadata
+
+# other values from the config, defined by the needs of env.py,
+# can be acquired:
+# my_important_option = config.get_main_option("my_important_option")
+# ... etc.
+
+
+def run_migrations_offline() -> None:
+    """Run migrations in 'offline' mode.
+
+    This configures the context with just a URL
+    and not an Engine, though an Engine is acceptable
+    here as well.  By skipping the Engine creation
+    we don't even need a DBAPI to be available.
+
+    Calls to context.execute() here emit the given string to the
+    script output.
+
+    """
+    url = config.get_main_option("sqlalchemy.url")
+    context.configure(
+        url=url,
+        target_metadata=target_metadata,
+        literal_binds=True,
+        dialect_opts={"paramstyle": "named"},
+        render_as_batch=True,
+    )
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online() -> None:
+    """Run migrations in 'online' mode.
+
+    In this scenario we need to create an Engine
+    and associate a connection with the context.
+
+    """
+    connectable = engine_from_config(
+        config.get_section(config.config_ini_section, {}),
+        prefix="sqlalchemy.",
+        poolclass=pool.NullPool,
+    )
+
+    with connectable.connect() as connection:
+        context.configure(
+            connection=connection, target_metadata=target_metadata, render_as_batch=True
+        )
+
+        with context.begin_transaction():
+            context.run_migrations()
+
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

geo_activity_playground/alembic/script.py.mako

@@ -0,0 +1,26 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+# revision identifiers, used by Alembic.
+revision: str = ${repr(up_revision)}
+down_revision: Union[str, None] = ${repr(down_revision)}
+branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
+depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
+
+
+def upgrade() -> None:
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade() -> None:
+    ${downgrades if downgrades else "pass"}

geo_activity_playground/alembic/versions/451e7836b53d_add_square_planner_bookmark.py

@@ -0,0 +1,33 @@
+from typing import Sequence
+from typing import Union
+
+import sqlalchemy as sa
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision: str = "451e7836b53d"
+down_revision: Union[str, None] = "ab83b9d23127"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "square_planner_bookmarks",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("zoom", sa.Integer(), nullable=False),
+        sa.Column("x", sa.Integer(), nullable=False),
+        sa.Column("y", sa.Integer(), nullable=False),
+        sa.Column("size", sa.Integer(), nullable=False),
+        sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint("zoom", "x", "y", "size", name="kinds_name"),
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table("square_planner_bookmarks")
+    # ### end Alembic commands ###

geo_activity_playground/alembic/versions/63d3b7f6f93c_initial_version.py

@@ -0,0 +1,73 @@
+from typing import Sequence
+from typing import Union
+
+import sqlalchemy as sa
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision: str = "63d3b7f6f93c"
+down_revision: Union[str, None] = None
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "equipments",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("name", sa.String(), nullable=False),
+        sa.Column("offset_km", sa.Integer(), nullable=False),
+        sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint("name", name="equipments_name"),
+    )
+    op.create_table(
+        "kinds",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("name", sa.String(), nullable=False),
+        sa.Column("consider_for_achievements", sa.Boolean(), nullable=False),
+        sa.Column("default_equipment_id", sa.Integer(), nullable=True),
+        sa.ForeignKeyConstraint(
+            ["default_equipment_id"], ["equipments.id"], name="default_equipment_id"
+        ),
+        sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint("name", name="kinds_name"),
+    )
+    op.create_table(
+        "activities",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("name", sa.String(), nullable=False),
+        sa.Column("path", sa.String(), nullable=True),
+        sa.Column("distance_km", sa.Float(), nullable=False),
+        sa.Column("start", sa.DateTime(), nullable=True),
+        sa.Column("elapsed_time", sa.Interval(), nullable=True),
+        sa.Column("moving_time", sa.Interval(), nullable=True),
+        sa.Column("start_latitude", sa.Float(), nullable=True),
+        sa.Column("start_longitude", sa.Float(), nullable=True),
+        sa.Column("end_latitude", sa.Float(), nullable=True),
+        sa.Column("end_longitude", sa.Float(), nullable=True),
+        sa.Column("elevation_gain", sa.Float(), nullable=True),
+        sa.Column("start_elevation", sa.Float(), nullable=True),
+        sa.Column("end_elevation", sa.Float(), nullable=True),
+        sa.Column("calories", sa.Integer(), nullable=True),
+        sa.Column("steps", sa.Integer(), nullable=True),
+        sa.Column("num_new_tiles_14", sa.Integer(), nullable=True),
+        sa.Column("num_new_tiles_17", sa.Integer(), nullable=True),
+        sa.Column("equipment_id", sa.Integer(), nullable=True),
+        sa.Column("kind_id", sa.Integer(), nullable=True),
+        sa.ForeignKeyConstraint(
+            ["equipment_id"], ["equipments.id"], name="equipment_id"
+        ),
+        sa.ForeignKeyConstraint(["kind_id"], ["kinds.id"], name="kind_id"),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table("activities")
+    op.drop_table("kinds")
+    op.drop_table("equipments")
+    # ### end Alembic commands ###

geo_activity_playground/alembic/versions/ab83b9d23127_add_upstream_id.py

@@ -0,0 +1,28 @@
+from typing import Sequence
+from typing import Union
+
+import sqlalchemy as sa
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision: str = "ab83b9d23127"
+down_revision: Union[str, None] = "b03491c593f6"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("activities", schema=None) as batch_op:
+        batch_op.add_column(sa.Column("upstream_id", sa.String(), nullable=True))
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("activities", schema=None) as batch_op:
+        batch_op.drop_column("upstream_id")
+
+    # ### end Alembic commands ###

geo_activity_playground/alembic/versions/b03491c593f6_add_crop_indices.py

@@ -0,0 +1,30 @@
+from typing import Sequence
+from typing import Union
+
+import sqlalchemy as sa
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision: str = "b03491c593f6"
+down_revision: Union[str, None] = "63d3b7f6f93c"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("activities", schema=None) as batch_op:
+        batch_op.add_column(sa.Column("index_begin", sa.Integer(), nullable=True))
+        batch_op.add_column(sa.Column("index_end", sa.Integer(), nullable=True))
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("activities", schema=None) as batch_op:
+        batch_op.drop_column("index_end")
+        batch_op.drop_column("index_begin")
+
+    # ### end Alembic commands ###

geo_activity_playground/alembic/versions/e02e27876deb_add_square_planner_bookmark_name.py

@@ -0,0 +1,28 @@
+from typing import Sequence
+from typing import Union
+
+import sqlalchemy as sa
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision: str = "e02e27876deb"
+down_revision: Union[str, None] = "451e7836b53d"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("square_planner_bookmarks", schema=None) as batch_op:
+        batch_op.add_column(sa.Column("name", sa.String(), nullable=False))
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("square_planner_bookmarks", schema=None) as batch_op:
+        batch_op.drop_column("name")
+
+    # ### end Alembic commands ###

geo_activity_playground/alembic/versions/script.py.mako

@@ -0,0 +1,28 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+# revision identifiers, used by Alembic.
+revision: str = ${repr(up_revision)}
+down_revision: Union[str, None] = ${repr(down_revision)}
+branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
+depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
+
+
+def upgrade() -> None:
+    """Upgrade schema."""
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    ${downgrades if downgrades else "pass"}

geo_activity_playground/core/activities.py

@@ -1,51 +1,25 @@
 import datetime
 import functools
-import json
 import logging
-import pickle
 from collections.abc import Callable
 from typing import Any
-from typing import Iterator
 from typing import Optional
-from typing import TypedDict
 
 import geojson
 import matplotlib
 import numpy as np
 import pandas as pd
+import sqlalchemy
 from tqdm import tqdm
 
-from geo_activity_playground.core.paths import activities_file
-from geo_activity_playground.core.paths import activity_enriched_meta_dir
-from geo_activity_playground.core.paths import activity_enriched_time_series_dir
-from geo_activity_playground.core.paths import activity_meta_override_dir
+from geo_activity_playground.core.datamodel import Activity
+from geo_activity_playground.core.datamodel import ActivityMeta
+from geo_activity_playground.core.datamodel import DB
+from geo_activity_playground.core.datamodel import Kind
 
 logger = logging.getLogger(__name__)
 
 
-class ActivityMeta(TypedDict):
-    average_speed_elapsed_kmh: float
-    average_speed_moving_kmh: float
-    calories: float
-    commute: bool
-    consider_for_achievements: bool
-    distance_km: float
-    elapsed_time: datetime.timedelta
-    elevation_gain: float
-    end_latitude: float
-    end_longitude: float
-    equipment: str
-    id: int
-    kind: str
-    moving_time: datetime.timedelta
-    name: str
-    path: str
-    start_latitude: float
-    start_longitude: float
-    start: np.datetime64
-    steps: int
-
-
 def make_activity_meta() -> ActivityMeta:
     return ActivityMeta(
         calories=None,
@@ -57,129 +31,69 @@ def make_activity_meta() -> ActivityMeta:
     )
 
 
-def build_activity_meta() -> None:
-    if activities_file().exists():
-        meta = pd.read_parquet(activities_file())
-        present_ids = set(meta["id"])
-    else:
-        meta = pd.DataFrame(columns=["id"])
-        present_ids = set()
-
-    available_ids = {
-        int(path.stem) for path in activity_enriched_meta_dir().glob("*.pickle")
-    }
-    new_ids = available_ids - present_ids
-    deleted_ids = present_ids - available_ids
-
-    # Remove updated activities and read these again.
-    if activities_file().exists():
-        meta_mtime = activities_file().stat().st_mtime
-        updated_ids = {
-            int(path.stem)
-            for path in activity_enriched_meta_dir().glob("*.pickle")
-            if path.stat().st_mtime > meta_mtime
-        }
-        new_ids.update(updated_ids)
-        deleted_ids.update(updated_ids & present_ids)
-
-    if deleted_ids:
-        logger.debug(f"Removing activities {deleted_ids} from repository.")
-        meta.drop(sorted(deleted_ids), axis="index", inplace=True)
-
-    rows = []
-    for new_id in tqdm(new_ids, desc="Register new activities"):
-        with open(activity_enriched_meta_dir() / f"{new_id}.pickle", "rb") as f:
-            data = pickle.load(f)
-        override_file = activity_meta_override_dir() / f"{new_id}.json"
-        if override_file.exists():
-            with open(override_file) as f:
-                data.update(json.load(f))
-        rows.append(data)
-
-    if rows:
-        new_shard = pd.DataFrame(rows)
-        new_shard.index = new_shard["id"]
-        new_shard.index.name = "index"
-        if len(meta):
-            meta = pd.concat([meta, new_shard])
-        else:
-            meta = new_shard
-
-    if len(meta):
-        assert pd.api.types.is_dtype_equal(meta["start"].dtype, "datetime64[ns]"), (
-            meta["start"].dtype,
-            meta["start"].iloc[0],
-        )
-
-        meta.sort_values("start", inplace=True)
-
-        meta.loc[meta["kind"] == "", "kind"] = "Unknown"
-        meta.loc[meta["equipment"] == "", "equipment"] = "Unknown"
-        meta["average_speed_moving_kmh"] = meta["distance_km"] / (
-            meta["moving_time"].dt.total_seconds() / 3_600
-        )
-        meta["average_speed_elapsed_kmh"] = meta["distance_km"] / (
-            meta["elapsed_time"].dt.total_seconds() / 3_600
-        )
-
-    meta.to_parquet(activities_file())
-
-
 class ActivityRepository:
-    def __init__(self) -> None:
-        self.meta = pd.DataFrame()
-
     def __len__(self) -> int:
-        return len(self.meta)
-
-    def reload(self) -> None:
-        self.meta = pd.read_parquet(activities_file())
+        return len(self.get_activity_ids())
 
     def has_activity(self, activity_id: int) -> bool:
-        if len(self.meta):
-            if activity_id in self.meta["id"]:
-                return True
-
-        return False
+        return bool(
+            DB.session.scalars(
+                sqlalchemy.query(Activity).where(Activity.id == activity_id)
+            ).all()
+        )
 
     def last_activity_date(self) -> Optional[datetime.datetime]:
-        if len(self.meta):
-            return self.meta.iloc[-1]["start"]
+        result = DB.session.scalars(
+            sqlalchemy.select(Activity).order_by(Activity.start)
+        ).all()
+        if result:
+            return result[-1].start
         else:
             return None
 
     def get_activity_ids(self, only_achievements: bool = False) -> list[int]:
+        query = sqlalchemy.select(Activity.id)
         if only_achievements:
-            return list(self.meta.loc[self.meta["consider_for_achievements"]].index)
-        else:
-            return list(self.meta.index)
-
-    def iter_activities(self, new_to_old=True, dropna=False) -> Iterator[ActivityMeta]:
+            query = query.where(Kind.consider_for_achievements)
+        result = DB.session.scalars(query).all()
+        return result
+
+    def iter_activities(self, new_to_old=True, drop_na=False) -> list[Activity]:
+        query = sqlalchemy.select(Activity)
+        if drop_na:
+            query = query.where(Activity.start.is_not(None))
+        result = DB.session.scalars(query.order_by(Activity.start)).all()
         direction = -1 if new_to_old else 1
-        for index, row in self.meta[::direction].iterrows():
-            if not dropna or not pd.isna(row["start"]):
-                yield row
+        return result[::direction]
 
-    def get_activity_by_id(self, id: int) -> ActivityMeta:
-        activity = self.meta.loc[id]
-        assert isinstance(activity["name"], str), activity["name"]
+    def get_activity_by_id(self, id: int) -> Activity:
+        activity = DB.session.scalar(
+            sqlalchemy.select(Activity).where(Activity.id == int(id))
+        )
+        if activity is None:
+            raise ValueError(f"Cannot find activity {id} in DB.session.")
         return activity
 
-    @functools.lru_cache(maxsize=3000)
     def get_time_series(self, id: int) -> pd.DataFrame:
-        path = activity_enriched_time_series_dir() / f"{id}.parquet"
-        try:
-            df = pd.read_parquet(path)
-        except OSError as e:
-            logger.error(f"Error while reading {path}, deleting cache file …")
-            path.unlink(missing_ok=True)
-            raise
-
+        return self.get_activity_by_id(id).time_series
+
+    @property
+    def meta(self) -> pd.DataFrame:
+        activities = self.iter_activities(new_to_old=False, drop_na=True)
+        df = pd.DataFrame([activity.to_dict() for activity in activities])
+        df["date"] = df["start"].dt.date
+        df["year"] = [start.year for start in df["start"]]
+        df["month"] = [start.month for start in df["start"]]
+        df["day"] = [start.day for start in df["start"]]
+        df["week"] = [start.isocalendar().week for start in df["start"]]
+        df["day_of_week"] = df["start"].dt.day_of_week
+        df["iso_year"] = [start.isocalendar().year for start in df["start"]]
+        df["hours"] = [
+            elapsed_time.total_seconds() / 3600 for elapsed_time in df["elapsed_time"]
+        ]
+        df.index = df["id"]
         return df
 
-    def save(self) -> None:
-        self.meta.to_parquet(activities_file())
-
 
 def make_geojson_from_time_series(time_series: pd.DataFrame) -> str:
     fc = geojson.FeatureCollection(

geo_activity_playground/core/config.py

@@ -5,14 +5,14 @@ import logging
 import pathlib
 from typing import Optional
 
-from geo_activity_playground.core.paths import new_config_file
-from geo_activity_playground.core.paths import strava_dynamic_config_path
+from .paths import new_config_file
+from .paths import strava_dynamic_config_path
 
 
 try:
     import tomllib
 except ModuleNotFoundError:
-    import tomli as tomllib
+    import tomli as tomllib  # type: ignore
 
 
 logger = logging.getLogger(__name__)