geo-activity-playground 0.35.0__py3-none-any.whl → 0.36.0__py3-none-any.whl

geo_activity_playground/__main__.py

@@ -12,6 +12,7 @@ from geo_activity_playground.core.config import import_old_config
 from geo_activity_playground.core.config import import_old_strava_config
 from geo_activity_playground.explorer.tile_visits import TileVisitAccessor
 from geo_activity_playground.explorer.video import explorer_video_main
+from geo_activity_playground.heatmap_video import main_heatmap_video
 from geo_activity_playground.webui.app import web_ui_main
 from geo_activity_playground.webui.upload_blueprint import scan_for_activities
 
@@ -80,6 +81,17 @@ def main() -> None:
     subparser = subparsers.add_parser("cache", help="Cache stuff")
     subparser.set_defaults(func=lambda options: main_cache(options.basedir))
 
+    subparser = subparsers.add_parser(
+        "heatmap-video", help="Create a video with the evolution of the heatmap"
+    )
+    subparser.add_argument("latitude", type=float)
+    subparser.add_argument("longitude", type=float)
+    subparser.add_argument("zoom", type=int)
+    subparser.add_argument("--decay", type=float, default=0.05)
+    subparser.add_argument("--video-width", type=int, default=1920)
+    subparser.add_argument("--video-height", type=int, default=1080)
+    subparser.set_defaults(func=main_heatmap_video)
+
     options = parser.parse_args()
     coloredlogs.install(
         fmt="%(asctime)s %(name)s %(levelname)s %(message)s",

geo_activity_playground/core/activities.py

@@ -3,6 +3,7 @@ import functools
 import json
 import logging
 import pickle
+from collections.abc import Callable
 from typing import Any
 from typing import Iterator
 from typing import Optional
@@ -180,41 +181,48 @@ def make_geojson_from_time_series(time_series: pd.DataFrame) -> str:
     return geojson.dumps(fc)
 
 
-def make_geojson_color_line(time_series: pd.DataFrame) -> str:
-    speed_without_na = time_series["speed"].dropna()
-    low = min(speed_without_na)
-    high = max(speed_without_na)
-    clamp_speed = lambda speed: min(max((speed - low) / (high - low), 0.0), 1.0)
+def inter_quartile_range(values):
+    return np.quantile(values, 0.75) - np.quantile(values, 0.25)
+
 
+def make_geojson_color_line(time_series: pd.DataFrame) -> str:
+    low, high, clamp_speed = _make_speed_clamp(time_series["speed"])
     cmap = matplotlib.colormaps["viridis"]
     features = [
         geojson.Feature(
             geometry=geojson.LineString(
                 coordinates=[
                     [row["longitude"], row["latitude"]],
-                    [next["longitude"], next["latitude"]],
+                    [next_row["longitude"], next_row["latitude"]],
                 ]
             ),
             properties={
-                "speed": next["speed"] if np.isfinite(next["speed"]) else 0.0,
-                "color": matplotlib.colors.to_hex(cmap(clamp_speed(next["speed"]))),
+                "speed": next_row["speed"] if np.isfinite(next_row["speed"]) else 0.0,
+                "color": matplotlib.colors.to_hex(cmap(clamp_speed(next_row["speed"]))),
             },
         )
         for _, group in time_series.groupby("segment_id")
-        for (_, row), (_, next) in zip(group.iterrows(), group.iloc[1:].iterrows())
+        for (_, row), (_, next_row) in zip(group.iterrows(), group.iloc[1:].iterrows())
     ]
     feature_collection = geojson.FeatureCollection(features)
     return geojson.dumps(feature_collection)
 
 
 def make_speed_color_bar(time_series: pd.DataFrame) -> dict[str, Any]:
-    speed_without_na = time_series["speed"].dropna()
-    low = min(speed_without_na)
-    high = max(speed_without_na)
+    low, high, clamp_speed = _make_speed_clamp(time_series["speed"])
     cmap = matplotlib.colormaps["viridis"]
-    clamp_speed = lambda speed: min(max((speed - low) / (high - low), 0.0), 1.0)
     colors = [
         (f"{speed:.1f}", matplotlib.colors.to_hex(cmap(clamp_speed(speed))))
         for speed in np.linspace(low, high, 10)
     ]
     return {"low": low, "high": high, "colors": colors}
+
+
+def _make_speed_clamp(speeds: pd.Series) -> tuple[float, float, Callable]:
+    speed_without_na = speeds.dropna()
+    low = min(speed_without_na)
+    high = min(
+        max(speed_without_na),
+        np.median(speed_without_na) + 1.5 * inter_quartile_range(speed_without_na),
+    )
+    return low, high, lambda speed: min(max((speed - low) / (high - low), 0.0), 1.0)

geo_activity_playground/core/raster_map.py

@@ -0,0 +1,246 @@
+import collections
+import dataclasses
+import functools
+import logging
+import pathlib
+import time
+import urllib.parse
+
+import numpy as np
+import requests
+from PIL import Image
+
+from geo_activity_playground.core.config import Config
+from geo_activity_playground.core.tiles import compute_tile_float
+from geo_activity_playground.core.tiles import get_tile_upper_left_lat_lon
+
+
+logger = logging.getLogger(__name__)
+
+
+OSM_TILE_SIZE = 256  # OSM tile size in pixel
+OSM_MAX_ZOOM = 19  # OSM maximum zoom level
+MAX_TILE_COUNT = 2000  # maximum number of tiles to download
+
+## Basic data types ##
+
+
+@dataclasses.dataclass
+class GeoBounds:
+    """
+    Models an area on the globe as a rectangle of latitude and longitude.
+
+    Latitude goes from South Pole (-90°) to North Pole (+90°). Longitude goes from West (-180°) to East (+180°). Be careful when converting latitude to Y-coordinates as increasing latitude will mean decreasing Y.
+    """
+
+    lat_min: float
+    lon_min: float
+    lat_max: float
+    lon_max: float
+
+
+@dataclasses.dataclass
+class TileBounds:
+    zoom: int
+    x1: float
+    y1: float
+    x2: float
+    y2: float
+
+    @property
+    def width(self) -> float:
+        return self.x2 - self.x1
+
+    @property
+    def height(self) -> float:
+        return self.y2 - self.y1
+
+
+@dataclasses.dataclass
+class PixelBounds:
+    x1: int
+    y1: int
+    x2: int
+    y2: int
+
+    @classmethod
+    def from_tile_bounds(cls, tile_bounds: TileBounds) -> "PixelBounds":
+        return pixel_bounds_from_tile_bounds(tile_bounds)
+
+    @property
+    def width(self) -> int:
+        return self.x2 - self.x1
+
+    @property
+    def height(self) -> int:
+        return self.y2 - self.y1
+
+    @property
+    def shape(self) -> tuple[int, int]:
+        return self.height, self.width
+
+
+@dataclasses.dataclass
+class RasterMapImage:
+    image: np.ndarray
+    tile_bounds: TileBounds
+    geo_bounds: GeoBounds
+    pixel_bounds: PixelBounds
+
+
+## Converter functions ##
+
+
+def pixel_bounds_from_tile_bounds(tile_bounds: TileBounds) -> PixelBounds:
+    return PixelBounds(
+        int(tile_bounds.x1 * OSM_TILE_SIZE),
+        int(tile_bounds.y1 * OSM_TILE_SIZE),
+        int(tile_bounds.x2 * OSM_TILE_SIZE),
+        int(tile_bounds.y2 * OSM_TILE_SIZE),
+    )
+
+
+def get_sensible_zoom_level(
+    bounds: GeoBounds, picture_size: tuple[int, int]
+) -> TileBounds:
+    zoom = OSM_MAX_ZOOM
+
+    while True:
+        x_tile_min, y_tile_max = map(
+            int, compute_tile_float(bounds.lat_min, bounds.lon_min, zoom)
+        )
+        x_tile_max, y_tile_min = map(
+            int, compute_tile_float(bounds.lat_max, bounds.lon_max, zoom)
+        )
+
+        x_tile_max += 1
+        y_tile_max += 1
+
+        if (x_tile_max - x_tile_min) * OSM_TILE_SIZE <= picture_size[0] and (
+            y_tile_max - y_tile_min
+        ) * OSM_TILE_SIZE <= picture_size[1]:
+            break
+
+        zoom -= 1
+
+    tile_count = (x_tile_max - x_tile_min) * (y_tile_max - y_tile_min)
+
+    if tile_count > MAX_TILE_COUNT:
+        raise RuntimeError("Zoom value too high, too many tiles to download")
+
+    return TileBounds(zoom, x_tile_min, y_tile_min, x_tile_max, y_tile_max)
+
+
+@functools.lru_cache()
+def get_tile(zoom: int, x: int, y: int, url_template: str) -> Image.Image:
+    destination = osm_tile_path(x, y, zoom, url_template)
+    if not destination.exists():
+        logger.info(f"Downloading OSM tile {x=}, {y=}, {zoom=} …")
+        url = url_template.format(x=x, y=y, zoom=zoom)
+        download_file(url, destination)
+    with Image.open(destination) as image:
+        image.load()
+        image = image.convert("RGB")
+    return image
+
+
+def tile_bounds_around_center(
+    tile_center: tuple[float, float], pixel_size: tuple[int, int], zoom: int
+) -> TileBounds:
+    x, y = tile_center
+    width = pixel_size[0] / OSM_TILE_SIZE
+    height = pixel_size[1] / OSM_TILE_SIZE
+    return TileBounds(
+        zoom, x - width / 2, y - height / 2, x + width / 2, y + height / 2
+    )
+
+
+def _paste_array(
+    target: np.ndarray, source: np.ndarray, offset_0: int, offset_1: int
+) -> None:
+    source_min_0 = 0
+    source_min_1 = 0
+    source_max_0 = source.shape[0]
+    source_max_1 = source.shape[1]
+
+    target_min_0 = offset_0
+    target_min_1 = offset_1
+    target_max_0 = offset_0 + source.shape[0]
+    target_max_1 = offset_1 + source.shape[1]
+
+    if target_min_1 < 0:
+        source_min_1 -= target_min_1
+        target_min_1 = 0
+    if target_min_0 < 0:
+        source_min_0 -= target_min_0
+        target_min_0 = 0
+    if target_max_1 > target.shape[1]:
+        a = target_max_1 - target.shape[1]
+        target_max_1 -= a
+        source_max_1 -= a
+    if target_max_0 > target.shape[0]:
+        a = target_max_0 - target.shape[0]
+        target_max_0 -= a
+        source_max_0 -= a
+
+    if source_max_1 < 0 or source_max_0 < 0:
+        return
+
+    target[target_min_0:target_max_0, target_min_1:target_max_1] = source[
+        source_min_0:source_max_0, source_min_1:source_max_1
+    ]
+
+
+def map_image_from_tile_bounds(tile_bounds: TileBounds, config: Config) -> np.ndarray:
+    pixel_bounds = pixel_bounds_from_tile_bounds(tile_bounds)
+    background = np.zeros((pixel_bounds.height, pixel_bounds.width, 3))
+
+    north_west = np.array([tile_bounds.x1, tile_bounds.y1])
+    offset = north_west % 1
+    tile_anchor = north_west - offset
+    pixel_anchor: np.ndarray = np.array([0, 0]) - np.array(
+        offset * OSM_TILE_SIZE, dtype=np.int64
+    )
+
+    num_tile_x = int(np.ceil(tile_bounds.width)) + 1
+    num_tile_y = int(np.ceil(tile_bounds.height)) + 1
+
+    for x in range(int(tile_anchor[0]), int(tile_anchor[0] + num_tile_x)):
+        for y in range(int(tile_anchor[1]), int(tile_anchor[1]) + num_tile_y):
+            tile = np.array(get_tile(tile_bounds.zoom, x, y, config.map_tile_url)) / 255
+            _paste_array(
+                background,
+                tile,
+                (y - int(tile_anchor[1])) * OSM_TILE_SIZE + int(pixel_anchor[1]),
+                (x - int(tile_anchor[0])) * OSM_TILE_SIZE + int(pixel_anchor[0]),
+            )
+
+    return background
+
+
+def convert_to_grayscale(image: np.ndarray) -> np.ndarray:
+    image = np.sum(image * [0.2126, 0.7152, 0.0722], axis=2)
+    image = np.dstack((image, image, image))
+    return image
+
+
+def osm_tile_path(x: int, y: int, zoom: int, url_template: str) -> pathlib.Path:
+    base_dir = pathlib.Path("Open Street Map Tiles")
+    dir_for_source = base_dir / urllib.parse.quote_plus(url_template)
+    path = dir_for_source / f"{zoom}/{x}/{y}.png"
+    path.parent.mkdir(parents=True, exist_ok=True)
+    return path
+
+
+def download_file(url: str, destination: pathlib.Path):
+    if not destination.parent.exists():
+        destination.parent.mkdir(exist_ok=True, parents=True)
+    r = requests.get(
+        url,
+        allow_redirects=True,
+        headers={"User-Agent": "Martin's Geo Activity Playground"},
+    )
+    assert r.ok
+    with open(destination, "wb") as f:
+        f.write(r.content)
+    time.sleep(0.1)

geo_activity_playground/core/tiles.py

@@ -1,34 +1,12 @@
-import functools
 import logging
 import math
-import pathlib
-import time
-import urllib.parse
 from typing import Iterator
 from typing import Optional
 
 import numpy as np
-import requests
-from PIL import Image
-
-logger = logging.getLogger(__name__)
-
-
-def osm_tile_path(x: int, y: int, zoom: int, url_template: str) -> pathlib.Path:
-    base_dir = pathlib.Path("Open Street Map Tiles")
-    dir_for_source = base_dir / urllib.parse.quote_plus(url_template)
-    path = dir_for_source / f"{zoom}/{x}/{y}.png"
-    path.parent.mkdir(parents=True, exist_ok=True)
-    return path
 
 
-def compute_tile(lat: float, lon: float, zoom: int) -> tuple[int, int]:
-    x = np.radians(lon)
-    y = np.arcsinh(np.tan(np.radians(lat)))
-    x = (1 + x / np.pi) / 2
-    y = (1 - y / np.pi) / 2
-    n = 2**zoom
-    return int(x * n), int(y * n)
+logger = logging.getLogger(__name__)
 
 
 def compute_tile_float(lat: float, lon: float, zoom: int) -> tuple[float, float]:
@@ -40,6 +18,11 @@ def compute_tile_float(lat: float, lon: float, zoom: int) -> tuple[float, float]
     return x * n, y * n
 
 
+def compute_tile(lat: float, lon: float, zoom: int) -> tuple[int, int]:
+    x, y = compute_tile_float(lat, lon, zoom)
+    return int(x), int(y)
+
+
 def get_tile_upper_left_lat_lon(
     tile_x: int, tile_y: int, zoom: int
 ) -> tuple[float, float]:
@@ -50,33 +33,6 @@ def get_tile_upper_left_lat_lon(
     return lat_deg, lon_deg
 
 
-def download_file(url: str, destination: pathlib.Path):
-    if not destination.parent.exists():
-        destination.parent.mkdir(exist_ok=True, parents=True)
-    r = requests.get(
-        url,
-        allow_redirects=True,
-        headers={"User-Agent": "Martin's Geo Activity Playground"},
-    )
-    assert r.ok
-    with open(destination, "wb") as f:
-        f.write(r.content)
-    time.sleep(0.1)
-
-
-@functools.lru_cache()
-def get_tile(zoom: int, x: int, y: int, url_template: str) -> Image.Image:
-    destination = osm_tile_path(x, y, zoom, url_template)
-    if not destination.exists():
-        logger.info(f"Downloading OSM tile {x=}, {y=}, {zoom=} …")
-        url = url_template.format(x=x, y=y, zoom=zoom)
-        download_file(url, destination)
-    with Image.open(destination) as image:
-        image.load()
-        image = image.convert("RGB")
-    return image
-
-
 def xy_to_latlon(x: float, y: float, zoom: int) -> tuple[float, float]:
     """
     Returns (lat, lon) in degree from OSM coordinates (x,y) rom https://wiki.openstreetmap.org/wiki/Slippy_map_tilenames

geo_activity_playground/explorer/video.py

@@ -13,7 +13,7 @@ from PIL import Image
 from PIL import ImageEnhance
 from tqdm import tqdm
 
-from ..core.tiles import get_tile
+from ..core.raster_map import get_tile
 
 # import scipy.interpolate
 

geo_activity_playground/heatmap_video.py

@@ -0,0 +1,93 @@
+import collections
+import os
+import pathlib
+
+import matplotlib.pyplot as pl
+import numpy as np
+import pandas as pd
+from PIL import Image
+from PIL import ImageDraw
+from tqdm import tqdm
+
+from geo_activity_playground.core.activities import ActivityRepository
+from geo_activity_playground.core.config import ConfigAccessor
+from geo_activity_playground.core.raster_map import convert_to_grayscale
+from geo_activity_playground.core.raster_map import map_image_from_tile_bounds
+from geo_activity_playground.core.raster_map import OSM_TILE_SIZE
+from geo_activity_playground.core.raster_map import tile_bounds_around_center
+from geo_activity_playground.core.tiles import compute_tile_float
+
+
+def main_heatmap_video(options) -> None:
+    zoom: int = options.zoom
+    print(options)
+    video_size = options.video_width, options.video_height
+    os.chdir(options.basedir)
+
+    repository = ActivityRepository()
+    repository.reload()
+    assert len(repository) > 0
+    config_accessor = ConfigAccessor()
+
+    center_xy = compute_tile_float(options.latitude, options.longitude, zoom)
+
+    tile_bounds = tile_bounds_around_center(center_xy, video_size, zoom)
+    background = map_image_from_tile_bounds(tile_bounds, config_accessor())
+
+    background = convert_to_grayscale(background)
+    background = 1.0 - background  # invert colors
+
+    activities_per_day = collections.defaultdict(set)
+    for activity in tqdm(
+        repository.iter_activities(), desc="Gather activities per day"
+    ):
+        activities_per_day[activity["start"].date()].add(activity["id"])
+
+    running_counts = np.zeros(background.shape[:2], np.float64)
+
+    output_dir = pathlib.Path("Heatmap Video")
+    output_dir.mkdir(exist_ok=True)
+
+    first_day = min(activities_per_day)
+    last_day = max(activities_per_day)
+    days = pd.date_range(first_day, last_day)
+    for current_day in tqdm(days, desc="Generate video frames"):
+        for activity_id in activities_per_day[current_day.date()]:
+            im = Image.new("L", video_size)
+            draw = ImageDraw.Draw(im)
+
+            time_series = repository.get_time_series(activity_id)
+            for _, group in time_series.groupby("segment_id"):
+                tile_xz = group["x"] * 2**zoom
+                tile_yz = group["y"] * 2**zoom
+
+                xy_pixels = list(
+                    zip(
+                        (tile_xz - center_xy[0]) * OSM_TILE_SIZE
+                        + options.video_width / 2,
+                        (tile_yz - center_xy[1]) * OSM_TILE_SIZE
+                        + options.video_height / 2,
+                    )
+                )
+                pixels = [int(value) for t in xy_pixels for value in t]
+                draw.line(pixels, fill=1, width=max(3, 6 * (zoom - 17)))
+                aim = np.array(im)
+                running_counts += aim
+
+        tile_counts = np.sqrt(running_counts) / 5
+        tile_counts[tile_counts > 1.0] = 1.0
+
+        cmap = pl.get_cmap(config_accessor().color_scheme_for_heatmap)
+        data_color = cmap(tile_counts)
+        data_color[data_color == cmap(0.0)] = 0.0  # remove background color
+
+        rendered = np.zeros_like(background)
+        for c in range(3):
+            rendered[:, :, c] = (1.0 - data_color[:, :, c]) * background[
+                :, :, c
+            ] + data_color[:, :, c]
+
+        img = Image.fromarray((rendered * 255).astype("uint8"), "RGB")
+        img.save(output_dir / f"{current_day.date()}.png", format="png")
+
+        running_counts *= 1 - options.decay

geo_activity_playground/importers/activity_parsers.py

@@ -24,7 +24,7 @@ class ActivityParseError(BaseException):
 
 
 def read_activity(path: pathlib.Path) -> tuple[ActivityMeta, pd.DataFrame]:
-    suffixes = path.suffixes
+    suffixes = [s.lower() for s in path.suffixes]
     metadata = ActivityMeta()
 
     if suffixes[-1] == ".gz":

geo_activity_playground/importers/directory.py

@@ -25,7 +25,7 @@ ACTIVITY_DIR = pathlib.Path("Activities")
 
 
 def import_from_directory(
-    metadata_extraction_regexes: list[str], num_processes: Optional[int], config: Config
+    metadata_extraction_regexes: list[str], config: Config
 ) -> None:
 
     activity_paths = [
@@ -63,20 +63,11 @@ def import_from_directory(
             del file_hashes[deleted_file]
             work_tracker.discard(deleted_file)
 
-    if num_processes == 1:
-        paths_with_errors = []
-        for path in tqdm(new_activity_paths, desc="Parse activity metadata (serially)"):
-            errors = _cache_single_file(path)
-            if errors:
-                paths_with_errors.append(errors)
-    else:
-        with multiprocessing.Pool(num_processes) as pool:
-            paths_with_errors = tqdm(
-                pool.imap(_cache_single_file, new_activity_paths),
-                desc="Parse activity metadata (concurrently)",
-                total=len(new_activity_paths),
-            )
-            paths_with_errors = [error for error in paths_with_errors if error]
+    paths_with_errors = []
+    for path in tqdm(new_activity_paths, desc="Parse activity metadata (serially)"):
+        errors = _cache_single_file(path)
+        if errors:
+            paths_with_errors.append(errors)
 
     for path in tqdm(new_activity_paths, desc="Collate activity metadata"):
         activity_id = get_file_hash(path)

geo_activity_playground/webui/activity/blueprint.py

@@ -1,6 +1,5 @@
 import json
 import urllib.parse
-from collections.abc import Collection
 
 from flask import Blueprint
 from flask import redirect
@@ -9,26 +8,20 @@ from flask import request
 from flask import Response
 from flask import url_for
 
-from ...core.activities import ActivityRepository
-from ...explorer.tile_visits import TileVisitAccessor
-from .controller import ActivityController
-from geo_activity_playground.core.config import Config
+from geo_activity_playground.core.activities import ActivityRepository
 from geo_activity_playground.core.paths import activity_meta_override_dir
-from geo_activity_playground.core.privacy_zones import PrivacyZone
+from geo_activity_playground.webui.activity.controller import ActivityController
 from geo_activity_playground.webui.authenticator import Authenticator
 from geo_activity_playground.webui.authenticator import needs_authentication
 
 
 def make_activity_blueprint(
+    activity_controller: ActivityController,
     repository: ActivityRepository,
-    tile_visit_accessor: TileVisitAccessor,
-    config: Config,
     authenticator: Authenticator,
 ) -> Blueprint:
     blueprint = Blueprint("activity", __name__, template_folder="templates")
 
-    activity_controller = ActivityController(repository, tile_visit_accessor, config)
-
     @blueprint.route("/all")
     def all():
         return render_template(