geo-activity-playground 0.23.0__py3-none-any.whl → 0.24.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- geo_activity_playground/__main__.py +1 -1
- geo_activity_playground/core/activities.py +13 -12
- geo_activity_playground/core/activity_parsers.py +8 -32
- geo_activity_playground/core/cache_migrations.py +24 -0
- geo_activity_playground/core/heatmap.py +21 -21
- geo_activity_playground/core/privacy_zones.py +16 -0
- geo_activity_playground/core/similarity.py +1 -1
- geo_activity_playground/core/test_time_conversion.py +37 -0
- geo_activity_playground/core/time_conversion.py +14 -0
- geo_activity_playground/explorer/tile_visits.py +44 -32
- geo_activity_playground/importers/__init__.py +0 -0
- geo_activity_playground/importers/directory.py +7 -2
- geo_activity_playground/importers/strava_api.py +6 -0
- geo_activity_playground/importers/strava_checkout.py +4 -3
- geo_activity_playground/webui/__init__.py +0 -0
- geo_activity_playground/webui/activity/__init__.py +0 -0
- geo_activity_playground/webui/activity/blueprint.py +58 -0
- geo_activity_playground/webui/{activity_controller.py → activity/controller.py} +128 -18
- geo_activity_playground/webui/{templates/activity-day.html.j2 → activity/templates/activity/day.html.j2} +14 -2
- geo_activity_playground/webui/{templates/activity-name.html.j2 → activity/templates/activity/name.html.j2} +1 -1
- geo_activity_playground/webui/{templates/activity.html.j2 → activity/templates/activity/show.html.j2} +9 -4
- geo_activity_playground/webui/app.py +54 -283
- geo_activity_playground/webui/calendar/__init__.py +0 -0
- geo_activity_playground/webui/calendar/blueprint.py +26 -0
- geo_activity_playground/webui/{calendar_controller.py → calendar/controller.py} +5 -5
- geo_activity_playground/webui/{templates/calendar.html.j2 → calendar/templates/calendar/index.html.j2} +3 -2
- geo_activity_playground/webui/{templates/calendar-month.html.j2 → calendar/templates/calendar/month.html.j2} +2 -2
- geo_activity_playground/webui/eddington/__init__.py +0 -0
- geo_activity_playground/webui/eddington/blueprint.py +19 -0
- geo_activity_playground/webui/{eddington_controller.py → eddington/controller.py} +14 -6
- geo_activity_playground/webui/eddington/templates/eddington/index.html.j2 +56 -0
- geo_activity_playground/webui/entry_controller.py +1 -1
- geo_activity_playground/webui/equipment/__init__.py +0 -0
- geo_activity_playground/webui/equipment/blueprint.py +19 -0
- geo_activity_playground/webui/{equipment_controller.py → equipment/controller.py} +5 -3
- geo_activity_playground/webui/explorer/__init__.py +0 -0
- geo_activity_playground/webui/explorer/blueprint.py +54 -0
- geo_activity_playground/webui/{templates/explorer.html.j2 → explorer/templates/explorer/index.html.j2} +2 -2
- geo_activity_playground/webui/heatmap/__init__.py +0 -0
- geo_activity_playground/webui/heatmap/blueprint.py +41 -0
- geo_activity_playground/webui/{heatmap_controller.py → heatmap/heatmap_controller.py} +35 -10
- geo_activity_playground/webui/{templates/heatmap.html.j2 → heatmap/templates/heatmap/index.html.j2} +17 -2
- geo_activity_playground/webui/search_controller.py +1 -9
- geo_activity_playground/webui/square_planner/__init__.py +0 -0
- geo_activity_playground/webui/square_planner/blueprint.py +38 -0
- geo_activity_playground/webui/summary/__init__.py +0 -0
- geo_activity_playground/webui/summary/blueprint.py +16 -0
- geo_activity_playground/webui/summary/controller.py +268 -0
- geo_activity_playground/webui/summary/templates/summary/index.html.j2 +135 -0
- geo_activity_playground/webui/templates/{index.html.j2 → home.html.j2} +1 -1
- geo_activity_playground/webui/templates/page.html.j2 +22 -19
- geo_activity_playground/webui/templates/search.html.j2 +1 -1
- geo_activity_playground/webui/tile/__init__.py +0 -0
- geo_activity_playground/webui/tile/blueprint.py +31 -0
- geo_activity_playground/webui/upload/__init__.py +0 -0
- geo_activity_playground/webui/upload/blueprint.py +28 -0
- geo_activity_playground/webui/{upload_controller.py → upload/controller.py} +1 -0
- geo_activity_playground/webui/{templates/upload.html.j2 → upload/templates/upload/index.html.j2} +1 -1
- {geo_activity_playground-0.23.0.dist-info → geo_activity_playground-0.24.0.dist-info}/METADATA +2 -1
- geo_activity_playground-0.24.0.dist-info/RECORD +95 -0
- geo_activity_playground/webui/config_controller.py +0 -12
- geo_activity_playground/webui/locations_controller.py +0 -28
- geo_activity_playground/webui/summary_controller.py +0 -60
- geo_activity_playground/webui/templates/config.html.j2 +0 -24
- geo_activity_playground/webui/templates/eddington.html.j2 +0 -18
- geo_activity_playground/webui/templates/locations.html.j2 +0 -38
- geo_activity_playground/webui/templates/summary.html.j2 +0 -21
- geo_activity_playground-0.23.0.dist-info/RECORD +0 -74
- /geo_activity_playground/webui/{templates/activity-lines.html.j2 → activity/templates/activity/lines.html.j2} +0 -0
- /geo_activity_playground/webui/{templates/equipment.html.j2 → equipment/templates/equipment/index.html.j2} +0 -0
- /geo_activity_playground/webui/{explorer_controller.py → explorer/controller.py} +0 -0
- /geo_activity_playground/webui/{square_planner_controller.py → square_planner/controller.py} +0 -0
- /geo_activity_playground/webui/{templates/square-planner.html.j2 → square_planner/templates/square_planner/index.html.j2} +0 -0
- /geo_activity_playground/webui/{tile_controller.py → tile/controller.py} +0 -0
- {geo_activity_playground-0.23.0.dist-info → geo_activity_playground-0.24.0.dist-info}/LICENSE +0 -0
- {geo_activity_playground-0.23.0.dist-info → geo_activity_playground-0.24.0.dist-info}/WHEEL +0 -0
- {geo_activity_playground-0.23.0.dist-info → geo_activity_playground-0.24.0.dist-info}/entry_points.txt +0 -0
@@ -13,7 +13,7 @@ from geo_activity_playground.core.config import get_config
|
|
13
13
|
from geo_activity_playground.explorer.tile_visits import TileVisitAccessor
|
14
14
|
from geo_activity_playground.explorer.video import explorer_video_main
|
15
15
|
from geo_activity_playground.webui.app import webui_main
|
16
|
-
from geo_activity_playground.webui.
|
16
|
+
from geo_activity_playground.webui.upload.controller import scan_for_activities
|
17
17
|
|
18
18
|
logger = logging.getLogger(__name__)
|
19
19
|
|
@@ -18,6 +18,7 @@ from geo_activity_playground.core.paths import activities_path
|
|
18
18
|
from geo_activity_playground.core.paths import activity_timeseries_path
|
19
19
|
from geo_activity_playground.core.tasks import WorkTracker
|
20
20
|
from geo_activity_playground.core.tiles import compute_tile_float
|
21
|
+
from geo_activity_playground.core.time_conversion import convert_to_datetime_ns
|
21
22
|
|
22
23
|
logger = logging.getLogger(__name__)
|
23
24
|
|
@@ -25,6 +26,7 @@ logger = logging.getLogger(__name__)
|
|
25
26
|
class ActivityMeta(TypedDict):
|
26
27
|
calories: float
|
27
28
|
commute: bool
|
29
|
+
consider_for_achievements: bool
|
28
30
|
distance_km: float
|
29
31
|
elapsed_time: datetime.timedelta
|
30
32
|
end_latitude: float
|
@@ -78,11 +80,6 @@ class ActivityRepository:
|
|
78
80
|
f"Adding {len(self._loose_activities)} activities to the repository …"
|
79
81
|
)
|
80
82
|
new_df = pd.DataFrame(self._loose_activities)
|
81
|
-
if not pd.api.types.is_dtype_equal(
|
82
|
-
new_df["start"].dtype, "datetime64[ns, UTC]"
|
83
|
-
):
|
84
|
-
new_df["start"] = new_df["start"].dt.tz_localize("UTC")
|
85
|
-
new_df["start"] = new_df["start"].dt.tz_convert("UTC")
|
86
83
|
if len(self.meta):
|
87
84
|
new_ids_set = set(new_df["id"])
|
88
85
|
is_kept = [
|
@@ -93,7 +90,7 @@ class ActivityRepository:
|
|
93
90
|
old_df = self.meta
|
94
91
|
self.meta = pd.concat([old_df, new_df])
|
95
92
|
assert pd.api.types.is_dtype_equal(
|
96
|
-
self.meta["start"].dtype, "datetime64[ns
|
93
|
+
self.meta["start"].dtype, "datetime64[ns]"
|
97
94
|
), (self.meta["start"].dtype, self.meta["start"].iloc[0])
|
98
95
|
self.save()
|
99
96
|
self._loose_activities = []
|
@@ -121,9 +118,11 @@ class ActivityRepository:
|
|
121
118
|
else:
|
122
119
|
return None
|
123
120
|
|
124
|
-
|
125
|
-
|
126
|
-
|
121
|
+
def get_activity_ids(self, only_achievements: bool = False) -> set[int]:
|
122
|
+
if only_achievements:
|
123
|
+
return set(self.meta.loc[self.meta["consider_for_achievements"]].index)
|
124
|
+
else:
|
125
|
+
return set(self.meta.index)
|
127
126
|
|
128
127
|
def iter_activities(self, new_to_old=True, dropna=False) -> Iterator[ActivityMeta]:
|
129
128
|
direction = -1 if new_to_old else 1
|
@@ -152,7 +151,7 @@ class ActivityRepository:
|
|
152
151
|
|
153
152
|
def embellish_time_series(repository: ActivityRepository) -> None:
|
154
153
|
work_tracker = WorkTracker("embellish-time-series")
|
155
|
-
activities_to_process = work_tracker.filter(repository.
|
154
|
+
activities_to_process = work_tracker.filter(repository.get_activity_ids())
|
156
155
|
for activity_id in tqdm(activities_to_process, desc="Embellish time series data"):
|
157
156
|
path = activity_timeseries_path(activity_id)
|
158
157
|
df = pd.read_parquet(path)
|
@@ -176,9 +175,11 @@ def embellish_single_time_series(
|
|
176
175
|
):
|
177
176
|
time = timeseries["time"]
|
178
177
|
del timeseries["time"]
|
179
|
-
timeseries["time"] = [
|
178
|
+
timeseries["time"] = [
|
179
|
+
convert_to_datetime_ns(start + datetime.timedelta(seconds=t)) for t in time
|
180
|
+
]
|
180
181
|
changed = True
|
181
|
-
assert pd.api.types.is_dtype_equal(timeseries["time"].dtype, "datetime64[ns
|
182
|
+
assert pd.api.types.is_dtype_equal(timeseries["time"].dtype, "datetime64[ns]")
|
182
183
|
|
183
184
|
distances = get_distance(
|
184
185
|
timeseries["latitude"].shift(1),
|
@@ -8,15 +8,13 @@ import charset_normalizer
|
|
8
8
|
import dateutil.parser
|
9
9
|
import fitdecode
|
10
10
|
import gpxpy
|
11
|
-
import numpy as np
|
12
11
|
import pandas as pd
|
13
12
|
import tcxreader.tcxreader
|
14
13
|
import xmltodict
|
15
|
-
from pandas._libs import NaTType
|
16
14
|
|
17
15
|
from geo_activity_playground.core.activities import ActivityMeta
|
18
16
|
from geo_activity_playground.core.activities import embellish_single_time_series
|
19
|
-
from geo_activity_playground.core.
|
17
|
+
from geo_activity_playground.core.time_conversion import convert_to_datetime_ns
|
20
18
|
|
21
19
|
logger = logging.getLogger(__name__)
|
22
20
|
|
@@ -58,24 +56,6 @@ def read_activity(path: pathlib.Path) -> tuple[ActivityMeta, pd.DataFrame]:
|
|
58
56
|
raise ActivityParseError(f"Unsupported file format: {file_type}")
|
59
57
|
|
60
58
|
if len(timeseries):
|
61
|
-
# Unify time zones to UTC.
|
62
|
-
try:
|
63
|
-
if timeseries["time"].dt.tz is not None:
|
64
|
-
timeseries["time"] = timeseries["time"].dt.tz_localize(None)
|
65
|
-
timeseries["time"] = timeseries["time"].dt.tz_localize("UTC")
|
66
|
-
except AttributeError as e:
|
67
|
-
print(timeseries)
|
68
|
-
print(timeseries.dtypes)
|
69
|
-
types = {}
|
70
|
-
for elem in timeseries["time"]:
|
71
|
-
t = str(type(elem))
|
72
|
-
if t not in types:
|
73
|
-
types[t] = elem
|
74
|
-
print(types)
|
75
|
-
raise ActivityParseError(
|
76
|
-
"It looks like the date parsing has gone wrong."
|
77
|
-
) from e
|
78
|
-
|
79
59
|
timeseries, changed = embellish_single_time_series(timeseries)
|
80
60
|
|
81
61
|
# Extract some meta data from the time series.
|
@@ -128,11 +108,12 @@ def read_fit_activity(path: pathlib.Path, open) -> tuple[ActivityMeta, pd.DataFr
|
|
128
108
|
):
|
129
109
|
time = values["timestamp"]
|
130
110
|
if isinstance(time, datetime.datetime):
|
131
|
-
|
111
|
+
pass
|
132
112
|
elif time is None or isinstance(time, int):
|
133
113
|
time = pd.NaT
|
134
114
|
else:
|
135
115
|
raise RuntimeError(f"Cannot parse time: {time} in {path}.")
|
116
|
+
time = convert_to_datetime_ns(time)
|
136
117
|
row = {
|
137
118
|
"time": time,
|
138
119
|
"latitude": values["position_lat"] / ((2**32) / 360),
|
@@ -207,13 +188,11 @@ def read_gpx_activity(path: pathlib.Path, open) -> pd.DataFrame:
|
|
207
188
|
for point in segment.points:
|
208
189
|
if isinstance(point.time, datetime.datetime):
|
209
190
|
time = point.time
|
210
|
-
time = time.astimezone(datetime.timezone.utc)
|
211
191
|
elif isinstance(point.time, str):
|
212
192
|
time = dateutil.parser.parse(str(point.time))
|
213
|
-
time = time.astimezone(datetime.timezone.utc)
|
214
193
|
else:
|
215
194
|
time = pd.NaT
|
216
|
-
|
195
|
+
time = convert_to_datetime_ns(time)
|
217
196
|
points.append((time, point.latitude, point.longitude, point.elevation))
|
218
197
|
|
219
198
|
df = pd.DataFrame(points, columns=["time", "latitude", "longitude", "altitude"])
|
@@ -251,7 +230,7 @@ def read_tcx_activity(path: pathlib.Path, opener) -> pd.DataFrame:
|
|
251
230
|
if trackpoint.latitude and trackpoint.longitude:
|
252
231
|
time = trackpoint.time
|
253
232
|
assert isinstance(time, datetime.datetime)
|
254
|
-
time = time
|
233
|
+
time = convert_to_datetime_ns(time)
|
255
234
|
row = {
|
256
235
|
"time": time,
|
257
236
|
"latitude": trackpoint.latitude,
|
@@ -279,7 +258,8 @@ def read_kml_activity(path: pathlib.Path, opener) -> pd.DataFrame:
|
|
279
258
|
track = placemark["gx:Track"]
|
280
259
|
rows = []
|
281
260
|
for when, where in zip(track["when"], track["gx:coord"]):
|
282
|
-
time = dateutil.parser.parse(when)
|
261
|
+
time = dateutil.parser.parse(when)
|
262
|
+
time = convert_to_datetime_ns(time)
|
283
263
|
parts = where.split(" ")
|
284
264
|
if len(parts) == 2:
|
285
265
|
lon, lat = parts
|
@@ -298,11 +278,7 @@ def read_simra_activity(path: pathlib.Path, opener) -> pd.DataFrame:
|
|
298
278
|
data["time"] = data["timeStamp"].apply(
|
299
279
|
lambda d: datetime.datetime.fromtimestamp(d / 1000)
|
300
280
|
)
|
301
|
-
|
302
|
-
datetime.datetime.now(datetime.timezone.utc).astimezone().tzinfo
|
303
|
-
) # get local timezone
|
304
|
-
data["time"] = data["time"].dt.tz_localize(tz)
|
305
|
-
data["time"] = data["time"].dt.tz_convert("UTC")
|
281
|
+
data["time"] = convert_to_datetime_ns(data["time"])
|
306
282
|
data = data.rename(columns={"lat": "latitude", "lon": "longitude"})
|
307
283
|
return data.dropna(subset=["latitude"], ignore_index=True)[
|
308
284
|
["time", "latitude", "longitude"]
|
@@ -34,6 +34,7 @@ def reset_time_series_embellishment() -> None:
|
|
34
34
|
|
35
35
|
def delete_tile_visits() -> None:
|
36
36
|
paths = [
|
37
|
+
pathlib.Path("Cache/activities-per-tile.pickle"),
|
37
38
|
pathlib.Path("Cache/tile-evolution-state.pickle"),
|
38
39
|
pathlib.Path("Cache/tile-history.pickle"),
|
39
40
|
pathlib.Path("Cache/tile-visits.pickle"),
|
@@ -78,6 +79,24 @@ def convert_distances_to_km() -> None:
|
|
78
79
|
time_series.to_parquet(time_series_path)
|
79
80
|
|
80
81
|
|
82
|
+
def add_consider_for_achievements() -> None:
|
83
|
+
activities_path = pathlib.Path("Cache/activities.parquet")
|
84
|
+
if activities_path.exists():
|
85
|
+
df = pd.read_parquet(activities_path)
|
86
|
+
if "consider_for_achievements" not in df.columns:
|
87
|
+
df["consider_for_achievements"] = True
|
88
|
+
else:
|
89
|
+
df.loc[
|
90
|
+
df["consider_for_achievements"].isna(), "consider_for_achievements"
|
91
|
+
] = True
|
92
|
+
df.to_parquet("Cache/activities.parquet")
|
93
|
+
|
94
|
+
|
95
|
+
def delete_everything() -> None:
|
96
|
+
if pathlib.Path("Cache").exists():
|
97
|
+
shutil.rmtree("Cache")
|
98
|
+
|
99
|
+
|
81
100
|
def apply_cache_migrations() -> None:
|
82
101
|
logger.info("Apply cache migration if needed …")
|
83
102
|
cache_status_file = pathlib.Path("Cache/status.json")
|
@@ -98,6 +117,11 @@ def apply_cache_migrations() -> None:
|
|
98
117
|
delete_activity_metadata,
|
99
118
|
delete_tile_visits,
|
100
119
|
delete_heatmap_cache,
|
120
|
+
add_consider_for_achievements,
|
121
|
+
delete_tile_visits,
|
122
|
+
delete_heatmap_cache,
|
123
|
+
delete_tile_visits,
|
124
|
+
delete_everything,
|
101
125
|
]
|
102
126
|
|
103
127
|
for migration in migrations[cache_status["num_applied_migrations"] :]:
|
@@ -29,7 +29,7 @@ def get_bounds(lat_lon_data: np.ndarray) -> GeoBounds:
|
|
29
29
|
def add_margin(lower: float, upper: float) -> tuple[float, float]:
|
30
30
|
spread = upper - lower
|
31
31
|
margin = spread / 20
|
32
|
-
return max(0, lower - margin), upper + margin
|
32
|
+
return max(0.0, lower - margin), upper + margin
|
33
33
|
|
34
34
|
|
35
35
|
def add_margin_to_geo_bounds(bounds: GeoBounds) -> GeoBounds:
|
@@ -51,11 +51,28 @@ class TileBounds:
|
|
51
51
|
y_tile_min: int
|
52
52
|
y_tile_max: int
|
53
53
|
|
54
|
+
|
55
|
+
@dataclasses.dataclass
|
56
|
+
class PixelBounds:
|
57
|
+
x_min: int
|
58
|
+
x_max: int
|
59
|
+
y_min: int
|
60
|
+
y_max: int
|
61
|
+
|
62
|
+
@classmethod
|
63
|
+
def from_tile_bounds(cls, tile_bounds: TileBounds) -> "PixelBounds":
|
64
|
+
return cls(
|
65
|
+
int(tile_bounds.x_tile_min) * OSM_TILE_SIZE,
|
66
|
+
int(tile_bounds.x_tile_max) * OSM_TILE_SIZE,
|
67
|
+
int(tile_bounds.y_tile_min) * OSM_TILE_SIZE,
|
68
|
+
int(tile_bounds.y_tile_max) * OSM_TILE_SIZE,
|
69
|
+
)
|
70
|
+
|
54
71
|
@property
|
55
72
|
def shape(self) -> tuple[int, int]:
|
56
73
|
return (
|
57
|
-
|
58
|
-
|
74
|
+
self.y_max - self.y_min,
|
75
|
+
self.x_max - self.x_min,
|
59
76
|
)
|
60
77
|
|
61
78
|
|
@@ -107,7 +124,7 @@ def get_sensible_zoom_level(
|
|
107
124
|
|
108
125
|
|
109
126
|
def build_map_from_tiles(tile_bounds: TileBounds) -> np.ndarray:
|
110
|
-
background = np.zeros((*tile_bounds.shape, 3))
|
127
|
+
background = np.zeros((*PixelBounds.from_tile_bounds(tile_bounds).shape, 3))
|
111
128
|
|
112
129
|
for x in range(tile_bounds.x_tile_min, tile_bounds.x_tile_max):
|
113
130
|
for y in range(tile_bounds.y_tile_min, tile_bounds.y_tile_max):
|
@@ -129,20 +146,3 @@ def convert_to_grayscale(image: np.ndarray) -> np.ndarray:
|
|
129
146
|
image = np.sum(image * [0.2126, 0.7152, 0.0722], axis=2)
|
130
147
|
image = np.dstack((image, image, image))
|
131
148
|
return image
|
132
|
-
|
133
|
-
|
134
|
-
def crop_image_to_bounds(
|
135
|
-
image: np.ndarray, geo_bounds: GeoBounds, tile_bounds: TileBounds
|
136
|
-
) -> np.ndarray:
|
137
|
-
min_x, min_y = compute_tile_float(
|
138
|
-
geo_bounds.lat_max, geo_bounds.lon_min, tile_bounds.zoom
|
139
|
-
)
|
140
|
-
max_x, max_y = compute_tile_float(
|
141
|
-
geo_bounds.lat_min, geo_bounds.lon_max, tile_bounds.zoom
|
142
|
-
)
|
143
|
-
min_x = int((min_x - tile_bounds.x_tile_min) * OSM_TILE_SIZE)
|
144
|
-
min_y = int((min_y - tile_bounds.y_tile_min) * OSM_TILE_SIZE)
|
145
|
-
max_x = int((max_x - tile_bounds.x_tile_min) * OSM_TILE_SIZE)
|
146
|
-
max_y = int((max_y - tile_bounds.y_tile_min) * OSM_TILE_SIZE)
|
147
|
-
image = image[min_y:max_y, min_x:max_x, :]
|
148
|
-
return image
|
@@ -0,0 +1,16 @@
|
|
1
|
+
import pandas as pd
|
2
|
+
import shapely
|
3
|
+
|
4
|
+
|
5
|
+
class PrivacyZone:
|
6
|
+
def __init__(self, points: list[list[float]]) -> None:
|
7
|
+
self.points = points
|
8
|
+
self._polygon = shapely.Polygon(points)
|
9
|
+
shapely.prepare(self._polygon)
|
10
|
+
|
11
|
+
def filter_time_series(self, time_series: pd.DataFrame) -> pd.DataFrame:
|
12
|
+
mask = [
|
13
|
+
not shapely.contains_xy(self._polygon, row["longitude"], row["latitude"])
|
14
|
+
for index, row in time_series.iterrows()
|
15
|
+
]
|
16
|
+
return time_series.loc[mask]
|
@@ -29,7 +29,7 @@ def precompute_activity_distances(repository: ActivityRepository) -> None:
|
|
29
29
|
with stored_object(fingerprint_path, {}) as fingerprints, stored_object(
|
30
30
|
distances_path, {}
|
31
31
|
) as distances:
|
32
|
-
activity_ids = repository.
|
32
|
+
activity_ids = repository.get_activity_ids()
|
33
33
|
|
34
34
|
activity_ids_without_fingerprint = [
|
35
35
|
activity_id
|
@@ -0,0 +1,37 @@
|
|
1
|
+
import datetime
|
2
|
+
|
3
|
+
import numpy as np
|
4
|
+
import pandas as pd
|
5
|
+
|
6
|
+
from .time_conversion import convert_to_datetime_ns
|
7
|
+
|
8
|
+
target = np.datetime64(datetime.datetime(2000, 1, 2, 3, 4, 5))
|
9
|
+
|
10
|
+
|
11
|
+
def test_convert_to_datetime_ns() -> None:
|
12
|
+
dt_local = datetime.datetime(2000, 1, 2, 3, 4, 5)
|
13
|
+
dt_tz = datetime.datetime(
|
14
|
+
2000, 1, 2, 3, 4, 5, tzinfo=datetime.timezone(datetime.timedelta(hours=3))
|
15
|
+
)
|
16
|
+
dt_utc = datetime.datetime(2000, 1, 2, 3, 4, 5, tzinfo=datetime.timezone.utc)
|
17
|
+
|
18
|
+
inputs = [
|
19
|
+
dt_local,
|
20
|
+
dt_tz,
|
21
|
+
dt_utc,
|
22
|
+
pd.Timestamp(dt_local),
|
23
|
+
pd.Timestamp(dt_tz),
|
24
|
+
pd.Timestamp(dt_utc),
|
25
|
+
]
|
26
|
+
|
27
|
+
for d in inputs:
|
28
|
+
actual = convert_to_datetime_ns(d)
|
29
|
+
# assert pd.api.types.is_dtype_equal(actual.dtype, "datetime64[ns]")
|
30
|
+
assert actual == target
|
31
|
+
|
32
|
+
actual = convert_to_datetime_ns(pd.Series([d]))
|
33
|
+
assert actual.iloc[0] == target
|
34
|
+
|
35
|
+
|
36
|
+
def test_NaT() -> None:
|
37
|
+
assert pd.isna(convert_to_datetime_ns(pd.NaT))
|
@@ -0,0 +1,14 @@
|
|
1
|
+
import numpy as np
|
2
|
+
import pandas as pd
|
3
|
+
|
4
|
+
|
5
|
+
def convert_to_datetime_ns(date) -> np.datetime64:
|
6
|
+
if isinstance(date, pd.Series):
|
7
|
+
ts = pd.to_datetime(date)
|
8
|
+
ts = ts.dt.tz_localize(None)
|
9
|
+
return ts
|
10
|
+
else:
|
11
|
+
ts = pd.to_datetime(date)
|
12
|
+
if ts.tzinfo is not None:
|
13
|
+
ts = ts.tz_localize(None)
|
14
|
+
return ts.to_datetime64()
|
@@ -25,11 +25,13 @@ class TileVisitAccessor:
|
|
25
25
|
TILE_EVOLUTION_STATES_PATH = pathlib.Path("Cache/tile-evolution-state.pickle")
|
26
26
|
TILE_HISTORIES_PATH = pathlib.Path(f"Cache/tile-history.pickle")
|
27
27
|
TILE_VISITS_PATH = pathlib.Path(f"Cache/tile-visits.pickle")
|
28
|
+
ACTIVITIES_PER_TILE_PATH = pathlib.Path(f"Cache/activities-per-tile.pickle")
|
28
29
|
|
29
30
|
def __init__(self) -> None:
|
30
31
|
self.visits: dict[int, dict[tuple[int, int], dict[str, Any]]] = try_load_pickle(
|
31
32
|
self.TILE_VISITS_PATH
|
32
33
|
) or collections.defaultdict(dict)
|
34
|
+
"zoom → (tile_x, tile_y) → tile_info"
|
33
35
|
|
34
36
|
self.histories: dict[int, pd.DataFrame] = try_load_pickle(
|
35
37
|
self.TILE_HISTORIES_PATH
|
@@ -39,6 +41,12 @@ class TileVisitAccessor:
|
|
39
41
|
self.TILE_EVOLUTION_STATES_PATH
|
40
42
|
) or collections.defaultdict(TileEvolutionState)
|
41
43
|
|
44
|
+
self.activities_per_tile: dict[
|
45
|
+
int, dict[tuple[int, int], set[int]]
|
46
|
+
] = try_load_pickle(self.ACTIVITIES_PER_TILE_PATH) or collections.defaultdict(
|
47
|
+
dict
|
48
|
+
)
|
49
|
+
|
42
50
|
def save(self) -> None:
|
43
51
|
with open(self.TILE_VISITS_PATH, "wb") as f:
|
44
52
|
pickle.dump(self.visits, f)
|
@@ -49,13 +57,16 @@ class TileVisitAccessor:
|
|
49
57
|
with open(self.TILE_EVOLUTION_STATES_PATH, "wb") as f:
|
50
58
|
pickle.dump(self.states, f)
|
51
59
|
|
60
|
+
with open(self.ACTIVITIES_PER_TILE_PATH, "wb") as f:
|
61
|
+
pickle.dump(self.activities_per_tile, f)
|
62
|
+
|
52
63
|
|
53
64
|
def compute_tile_visits(
|
54
65
|
repository: ActivityRepository, tile_visits_accessor: TileVisitAccessor
|
55
66
|
) -> None:
|
56
67
|
|
57
68
|
work_tracker = WorkTracker("tile-visits")
|
58
|
-
activity_ids_to_process = work_tracker.filter(repository.
|
69
|
+
activity_ids_to_process = work_tracker.filter(repository.get_activity_ids())
|
59
70
|
new_tile_history_rows = collections.defaultdict(list)
|
60
71
|
for activity_id in tqdm(
|
61
72
|
activity_ids_to_process, desc="Extract explorer tile visits"
|
@@ -64,41 +75,42 @@ def compute_tile_visits(
|
|
64
75
|
for zoom in range(20):
|
65
76
|
for time, tile_x, tile_y in _tiles_from_points(time_series, zoom):
|
66
77
|
tile = (tile_x, tile_y)
|
67
|
-
if tile in tile_visits_accessor.
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
d
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
"
|
79
|
-
|
80
|
-
|
81
|
-
"
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
"
|
87
|
-
"
|
88
|
-
"
|
89
|
-
"tile_y": tile_y,
|
78
|
+
if not tile in tile_visits_accessor.activities_per_tile[zoom]:
|
79
|
+
tile_visits_accessor.activities_per_tile[zoom][tile] = set()
|
80
|
+
tile_visits_accessor.activities_per_tile[zoom][tile].add(activity_id)
|
81
|
+
|
82
|
+
activity = repository.get_activity_by_id(activity_id)
|
83
|
+
if activity["consider_for_achievements"]:
|
84
|
+
if tile in tile_visits_accessor.visits[zoom]:
|
85
|
+
d = tile_visits_accessor.visits[zoom][tile]
|
86
|
+
if d["first_time"] > time:
|
87
|
+
d["first_time"] = time
|
88
|
+
d["first_id"] = activity_id
|
89
|
+
if d["last_time"] < time:
|
90
|
+
d["last_time"] = time
|
91
|
+
d["last_id"] = activity_id
|
92
|
+
d["activity_ids"].add(activity_id)
|
93
|
+
else:
|
94
|
+
tile_visits_accessor.visits[zoom][tile] = {
|
95
|
+
"first_time": time,
|
96
|
+
"first_id": activity_id,
|
97
|
+
"last_time": time,
|
98
|
+
"last_id": activity_id,
|
99
|
+
"activity_ids": {activity_id},
|
90
100
|
}
|
91
|
-
|
101
|
+
new_tile_history_rows[zoom].append(
|
102
|
+
{
|
103
|
+
"activity_id": activity_id,
|
104
|
+
"time": time,
|
105
|
+
"tile_x": tile_x,
|
106
|
+
"tile_y": tile_y,
|
107
|
+
}
|
108
|
+
)
|
92
109
|
work_tracker.mark_done(activity_id)
|
93
110
|
|
94
|
-
if
|
111
|
+
if new_tile_history_rows:
|
95
112
|
for zoom, new_rows in new_tile_history_rows.items():
|
96
113
|
new_df = pd.DataFrame(new_rows)
|
97
|
-
if not pd.api.types.is_dtype_equal(
|
98
|
-
new_df["time"].dtype, "datetime64[ns, UTC]"
|
99
|
-
):
|
100
|
-
new_df["time"] = new_df["time"].dt.tz_localize("UTC")
|
101
|
-
new_df["time"] = new_df["time"].dt.tz_convert("UTC")
|
102
114
|
new_df.sort_values("time", inplace=True)
|
103
115
|
tile_visits_accessor.histories[zoom] = pd.concat(
|
104
116
|
[tile_visits_accessor.histories[zoom], new_df]
|
@@ -112,7 +124,7 @@ def compute_tile_visits(
|
|
112
124
|
def _tiles_from_points(
|
113
125
|
time_series: pd.DataFrame, zoom: int
|
114
126
|
) -> Iterator[tuple[datetime.datetime, int, int]]:
|
115
|
-
assert pd.api.types.is_dtype_equal(time_series["time"].dtype, "datetime64[ns
|
127
|
+
assert pd.api.types.is_dtype_equal(time_series["time"].dtype, "datetime64[ns]")
|
116
128
|
xf = time_series["x"] * 2**zoom
|
117
129
|
yf = time_series["y"] * 2**zoom
|
118
130
|
for t1, x1, y1, x2, y2, s1, s2 in zip(
|
File without changes
|
@@ -6,6 +6,7 @@ import pickle
|
|
6
6
|
import re
|
7
7
|
import sys
|
8
8
|
import traceback
|
9
|
+
from typing import Any
|
9
10
|
from typing import Optional
|
10
11
|
|
11
12
|
import pandas as pd
|
@@ -23,7 +24,9 @@ ACTIVITY_DIR = pathlib.Path("Activities")
|
|
23
24
|
|
24
25
|
|
25
26
|
def import_from_directory(
|
26
|
-
repository: ActivityRepository,
|
27
|
+
repository: ActivityRepository,
|
28
|
+
kind_defaults: dict[str, Any] = {},
|
29
|
+
metadata_extraction_regexes: list[str] = [],
|
27
30
|
) -> None:
|
28
31
|
paths_with_errors = []
|
29
32
|
work_tracker = WorkTracker("parse-activity-files")
|
@@ -66,9 +69,11 @@ def import_from_directory(
|
|
66
69
|
path=str(path),
|
67
70
|
kind="Unknown",
|
68
71
|
equipment="Unknown",
|
72
|
+
consider_for_achievements=True,
|
69
73
|
)
|
70
74
|
activity_meta.update(activity_meta_from_file)
|
71
75
|
activity_meta.update(_get_metadata_from_path(path, metadata_extraction_regexes))
|
76
|
+
activity_meta.update(kind_defaults.get(activity_meta["kind"], {}))
|
72
77
|
repository.add_activity(activity_meta)
|
73
78
|
|
74
79
|
if paths_with_errors:
|
@@ -97,7 +102,7 @@ def _cache_single_file(path: pathlib.Path) -> Optional[tuple[pathlib.Path, str]]
|
|
97
102
|
except ActivityParseError as e:
|
98
103
|
logger.error(f"Error while parsing file {path}:")
|
99
104
|
traceback.print_exc()
|
100
|
-
return
|
105
|
+
return path, str(e)
|
101
106
|
except:
|
102
107
|
logger.error(f"Encountered a problem with {path=}, see details below.")
|
103
108
|
raise
|
@@ -9,6 +9,7 @@ from typing import Any
|
|
9
9
|
|
10
10
|
import pandas as pd
|
11
11
|
from stravalib import Client
|
12
|
+
from stravalib.exc import Fault
|
12
13
|
from stravalib.exc import ObjectNotFound
|
13
14
|
from stravalib.exc import RateLimitExceeded
|
14
15
|
from tqdm import tqdm
|
@@ -173,6 +174,11 @@ def try_import_strava(repository: ActivityRepository) -> bool:
|
|
173
174
|
limit_exceeded = False
|
174
175
|
except RateLimitExceeded:
|
175
176
|
limit_exceeded = True
|
177
|
+
except Fault as e:
|
178
|
+
if "Too Many Requests" in str(e):
|
179
|
+
limit_exceeded = True
|
180
|
+
else:
|
181
|
+
raise
|
176
182
|
|
177
183
|
repository.commit()
|
178
184
|
|
@@ -16,6 +16,7 @@ from geo_activity_playground.core.activities import ActivityRepository
|
|
16
16
|
from geo_activity_playground.core.activity_parsers import ActivityParseError
|
17
17
|
from geo_activity_playground.core.activity_parsers import read_activity
|
18
18
|
from geo_activity_playground.core.tasks import WorkTracker
|
19
|
+
from geo_activity_playground.core.time_conversion import convert_to_datetime_ns
|
19
20
|
|
20
21
|
|
21
22
|
logger = logging.getLogger(__name__)
|
@@ -174,9 +175,9 @@ def import_from_strava_checkout(repository: ActivityRepository) -> None:
|
|
174
175
|
"id": activity_id,
|
175
176
|
"name": row["Activity Name"],
|
176
177
|
"path": str(activity_file),
|
177
|
-
"start":
|
178
|
-
row["Activity Date"], dayfirst=dayfirst
|
179
|
-
)
|
178
|
+
"start": convert_to_datetime_ns(
|
179
|
+
dateutil.parser.parse(row["Activity Date"], dayfirst=dayfirst)
|
180
|
+
),
|
180
181
|
}
|
181
182
|
|
182
183
|
time_series_path = activity_stream_dir / f"{activity_id}.parquet"
|
File without changes
|
File without changes
|
@@ -0,0 +1,58 @@
|
|
1
|
+
import urllib.parse
|
2
|
+
from collections.abc import Collection
|
3
|
+
|
4
|
+
from flask import Blueprint
|
5
|
+
from flask import render_template
|
6
|
+
from flask import Response
|
7
|
+
|
8
|
+
from ...core.activities import ActivityRepository
|
9
|
+
from ...explorer.tile_visits import TileVisitAccessor
|
10
|
+
from .controller import ActivityController
|
11
|
+
from geo_activity_playground.core.privacy_zones import PrivacyZone
|
12
|
+
|
13
|
+
|
14
|
+
def make_activity_blueprint(
|
15
|
+
repository: ActivityRepository,
|
16
|
+
tile_visit_accessor: TileVisitAccessor,
|
17
|
+
privacy_zones: Collection[PrivacyZone],
|
18
|
+
) -> Blueprint:
|
19
|
+
blueprint = Blueprint("activity", __name__, template_folder="templates")
|
20
|
+
|
21
|
+
activity_controller = ActivityController(
|
22
|
+
repository, tile_visit_accessor, privacy_zones
|
23
|
+
)
|
24
|
+
|
25
|
+
@blueprint.route("/activity/all")
|
26
|
+
def all():
|
27
|
+
return render_template(
|
28
|
+
"activity/lines.html.j2", **activity_controller.render_all()
|
29
|
+
)
|
30
|
+
|
31
|
+
@blueprint.route("/<id>")
|
32
|
+
def show(id: str):
|
33
|
+
return render_template(
|
34
|
+
"activity/show.html.j2", **activity_controller.render_activity(int(id))
|
35
|
+
)
|
36
|
+
|
37
|
+
@blueprint.route("/<id>/sharepic.png")
|
38
|
+
def sharepic(id: str):
|
39
|
+
return Response(
|
40
|
+
activity_controller.render_sharepic(int(id)),
|
41
|
+
mimetype="image/png",
|
42
|
+
)
|
43
|
+
|
44
|
+
@blueprint.route("/day/<year>/<month>/<day>")
|
45
|
+
def day(year: str, month: str, day: str):
|
46
|
+
return render_template(
|
47
|
+
"activity/day.html.j2",
|
48
|
+
**activity_controller.render_day(int(year), int(month), int(day))
|
49
|
+
)
|
50
|
+
|
51
|
+
@blueprint.route("/name/<name>")
|
52
|
+
def name(name: str):
|
53
|
+
return render_template(
|
54
|
+
"activity/name.html.j2",
|
55
|
+
**activity_controller.render_name(urllib.parse.unquote(name))
|
56
|
+
)
|
57
|
+
|
58
|
+
return blueprint
|