geo-activity-playground 0.24.2__py3-none-any.whl → 0.26.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- geo_activity_playground/__main__.py +0 -2
- geo_activity_playground/core/activities.py +71 -149
- geo_activity_playground/core/enrichment.py +164 -0
- geo_activity_playground/core/paths.py +34 -15
- geo_activity_playground/core/tasks.py +27 -4
- geo_activity_playground/explorer/tile_visits.py +78 -42
- geo_activity_playground/{core → importers}/activity_parsers.py +7 -14
- geo_activity_playground/importers/directory.py +36 -27
- geo_activity_playground/importers/strava_api.py +45 -38
- geo_activity_playground/importers/strava_checkout.py +24 -16
- geo_activity_playground/webui/activity/controller.py +2 -2
- geo_activity_playground/webui/activity/templates/activity/show.html.j2 +2 -0
- geo_activity_playground/webui/app.py +11 -31
- geo_activity_playground/webui/entry_controller.py +5 -5
- geo_activity_playground/webui/heatmap/heatmap_controller.py +6 -0
- geo_activity_playground/webui/static/bootstrap-dark-mode.js +78 -0
- geo_activity_playground/webui/strava/__init__.py +0 -0
- geo_activity_playground/webui/strava/blueprint.py +33 -0
- geo_activity_playground/webui/strava/controller.py +49 -0
- geo_activity_playground/webui/strava/templates/strava/client-id.html.j2 +36 -0
- geo_activity_playground/webui/strava/templates/strava/connected.html.j2 +14 -0
- geo_activity_playground/webui/templates/home.html.j2 +5 -0
- geo_activity_playground/webui/templates/page.html.j2 +44 -12
- geo_activity_playground/webui/templates/settings.html.j2 +24 -0
- geo_activity_playground/webui/upload/controller.py +13 -17
- {geo_activity_playground-0.24.2.dist-info → geo_activity_playground-0.26.0.dist-info}/METADATA +1 -1
- {geo_activity_playground-0.24.2.dist-info → geo_activity_playground-0.26.0.dist-info}/RECORD +30 -25
- geo_activity_playground/core/cache_migrations.py +0 -133
- geo_activity_playground/webui/strava_controller.py +0 -27
- geo_activity_playground/webui/templates/strava-connect.html.j2 +0 -30
- {geo_activity_playground-0.24.2.dist-info → geo_activity_playground-0.26.0.dist-info}/LICENSE +0 -0
- {geo_activity_playground-0.24.2.dist-info → geo_activity_playground-0.26.0.dist-info}/WHEEL +0 -0
- {geo_activity_playground-0.24.2.dist-info → geo_activity_playground-0.26.0.dist-info}/entry_points.txt +0 -0
@@ -7,12 +7,15 @@ import pickle
|
|
7
7
|
from typing import Any
|
8
8
|
from typing import Iterator
|
9
9
|
from typing import Optional
|
10
|
+
from typing import TypedDict
|
10
11
|
|
11
12
|
import pandas as pd
|
12
13
|
from tqdm import tqdm
|
13
14
|
|
14
15
|
from geo_activity_playground.core.activities import ActivityRepository
|
16
|
+
from geo_activity_playground.core.paths import tiles_per_time_series
|
15
17
|
from geo_activity_playground.core.tasks import try_load_pickle
|
18
|
+
from geo_activity_playground.core.tasks import work_tracker_path
|
16
19
|
from geo_activity_playground.core.tasks import WorkTracker
|
17
20
|
from geo_activity_playground.core.tiles import adjacent_to
|
18
21
|
from geo_activity_playground.core.tiles import interpolate_missing_tile
|
@@ -21,6 +24,12 @@ from geo_activity_playground.core.tiles import interpolate_missing_tile
|
|
21
24
|
logger = logging.getLogger(__name__)
|
22
25
|
|
23
26
|
|
27
|
+
class TileHistoryRow(TypedDict):
|
28
|
+
time: datetime.datetime
|
29
|
+
tile_x: int
|
30
|
+
tile_y: int
|
31
|
+
|
32
|
+
|
24
33
|
class TileVisitAccessor:
|
25
34
|
TILE_EVOLUTION_STATES_PATH = pathlib.Path("Cache/tile-evolution-state.pickle")
|
26
35
|
TILE_HISTORIES_PATH = pathlib.Path(f"Cache/tile-history.pickle")
|
@@ -64,60 +73,87 @@ class TileVisitAccessor:
|
|
64
73
|
def compute_tile_visits(
|
65
74
|
repository: ActivityRepository, tile_visits_accessor: TileVisitAccessor
|
66
75
|
) -> None:
|
76
|
+
present_activity_ids = repository.get_activity_ids()
|
77
|
+
work_tracker = WorkTracker(work_tracker_path("tile-visits"))
|
78
|
+
|
79
|
+
changed_zoom_tile = collections.defaultdict(set)
|
80
|
+
|
81
|
+
# Delete visits from removed activities.
|
82
|
+
for zoom, activities_per_tile in tile_visits_accessor.activities_per_tile.items():
|
83
|
+
for tile, activity_ids in activities_per_tile.items():
|
84
|
+
deleted_ids = activity_ids - present_activity_ids
|
85
|
+
if deleted_ids:
|
86
|
+
logger.debug(
|
87
|
+
f"Removing activities {deleted_ids} from tile {tile} at {zoom=}."
|
88
|
+
)
|
89
|
+
for activity_id in deleted_ids:
|
90
|
+
activity_ids.remove(activity_id)
|
91
|
+
work_tracker.discard(activity_id)
|
92
|
+
changed_zoom_tile[zoom].add(tile)
|
67
93
|
|
68
|
-
|
94
|
+
# Add visits from new activities.
|
69
95
|
activity_ids_to_process = work_tracker.filter(repository.get_activity_ids())
|
70
|
-
new_tile_history_rows = collections.defaultdict(list)
|
71
96
|
for activity_id in tqdm(
|
72
97
|
activity_ids_to_process, desc="Extract explorer tile visits"
|
73
98
|
):
|
74
|
-
time_series = repository.get_time_series(activity_id)
|
75
99
|
for zoom in range(20):
|
76
|
-
for time, tile_x, tile_y in _tiles_from_points(
|
100
|
+
for time, tile_x, tile_y in _tiles_from_points(
|
101
|
+
repository.get_time_series(activity_id), zoom
|
102
|
+
):
|
77
103
|
tile = (tile_x, tile_y)
|
78
|
-
if not
|
104
|
+
if tile not in tile_visits_accessor.activities_per_tile[zoom]:
|
79
105
|
tile_visits_accessor.activities_per_tile[zoom][tile] = set()
|
80
106
|
tile_visits_accessor.activities_per_tile[zoom][tile].add(activity_id)
|
81
|
-
|
82
|
-
activity = repository.get_activity_by_id(activity_id)
|
83
|
-
if activity["consider_for_achievements"]:
|
84
|
-
if tile in tile_visits_accessor.visits[zoom]:
|
85
|
-
d = tile_visits_accessor.visits[zoom][tile]
|
86
|
-
if d["first_time"] > time:
|
87
|
-
d["first_time"] = time
|
88
|
-
d["first_id"] = activity_id
|
89
|
-
if d["last_time"] < time:
|
90
|
-
d["last_time"] = time
|
91
|
-
d["last_id"] = activity_id
|
92
|
-
d["activity_ids"].add(activity_id)
|
93
|
-
else:
|
94
|
-
tile_visits_accessor.visits[zoom][tile] = {
|
95
|
-
"first_time": time,
|
96
|
-
"first_id": activity_id,
|
97
|
-
"last_time": time,
|
98
|
-
"last_id": activity_id,
|
99
|
-
"activity_ids": {activity_id},
|
100
|
-
}
|
101
|
-
new_tile_history_rows[zoom].append(
|
102
|
-
{
|
103
|
-
"activity_id": activity_id,
|
104
|
-
"time": time,
|
105
|
-
"tile_x": tile_x,
|
106
|
-
"tile_y": tile_y,
|
107
|
-
}
|
108
|
-
)
|
107
|
+
changed_zoom_tile[zoom].add(tile)
|
109
108
|
work_tracker.mark_done(activity_id)
|
110
109
|
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
110
|
+
# Update tile visits structure.
|
111
|
+
for zoom, changed_tiles in tqdm(
|
112
|
+
changed_zoom_tile.items(), desc="Incorporate changes in tiles"
|
113
|
+
):
|
114
|
+
soa = {"activity_id": [], "time": [], "tile_x": [], "tile_y": []}
|
115
|
+
|
116
|
+
for tile in changed_tiles:
|
117
|
+
activity_ids = tile_visits_accessor.activities_per_tile[zoom][tile]
|
118
|
+
activities = [
|
119
|
+
repository.get_activity_by_id(activity_id)
|
120
|
+
for activity_id in activity_ids
|
121
|
+
]
|
122
|
+
activities_to_consider = [
|
123
|
+
activity
|
124
|
+
for activity in activities
|
125
|
+
if activity["consider_for_achievements"]
|
126
|
+
]
|
127
|
+
activities_to_consider.sort(key=lambda activity: activity["start"])
|
128
|
+
|
129
|
+
if activities_to_consider:
|
130
|
+
tile_visits_accessor.visits[zoom][tile] = {
|
131
|
+
"first_time": activities_to_consider[0]["start"],
|
132
|
+
"first_id": activities_to_consider[0]["id"],
|
133
|
+
"last_time": activities_to_consider[-1]["start"],
|
134
|
+
"last_id": activities_to_consider[-1]["id"],
|
135
|
+
"activity_ids": {
|
136
|
+
activity["id"] for activity in activities_to_consider
|
137
|
+
},
|
138
|
+
}
|
139
|
+
|
140
|
+
soa["activity_id"].append(activities_to_consider[0]["id"])
|
141
|
+
soa["time"].append(activities_to_consider[0]["start"])
|
142
|
+
soa["tile_x"].append(tile[0])
|
143
|
+
soa["tile_y"].append(tile[1])
|
144
|
+
else:
|
145
|
+
if tile in tile_visits_accessor.visits[zoom]:
|
146
|
+
del tile_visits_accessor.visits[zoom][tile]
|
147
|
+
|
148
|
+
df = pd.DataFrame(soa)
|
149
|
+
if len(df) > 0:
|
150
|
+
df = pd.concat([tile_visits_accessor.histories[zoom], df])
|
151
|
+
df.sort_values("time", inplace=True)
|
152
|
+
tile_visits_accessor.histories[zoom] = df.groupby(
|
153
|
+
["tile_x", "tile_y"]
|
154
|
+
).head(1)
|
120
155
|
|
156
|
+
tile_visits_accessor.save()
|
121
157
|
work_tracker.close()
|
122
158
|
|
123
159
|
|
@@ -7,13 +7,13 @@ import xml
|
|
7
7
|
import charset_normalizer
|
8
8
|
import dateutil.parser
|
9
9
|
import fitdecode
|
10
|
+
import fitdecode.exceptions
|
10
11
|
import gpxpy
|
11
12
|
import pandas as pd
|
12
13
|
import tcxreader.tcxreader
|
13
14
|
import xmltodict
|
14
15
|
|
15
16
|
from geo_activity_playground.core.activities import ActivityMeta
|
16
|
-
from geo_activity_playground.core.activities import embellish_single_time_series
|
17
17
|
from geo_activity_playground.core.time_conversion import convert_to_datetime_ns
|
18
18
|
|
19
19
|
logger = logging.getLogger(__name__)
|
@@ -42,7 +42,12 @@ def read_activity(path: pathlib.Path) -> tuple[ActivityMeta, pd.DataFrame]:
|
|
42
42
|
except UnicodeDecodeError as e:
|
43
43
|
raise ActivityParseError(f"Encoding issue") from e
|
44
44
|
elif file_type == ".fit":
|
45
|
-
|
45
|
+
try:
|
46
|
+
metadata, timeseries = read_fit_activity(path, opener)
|
47
|
+
except fitdecode.exceptions.FitError as e:
|
48
|
+
raise ActivityParseError(f"Error in FIT file") from e
|
49
|
+
except KeyError as e:
|
50
|
+
raise ActivityParseError(f"Key error while parsing FIT file") from e
|
46
51
|
elif file_type == ".tcx":
|
47
52
|
try:
|
48
53
|
timeseries = read_tcx_activity(path, opener)
|
@@ -55,18 +60,6 @@ def read_activity(path: pathlib.Path) -> tuple[ActivityMeta, pd.DataFrame]:
|
|
55
60
|
else:
|
56
61
|
raise ActivityParseError(f"Unsupported file format: {file_type}")
|
57
62
|
|
58
|
-
if len(timeseries):
|
59
|
-
timeseries, changed = embellish_single_time_series(timeseries)
|
60
|
-
|
61
|
-
# Extract some meta data from the time series.
|
62
|
-
metadata["start"] = timeseries["time"].iloc[0]
|
63
|
-
metadata["elapsed_time"] = (
|
64
|
-
timeseries["time"].iloc[-1] - timeseries["time"].iloc[0]
|
65
|
-
)
|
66
|
-
metadata["distance_km"] = timeseries["distance_km"].iloc[-1]
|
67
|
-
if "calories" in timeseries.columns:
|
68
|
-
metadata["calories"] = timeseries["calories"].iloc[-1]
|
69
|
-
|
70
63
|
return metadata, timeseries
|
71
64
|
|
72
65
|
|
@@ -4,44 +4,58 @@ import multiprocessing
|
|
4
4
|
import pathlib
|
5
5
|
import pickle
|
6
6
|
import re
|
7
|
-
import sys
|
8
7
|
import traceback
|
9
|
-
from typing import Any
|
10
8
|
from typing import Optional
|
11
9
|
|
12
|
-
import pandas as pd
|
13
10
|
from tqdm import tqdm
|
14
11
|
|
15
12
|
from geo_activity_playground.core.activities import ActivityMeta
|
16
|
-
from geo_activity_playground.core.
|
17
|
-
from geo_activity_playground.core.
|
18
|
-
from geo_activity_playground.core.
|
13
|
+
from geo_activity_playground.core.paths import activity_extracted_dir
|
14
|
+
from geo_activity_playground.core.paths import activity_extracted_meta_dir
|
15
|
+
from geo_activity_playground.core.paths import activity_extracted_time_series_dir
|
16
|
+
from geo_activity_playground.core.tasks import stored_object
|
19
17
|
from geo_activity_playground.core.tasks import WorkTracker
|
18
|
+
from geo_activity_playground.importers.activity_parsers import ActivityParseError
|
19
|
+
from geo_activity_playground.importers.activity_parsers import read_activity
|
20
20
|
|
21
21
|
logger = logging.getLogger(__name__)
|
22
22
|
|
23
23
|
ACTIVITY_DIR = pathlib.Path("Activities")
|
24
24
|
|
25
25
|
|
26
|
-
def import_from_directory(
|
27
|
-
repository: ActivityRepository,
|
28
|
-
kind_defaults: dict[str, Any] = {},
|
29
|
-
metadata_extraction_regexes: list[str] = [],
|
30
|
-
) -> None:
|
31
|
-
paths_with_errors = []
|
32
|
-
work_tracker = WorkTracker("parse-activity-files")
|
26
|
+
def import_from_directory(metadata_extraction_regexes: list[str] = []) -> None:
|
33
27
|
|
34
28
|
activity_paths = [
|
35
29
|
path
|
36
30
|
for path in ACTIVITY_DIR.rglob("*.*")
|
37
31
|
if path.is_file() and path.suffixes and not path.stem.startswith(".")
|
38
32
|
]
|
33
|
+
work_tracker = WorkTracker(activity_extracted_dir() / "work-tracker-extract.pickle")
|
39
34
|
new_activity_paths = work_tracker.filter(activity_paths)
|
40
35
|
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
36
|
+
with stored_object(
|
37
|
+
activity_extracted_dir() / "file-hashes.pickle", {}
|
38
|
+
) as file_hashes:
|
39
|
+
for path in tqdm(new_activity_paths, desc="Detect deleted activities"):
|
40
|
+
file_hashes[path] = get_file_hash(path)
|
41
|
+
|
42
|
+
deleted_files = set(file_hashes.keys()) - set(activity_paths)
|
43
|
+
deleted_hashes = [file_hashes[path] for path in deleted_files]
|
44
|
+
for deleted_hash in deleted_hashes:
|
45
|
+
activity_extracted_meta_path = (
|
46
|
+
activity_extracted_meta_dir() / f"{deleted_hash}.pickle"
|
47
|
+
)
|
48
|
+
activity_extracted_time_series_path = (
|
49
|
+
activity_extracted_time_series_dir() / f"{deleted_hash}.parquet"
|
50
|
+
)
|
51
|
+
logger.warning(f"Deleting {activity_extracted_meta_path}")
|
52
|
+
logger.warning(f"Deleting {activity_extracted_time_series_path}")
|
53
|
+
activity_extracted_meta_path.unlink(missing_ok=True)
|
54
|
+
activity_extracted_time_series_path.unlink(missing_ok=True)
|
55
|
+
for deleted_file in deleted_files:
|
56
|
+
logger.warning(f"Deleting {deleted_file}")
|
57
|
+
del file_hashes[deleted_file]
|
58
|
+
work_tracker.discard(deleted_file)
|
45
59
|
|
46
60
|
with multiprocessing.Pool() as pool:
|
47
61
|
paths_with_errors = tqdm(
|
@@ -53,7 +67,7 @@ def import_from_directory(
|
|
53
67
|
|
54
68
|
for path in tqdm(new_activity_paths, desc="Collate activity metadata"):
|
55
69
|
activity_id = get_file_hash(path)
|
56
|
-
file_metadata_path =
|
70
|
+
file_metadata_path = activity_extracted_meta_dir() / f"{activity_id}.pickle"
|
57
71
|
work_tracker.mark_done(path)
|
58
72
|
|
59
73
|
if not file_metadata_path.exists():
|
@@ -73,8 +87,8 @@ def import_from_directory(
|
|
73
87
|
)
|
74
88
|
activity_meta.update(activity_meta_from_file)
|
75
89
|
activity_meta.update(_get_metadata_from_path(path, metadata_extraction_regexes))
|
76
|
-
|
77
|
-
|
90
|
+
with open(file_metadata_path, "wb") as f:
|
91
|
+
pickle.dump(activity_meta, f)
|
78
92
|
|
79
93
|
if paths_with_errors:
|
80
94
|
logger.warning(
|
@@ -83,18 +97,13 @@ def import_from_directory(
|
|
83
97
|
for path, error in paths_with_errors:
|
84
98
|
logger.error(f"{path}: {error}")
|
85
99
|
|
86
|
-
repository.commit()
|
87
|
-
|
88
100
|
work_tracker.close()
|
89
101
|
|
90
102
|
|
91
103
|
def _cache_single_file(path: pathlib.Path) -> Optional[tuple[pathlib.Path, str]]:
|
92
|
-
activity_stream_dir = pathlib.Path("Cache/Activity Timeseries")
|
93
|
-
file_metadata_dir = pathlib.Path("Cache/Activity Metadata")
|
94
|
-
|
95
104
|
activity_id = get_file_hash(path)
|
96
|
-
timeseries_path =
|
97
|
-
file_metadata_path =
|
105
|
+
timeseries_path = activity_extracted_time_series_dir() / f"{activity_id}.parquet"
|
106
|
+
file_metadata_path = activity_extracted_meta_dir() / f"{activity_id}.pickle"
|
98
107
|
|
99
108
|
if not timeseries_path.exists():
|
100
109
|
try:
|
@@ -14,9 +14,13 @@ from stravalib.exc import ObjectNotFound
|
|
14
14
|
from stravalib.exc import RateLimitExceeded
|
15
15
|
from tqdm import tqdm
|
16
16
|
|
17
|
-
from geo_activity_playground.core.activities import
|
17
|
+
from geo_activity_playground.core.activities import ActivityMeta
|
18
18
|
from geo_activity_playground.core.config import get_config
|
19
|
+
from geo_activity_playground.core.paths import activity_extracted_meta_dir
|
20
|
+
from geo_activity_playground.core.paths import activity_extracted_time_series_dir
|
19
21
|
from geo_activity_playground.core.paths import cache_dir
|
22
|
+
from geo_activity_playground.core.paths import strava_api_dir
|
23
|
+
from geo_activity_playground.core.paths import strava_dynamic_config_path
|
20
24
|
from geo_activity_playground.core.time_conversion import convert_to_datetime_ns
|
21
25
|
|
22
26
|
|
@@ -35,31 +39,22 @@ def set_state(path: pathlib.Path, state: Any) -> None:
|
|
35
39
|
json.dump(state, f, indent=2, sort_keys=True, ensure_ascii=False)
|
36
40
|
|
37
41
|
|
38
|
-
@functools.cache
|
39
|
-
def strava_api_dir() -> pathlib.Path:
|
40
|
-
result = pathlib.Path.cwd() / "Strava API"
|
41
|
-
result.mkdir(exist_ok=True, parents=True)
|
42
|
-
return result
|
43
|
-
|
44
|
-
|
45
|
-
@functools.cache
|
46
|
-
def activity_stream_dir() -> pathlib.Path:
|
47
|
-
path = pathlib.Path("Cache/Activity Timeseries")
|
48
|
-
path.mkdir(exist_ok=True, parents=True)
|
49
|
-
return path
|
50
|
-
|
51
|
-
|
52
42
|
def get_current_access_token() -> str:
|
53
|
-
|
43
|
+
if strava_dynamic_config_path().exists():
|
44
|
+
with open(strava_dynamic_config_path()) as f:
|
45
|
+
strava_config = json.load(f)
|
46
|
+
else:
|
47
|
+
config = get_config()
|
48
|
+
strava_config = config["strava"]
|
54
49
|
|
55
50
|
tokens = get_state(strava_api_dir() / "strava_tokens.json")
|
56
51
|
if not tokens:
|
57
52
|
logger.info("Create Strava access token …")
|
58
53
|
client = Client()
|
59
54
|
token_response = client.exchange_code_for_token(
|
60
|
-
client_id=
|
61
|
-
client_secret=
|
62
|
-
code=
|
55
|
+
client_id=strava_config["client_id"],
|
56
|
+
client_secret=strava_config["client_secret"],
|
57
|
+
code=strava_config["code"],
|
63
58
|
)
|
64
59
|
tokens = {
|
65
60
|
"access": token_response["access_token"],
|
@@ -71,8 +66,8 @@ def get_current_access_token() -> str:
|
|
71
66
|
logger.info("Renew Strava access token …")
|
72
67
|
client = Client()
|
73
68
|
token_response = client.refresh_access_token(
|
74
|
-
client_id=
|
75
|
-
client_secret=
|
69
|
+
client_id=strava_config["client_id"],
|
70
|
+
client_secret=strava_config["client_secret"],
|
76
71
|
refresh_token=tokens["refresh"],
|
77
72
|
)
|
78
73
|
tokens = {
|
@@ -94,8 +89,8 @@ def round_to_next_quarter_hour(date: datetime.datetime) -> datetime.datetime:
|
|
94
89
|
return next_quarter
|
95
90
|
|
96
91
|
|
97
|
-
def import_from_strava_api(
|
98
|
-
while try_import_strava(
|
92
|
+
def import_from_strava_api() -> None:
|
93
|
+
while try_import_strava():
|
99
94
|
now = datetime.datetime.now()
|
100
95
|
next_quarter = round_to_next_quarter_hour(now)
|
101
96
|
seconds_to_wait = (next_quarter - now).total_seconds() + 10
|
@@ -105,12 +100,13 @@ def import_from_strava_api(repository: ActivityRepository) -> None:
|
|
105
100
|
time.sleep(seconds_to_wait)
|
106
101
|
|
107
102
|
|
108
|
-
def try_import_strava(
|
109
|
-
|
110
|
-
if
|
111
|
-
|
103
|
+
def try_import_strava() -> bool:
|
104
|
+
last_activity_date_path = cache_dir() / "strava-last-activity-date.json"
|
105
|
+
if last_activity_date_path.exists():
|
106
|
+
with open(last_activity_date_path) as f:
|
107
|
+
get_after = json.load(f)
|
112
108
|
else:
|
113
|
-
get_after =
|
109
|
+
get_after = "2000-01-01T00:00:00Z"
|
114
110
|
|
115
111
|
gear_names = {None: "None"}
|
116
112
|
|
@@ -120,22 +116,26 @@ def try_import_strava(repository: ActivityRepository) -> bool:
|
|
120
116
|
for activity in tqdm(
|
121
117
|
client.get_activities(after=get_after), desc="Downloading Strava activities"
|
122
118
|
):
|
123
|
-
# Sometimes we still get an activity here although it has already been imported from the Strava checkout.
|
124
|
-
if repository.has_activity(activity.id):
|
125
|
-
continue
|
126
119
|
cache_file = (
|
127
|
-
pathlib.Path("Cache")
|
120
|
+
pathlib.Path("Cache")
|
121
|
+
/ "Strava Activity Metadata"
|
122
|
+
/ f"{activity.id}.pickle"
|
128
123
|
)
|
124
|
+
# Sometimes we still get an activity here although it has already been imported from the Strava checkout.
|
125
|
+
if cache_file.exists():
|
126
|
+
continue
|
129
127
|
cache_file.parent.mkdir(exist_ok=True, parents=True)
|
130
128
|
with open(cache_file, "wb") as f:
|
131
129
|
pickle.dump(activity, f)
|
132
|
-
if
|
130
|
+
if activity.gear_id not in gear_names:
|
133
131
|
gear = client.get_gear(activity.gear_id)
|
134
132
|
gear_names[activity.gear_id] = (
|
135
133
|
f"{gear.name}" or f"{gear.brand_name} {gear.model_name}"
|
136
134
|
)
|
137
135
|
|
138
|
-
time_series_path =
|
136
|
+
time_series_path = (
|
137
|
+
activity_extracted_time_series_dir() / f"{activity.id}.parquet"
|
138
|
+
)
|
139
139
|
if time_series_path.exists():
|
140
140
|
time_series = pd.read_parquet(time_series_path)
|
141
141
|
else:
|
@@ -159,8 +159,8 @@ def try_import_strava(repository: ActivityRepository) -> bool:
|
|
159
159
|
detailed_activity = get_detailed_activity(activity.id, client)
|
160
160
|
|
161
161
|
if len(time_series) > 0 and "latitude" in time_series.columns:
|
162
|
-
|
163
|
-
{
|
162
|
+
activity_meta = ActivityMeta(
|
163
|
+
**{
|
164
164
|
"id": activity.id,
|
165
165
|
"commute": activity.commute,
|
166
166
|
"distance_km": activity.distance.magnitude / 1000,
|
@@ -170,8 +170,17 @@ def try_import_strava(repository: ActivityRepository) -> bool:
|
|
170
170
|
"elapsed_time": activity.elapsed_time,
|
171
171
|
"equipment": gear_names[activity.gear_id],
|
172
172
|
"calories": detailed_activity.calories,
|
173
|
+
"moving_time": activity.moving_time,
|
173
174
|
}
|
174
175
|
)
|
176
|
+
with open(
|
177
|
+
activity_extracted_meta_dir() / f"{activity.id}.pickle", "wb"
|
178
|
+
) as f:
|
179
|
+
pickle.dump(activity_meta, f)
|
180
|
+
|
181
|
+
with open(last_activity_date_path, "w") as f:
|
182
|
+
json.dump(activity.start_date.isoformat().replace("+00:00", "Z"), f)
|
183
|
+
|
175
184
|
limit_exceeded = False
|
176
185
|
except RateLimitExceeded:
|
177
186
|
limit_exceeded = True
|
@@ -181,8 +190,6 @@ def try_import_strava(repository: ActivityRepository) -> bool:
|
|
181
190
|
else:
|
182
191
|
raise
|
183
192
|
|
184
|
-
repository.commit()
|
185
|
-
|
186
193
|
return limit_exceeded
|
187
194
|
|
188
195
|
|
@@ -1,6 +1,7 @@
|
|
1
1
|
import datetime
|
2
2
|
import logging
|
3
3
|
import pathlib
|
4
|
+
import pickle
|
4
5
|
import shutil
|
5
6
|
import sys
|
6
7
|
import traceback
|
@@ -12,11 +13,13 @@ import numpy as np
|
|
12
13
|
import pandas as pd
|
13
14
|
from tqdm import tqdm
|
14
15
|
|
15
|
-
from geo_activity_playground.core.
|
16
|
-
from geo_activity_playground.core.
|
17
|
-
from geo_activity_playground.core.
|
16
|
+
from geo_activity_playground.core.paths import activity_extracted_meta_dir
|
17
|
+
from geo_activity_playground.core.paths import activity_extracted_time_series_dir
|
18
|
+
from geo_activity_playground.core.tasks import work_tracker_path
|
18
19
|
from geo_activity_playground.core.tasks import WorkTracker
|
19
20
|
from geo_activity_playground.core.time_conversion import convert_to_datetime_ns
|
21
|
+
from geo_activity_playground.importers.activity_parsers import ActivityParseError
|
22
|
+
from geo_activity_playground.importers.activity_parsers import read_activity
|
20
23
|
|
21
24
|
|
22
25
|
logger = logging.getLogger(__name__)
|
@@ -116,6 +119,13 @@ EXPECTED_COLUMNS = [
|
|
116
119
|
"Newly Explored Dirt Distance",
|
117
120
|
"Activity Count",
|
118
121
|
"Total Steps",
|
122
|
+
"Carbon Saved",
|
123
|
+
"Pool Length",
|
124
|
+
"Training Load",
|
125
|
+
"Intensity",
|
126
|
+
"Average Grade Adjusted Pace",
|
127
|
+
"Timer Time",
|
128
|
+
"Total Cycles",
|
119
129
|
"Media",
|
120
130
|
]
|
121
131
|
|
@@ -127,7 +137,7 @@ def float_or_none(x: Union[float, str]) -> Optional[float]:
|
|
127
137
|
return None
|
128
138
|
|
129
139
|
|
130
|
-
def import_from_strava_checkout(
|
140
|
+
def import_from_strava_checkout() -> None:
|
131
141
|
checkout_path = pathlib.Path("Strava Export")
|
132
142
|
activities = pd.read_csv(checkout_path / "activities.csv")
|
133
143
|
|
@@ -146,19 +156,20 @@ def import_from_strava_checkout(repository: ActivityRepository) -> None:
|
|
146
156
|
dayfirst = True
|
147
157
|
|
148
158
|
activities.index = activities["Activity ID"]
|
149
|
-
work_tracker = WorkTracker("import-strava-checkout-activities")
|
159
|
+
work_tracker = WorkTracker(work_tracker_path("import-strava-checkout-activities"))
|
150
160
|
activities_ids_to_parse = work_tracker.filter(activities["Activity ID"])
|
151
161
|
activities_ids_to_parse = [
|
152
162
|
activity_id
|
153
163
|
for activity_id in activities_ids_to_parse
|
154
|
-
if not
|
164
|
+
if not (activity_extracted_meta_dir() / f"{activity_id.pickle}").exists()
|
155
165
|
]
|
156
166
|
|
157
|
-
activity_stream_dir = pathlib.Path("Cache/Activity Timeseries")
|
158
|
-
activity_stream_dir.mkdir(exist_ok=True, parents=True)
|
159
|
-
|
160
167
|
for activity_id in tqdm(activities_ids_to_parse, desc="Import from Strava export"):
|
168
|
+
work_tracker.mark_done(activity_id)
|
161
169
|
row = activities.loc[activity_id]
|
170
|
+
# Some manually recorded activities have no file name. Pandas reads that as a float. We skip those.
|
171
|
+
if isinstance(row["Filename"], float):
|
172
|
+
continue
|
162
173
|
activity_file = checkout_path / row["Filename"]
|
163
174
|
table_activity_meta = {
|
164
175
|
"calories": float_or_none(row["Calories"]),
|
@@ -179,8 +190,11 @@ def import_from_strava_checkout(repository: ActivityRepository) -> None:
|
|
179
190
|
dateutil.parser.parse(row["Activity Date"], dayfirst=dayfirst)
|
180
191
|
),
|
181
192
|
}
|
193
|
+
meta_path = activity_extracted_meta_dir / f"{activity_id}.pickle"
|
194
|
+
with open(meta_path, "wb") as f:
|
195
|
+
pickle.dump(table_activity_meta, meta_path)
|
182
196
|
|
183
|
-
time_series_path =
|
197
|
+
time_series_path = activity_extracted_time_series_dir / f"{activity_id}.parquet"
|
184
198
|
if time_series_path.exists():
|
185
199
|
time_series = pd.read_parquet(time_series_path)
|
186
200
|
else:
|
@@ -196,8 +210,6 @@ def import_from_strava_checkout(repository: ActivityRepository) -> None:
|
|
196
210
|
)
|
197
211
|
raise
|
198
212
|
|
199
|
-
work_tracker.mark_done(activity_id)
|
200
|
-
|
201
213
|
if not len(time_series):
|
202
214
|
continue
|
203
215
|
|
@@ -205,10 +217,6 @@ def import_from_strava_checkout(repository: ActivityRepository) -> None:
|
|
205
217
|
continue
|
206
218
|
|
207
219
|
time_series.to_parquet(time_series_path)
|
208
|
-
|
209
|
-
repository.add_activity(table_activity_meta)
|
210
|
-
|
211
|
-
repository.commit()
|
212
220
|
work_tracker.close()
|
213
221
|
|
214
222
|
|
@@ -452,9 +452,9 @@ def make_sharepic(activity: ActivityMeta, time_series: pd.DataFrame) -> bytes:
|
|
452
452
|
f"\n{activity['distance_km']:.1f} km",
|
453
453
|
re.sub(r"^0 days ", "", f"{activity['elapsed_time']}"),
|
454
454
|
]
|
455
|
-
if activity
|
455
|
+
if activity.get("calories", 0) and not pd.isna(activity["calories"]):
|
456
456
|
facts.append(f"{activity['calories']:.0f} kcal")
|
457
|
-
if activity
|
457
|
+
if activity.get("steps", 0) and not pd.isna(activity["steps"]):
|
458
458
|
facts.append(f"{activity['steps']:.0f} steps")
|
459
459
|
|
460
460
|
draw.text((35, img.height - 70 + 10), " ".join(facts), font_size=20)
|
@@ -20,6 +20,8 @@
|
|
20
20
|
<dd>{{ activity.distance_km|round(1) }} km</dd>
|
21
21
|
<dt>Elapsed time</dt>
|
22
22
|
<dd>{{ activity.elapsed_time }}</dd>
|
23
|
+
<dt>Moving time</dt>
|
24
|
+
<dd>{{ activity.moving_time }}</dd>
|
23
25
|
<dt>Start time</dt>
|
24
26
|
<dd><a href="{{ url_for('activity.day', year=date.year, month=date.month, day=date.day) }}">{{ date }}</a>
|
25
27
|
{{ time }}
|