mapillary-tools 0.14.0__tar.gz → 0.14.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/PKG-INFO +1 -1
- mapillary_tools-0.14.4/mapillary_tools/__init__.py +1 -0
- mapillary_tools-0.14.4/mapillary_tools/api_v4.py +176 -0
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/authenticate.py +41 -34
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/commands/__main__.py +8 -12
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/commands/upload.py +7 -0
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/config.py +5 -0
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/constants.py +5 -3
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/exif_read.py +2 -1
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/geotag/base.py +6 -2
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/geotag/factory.py +2 -1
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/geotag/geotag_images_from_exiftool.py +1 -1
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/geotag/geotag_videos_from_exiftool.py +30 -9
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/geotag/options.py +4 -1
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/geotag/utils.py +9 -12
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/geotag/video_extractors/gpx.py +2 -1
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/history.py +88 -53
- mapillary_tools-0.14.4/mapillary_tools/http.py +211 -0
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/process_geotag_properties.py +12 -14
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/sample_video.py +1 -2
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/serializer/description.py +12 -2
- mapillary_tools-0.14.4/mapillary_tools/store.py +128 -0
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/upload.py +47 -47
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/upload_api_v4.py +8 -20
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/uploader.py +576 -171
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/utils.py +50 -5
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools.egg-info/PKG-INFO +1 -1
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools.egg-info/SOURCES.txt +2 -0
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/pyproject.toml +1 -0
- mapillary_tools-0.14.0/mapillary_tools/__init__.py +0 -1
- mapillary_tools-0.14.0/mapillary_tools/api_v4.py +0 -416
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/LICENSE +0 -0
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/README.md +0 -0
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/blackvue_parser.py +0 -0
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/camm/camm_builder.py +0 -0
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/camm/camm_parser.py +0 -0
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/commands/__init__.py +0 -0
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/commands/authenticate.py +0 -0
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/commands/process.py +0 -0
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/commands/process_and_upload.py +0 -0
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/commands/sample_video.py +0 -0
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/commands/video_process.py +0 -0
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/commands/video_process_and_upload.py +0 -0
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/commands/zip.py +0 -0
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/exceptions.py +0 -0
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/exif_write.py +0 -0
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/exiftool_read.py +0 -0
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/exiftool_read_video.py +0 -0
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/exiftool_runner.py +0 -0
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/ffmpeg.py +0 -0
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/geo.py +0 -0
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/geotag/__init__.py +0 -0
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/geotag/geotag_images_from_exif.py +0 -0
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/geotag/geotag_images_from_gpx.py +0 -0
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/geotag/geotag_images_from_gpx_file.py +0 -0
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/geotag/geotag_images_from_nmea_file.py +0 -0
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/geotag/geotag_images_from_video.py +0 -0
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/geotag/geotag_videos_from_gpx.py +0 -0
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/geotag/geotag_videos_from_video.py +0 -0
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/geotag/image_extractors/base.py +0 -0
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/geotag/image_extractors/exif.py +0 -0
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/geotag/image_extractors/exiftool.py +0 -0
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/geotag/video_extractors/base.py +0 -0
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/geotag/video_extractors/exiftool.py +0 -0
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/geotag/video_extractors/native.py +0 -0
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/gpmf/gpmf_gps_filter.py +0 -0
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/gpmf/gpmf_parser.py +0 -0
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/gpmf/gps_filter.py +0 -0
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/ipc.py +0 -0
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/mp4/__init__.py +0 -0
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/mp4/construct_mp4_parser.py +0 -0
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/mp4/io_utils.py +0 -0
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/mp4/mp4_sample_parser.py +0 -0
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/mp4/simple_mp4_builder.py +0 -0
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/mp4/simple_mp4_parser.py +0 -0
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/process_sequence_properties.py +0 -0
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/serializer/gpx.py +0 -0
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/telemetry.py +0 -0
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/types.py +0 -0
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools.egg-info/dependency_links.txt +0 -0
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools.egg-info/entry_points.txt +0 -0
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools.egg-info/requires.txt +0 -0
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools.egg-info/top_level.txt +0 -0
- {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/setup.cfg +0 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
VERSION = "0.14.4"
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import enum
|
|
4
|
+
import logging
|
|
5
|
+
import os
|
|
6
|
+
import typing as T
|
|
7
|
+
|
|
8
|
+
import requests
|
|
9
|
+
|
|
10
|
+
from . import http
|
|
11
|
+
|
|
12
|
+
LOG = logging.getLogger(__name__)
|
|
13
|
+
MAPILLARY_CLIENT_TOKEN = os.getenv(
|
|
14
|
+
"MAPILLARY_CLIENT_TOKEN", "MLY|5675152195860640|6b02c72e6e3c801e5603ab0495623282"
|
|
15
|
+
)
|
|
16
|
+
MAPILLARY_GRAPH_API_ENDPOINT = os.getenv(
|
|
17
|
+
"MAPILLARY_GRAPH_API_ENDPOINT", "https://graph.mapillary.com"
|
|
18
|
+
)
|
|
19
|
+
REQUESTS_TIMEOUT: float = 60 # 1 minutes
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class HTTPContentError(Exception):
|
|
23
|
+
"""
|
|
24
|
+
Raised when the HTTP response is ok (200) but the content is not as expected
|
|
25
|
+
e.g. not JSON or not a valid response.
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
def __init__(self, message: str, response: requests.Response):
|
|
29
|
+
self.response = response
|
|
30
|
+
super().__init__(message)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class ClusterFileType(enum.Enum):
|
|
34
|
+
ZIP = "zip"
|
|
35
|
+
BLACKVUE = "mly_blackvue_video"
|
|
36
|
+
CAMM = "mly_camm_video"
|
|
37
|
+
MLY_BUNDLE_MANIFEST = "mly_bundle_manifest"
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def create_user_session(user_access_token: str) -> requests.Session:
|
|
41
|
+
session = http.Session()
|
|
42
|
+
session.headers["Authorization"] = f"OAuth {user_access_token}"
|
|
43
|
+
return session
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def create_client_session(disable_logging: bool = False) -> requests.Session:
|
|
47
|
+
session = http.Session()
|
|
48
|
+
session.headers["Authorization"] = f"OAuth {MAPILLARY_CLIENT_TOKEN}"
|
|
49
|
+
if disable_logging:
|
|
50
|
+
session.disable_logging_request = True
|
|
51
|
+
session.disable_logging_response = True
|
|
52
|
+
return session
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def is_auth_error(resp: requests.Response) -> bool:
|
|
56
|
+
if resp.status_code in [401, 403]:
|
|
57
|
+
return True
|
|
58
|
+
|
|
59
|
+
if resp.status_code in [400]:
|
|
60
|
+
try:
|
|
61
|
+
error_body = resp.json()
|
|
62
|
+
except Exception:
|
|
63
|
+
error_body = {}
|
|
64
|
+
|
|
65
|
+
type = error_body.get("debug_info", {}).get("type")
|
|
66
|
+
if type in ["NotAuthorizedError"]:
|
|
67
|
+
return True
|
|
68
|
+
|
|
69
|
+
return False
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def extract_auth_error_message(resp: requests.Response) -> str:
|
|
73
|
+
assert is_auth_error(resp), "has to be an auth error"
|
|
74
|
+
|
|
75
|
+
try:
|
|
76
|
+
error_body = resp.json()
|
|
77
|
+
except Exception:
|
|
78
|
+
error_body = {}
|
|
79
|
+
|
|
80
|
+
# from Graph APIs
|
|
81
|
+
message = error_body.get("error", {}).get("message")
|
|
82
|
+
if message is not None:
|
|
83
|
+
return str(message)
|
|
84
|
+
|
|
85
|
+
# from upload service
|
|
86
|
+
message = error_body.get("debug_info", {}).get("message")
|
|
87
|
+
if message is not None:
|
|
88
|
+
return str(message)
|
|
89
|
+
|
|
90
|
+
return resp.text
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def get_upload_token(
|
|
94
|
+
client_session: requests.Session, email: str, password: str
|
|
95
|
+
) -> requests.Response:
|
|
96
|
+
url = f"{MAPILLARY_GRAPH_API_ENDPOINT}/login"
|
|
97
|
+
json_data = {"email": email, "password": password, "locale": "en_US"}
|
|
98
|
+
|
|
99
|
+
resp = client_session.post(url, json=json_data, timeout=REQUESTS_TIMEOUT)
|
|
100
|
+
resp.raise_for_status()
|
|
101
|
+
|
|
102
|
+
return resp
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def fetch_organization(
|
|
106
|
+
user_session: requests.Session, organization_id: int | str
|
|
107
|
+
) -> requests.Response:
|
|
108
|
+
url = f"{MAPILLARY_GRAPH_API_ENDPOINT}/{organization_id}"
|
|
109
|
+
params = {"fields": ",".join(["slug", "description", "name"])}
|
|
110
|
+
|
|
111
|
+
resp = user_session.get(url, params=params, timeout=REQUESTS_TIMEOUT)
|
|
112
|
+
resp.raise_for_status()
|
|
113
|
+
|
|
114
|
+
return resp
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def fetch_user_or_me(
|
|
118
|
+
user_session: requests.Session, user_id: int | str | None = None
|
|
119
|
+
) -> requests.Response:
|
|
120
|
+
if user_id is None:
|
|
121
|
+
url = f"{MAPILLARY_GRAPH_API_ENDPOINT}/me"
|
|
122
|
+
else:
|
|
123
|
+
url = f"{MAPILLARY_GRAPH_API_ENDPOINT}/{user_id}"
|
|
124
|
+
params = {"fields": ",".join(["id", "username"])}
|
|
125
|
+
|
|
126
|
+
resp = user_session.get(url, params=params, timeout=REQUESTS_TIMEOUT)
|
|
127
|
+
resp.raise_for_status()
|
|
128
|
+
|
|
129
|
+
return resp
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
ActionType = T.Literal[
|
|
133
|
+
"upload_started_upload", "upload_finished_upload", "upload_failed_upload"
|
|
134
|
+
]
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def log_event(
|
|
138
|
+
client_session: requests.Session, action_type: ActionType, properties: dict
|
|
139
|
+
) -> requests.Response:
|
|
140
|
+
url = f"{MAPILLARY_GRAPH_API_ENDPOINT}/logging"
|
|
141
|
+
json_data = {"action_type": action_type, "properties": properties}
|
|
142
|
+
|
|
143
|
+
resp = client_session.post(url, json=json_data, timeout=REQUESTS_TIMEOUT)
|
|
144
|
+
resp.raise_for_status()
|
|
145
|
+
|
|
146
|
+
return resp
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def finish_upload(
|
|
150
|
+
user_session: requests.Session,
|
|
151
|
+
file_handle: str,
|
|
152
|
+
cluster_filetype: ClusterFileType,
|
|
153
|
+
organization_id: int | str | None = None,
|
|
154
|
+
) -> requests.Response:
|
|
155
|
+
url = f"{MAPILLARY_GRAPH_API_ENDPOINT}/finish_upload"
|
|
156
|
+
json_data: dict[str, str | int] = {
|
|
157
|
+
"file_handle": file_handle,
|
|
158
|
+
"file_type": cluster_filetype.value,
|
|
159
|
+
}
|
|
160
|
+
if organization_id is not None:
|
|
161
|
+
json_data["organization_id"] = organization_id
|
|
162
|
+
|
|
163
|
+
resp = user_session.post(url, json=json_data, timeout=REQUESTS_TIMEOUT)
|
|
164
|
+
resp.raise_for_status()
|
|
165
|
+
|
|
166
|
+
return resp
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def jsonify_response(resp: requests.Response) -> T.Any:
|
|
170
|
+
"""
|
|
171
|
+
Convert the response to JSON, raising HTTPContentError if the response is not JSON.
|
|
172
|
+
"""
|
|
173
|
+
try:
|
|
174
|
+
return resp.json()
|
|
175
|
+
except requests.JSONDecodeError as ex:
|
|
176
|
+
raise HTTPContentError("Invalid JSON response", resp) from ex
|
|
@@ -10,7 +10,7 @@ import jsonschema
|
|
|
10
10
|
|
|
11
11
|
import requests
|
|
12
12
|
|
|
13
|
-
from . import api_v4, config, constants, exceptions
|
|
13
|
+
from . import api_v4, config, constants, exceptions, http
|
|
14
14
|
|
|
15
15
|
|
|
16
16
|
LOG = logging.getLogger(__name__)
|
|
@@ -77,11 +77,11 @@ def authenticate(
|
|
|
77
77
|
# TODO: print more user information
|
|
78
78
|
if profile_name in all_user_items:
|
|
79
79
|
LOG.info(
|
|
80
|
-
'Profile "%s" updated: %s', profile_name,
|
|
80
|
+
'Profile "%s" updated: %s', profile_name, http._sanitize(user_items)
|
|
81
81
|
)
|
|
82
82
|
else:
|
|
83
83
|
LOG.info(
|
|
84
|
-
'Profile "%s" created: %s', profile_name,
|
|
84
|
+
'Profile "%s" created: %s', profile_name, http._sanitize(user_items)
|
|
85
85
|
)
|
|
86
86
|
|
|
87
87
|
|
|
@@ -127,21 +127,28 @@ def fetch_user_items(
|
|
|
127
127
|
|
|
128
128
|
assert profile_name is not None, "profile_name should be set"
|
|
129
129
|
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
f'Uploading to profile "{profile_name}": {user_items.get("MAPSettingsUsername")} (ID: {user_items.get("MAPSettingsUserKey")})'
|
|
134
|
-
)
|
|
130
|
+
try:
|
|
131
|
+
LOG.info(f'Verifying profile "{profile_name}"...')
|
|
132
|
+
user_items = _verify_user_auth(_validate_profile(user_items))
|
|
135
133
|
|
|
136
|
-
if organization_key is not None:
|
|
137
|
-
resp = api_v4.fetch_organization(
|
|
138
|
-
user_items["user_upload_token"], organization_key
|
|
139
|
-
)
|
|
140
|
-
data = api_v4.jsonify_response(resp)
|
|
141
134
|
LOG.info(
|
|
142
|
-
f
|
|
135
|
+
f'Uploading to profile "{profile_name}": {user_items.get("MAPSettingsUsername")} (ID: {user_items.get("MAPSettingsUserKey")})'
|
|
143
136
|
)
|
|
144
|
-
|
|
137
|
+
|
|
138
|
+
if organization_key is not None:
|
|
139
|
+
with api_v4.create_user_session(user_items["user_upload_token"]) as session:
|
|
140
|
+
resp = api_v4.fetch_organization(session, organization_key)
|
|
141
|
+
data = api_v4.jsonify_response(resp)
|
|
142
|
+
LOG.info(
|
|
143
|
+
f"Uploading to organization: {data.get('name')} (ID: {data.get('id')})"
|
|
144
|
+
)
|
|
145
|
+
user_items["MAPOrganizationKey"] = data.get("id")
|
|
146
|
+
|
|
147
|
+
except requests.Timeout as ex:
|
|
148
|
+
raise exceptions.MapillaryUploadTimeoutError(str(ex)) from ex
|
|
149
|
+
|
|
150
|
+
except requests.ConnectionError as ex:
|
|
151
|
+
raise exceptions.MapillaryUploadConnectionError(str(ex)) from ex
|
|
145
152
|
|
|
146
153
|
return user_items
|
|
147
154
|
|
|
@@ -158,7 +165,7 @@ def _prompt(message: str) -> str:
|
|
|
158
165
|
|
|
159
166
|
def _validate_profile(user_items: config.UserItem) -> config.UserItem:
|
|
160
167
|
try:
|
|
161
|
-
|
|
168
|
+
config.UserItemSchemaValidator.validate(user_items)
|
|
162
169
|
except jsonschema.ValidationError as ex:
|
|
163
170
|
raise exceptions.MapillaryBadParameterError(
|
|
164
171
|
f"Invalid profile format: {ex.message}"
|
|
@@ -173,16 +180,15 @@ def _verify_user_auth(user_items: config.UserItem) -> config.UserItem:
|
|
|
173
180
|
if constants._AUTH_VERIFICATION_DISABLED:
|
|
174
181
|
return user_items
|
|
175
182
|
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
raise ex
|
|
183
|
+
with api_v4.create_user_session(user_items["user_upload_token"]) as session:
|
|
184
|
+
try:
|
|
185
|
+
resp = api_v4.fetch_user_or_me(session)
|
|
186
|
+
except requests.HTTPError as ex:
|
|
187
|
+
if api_v4.is_auth_error(ex.response):
|
|
188
|
+
message = api_v4.extract_auth_error_message(ex.response)
|
|
189
|
+
raise exceptions.MapillaryUploadUnauthorizedError(message)
|
|
190
|
+
else:
|
|
191
|
+
raise ex
|
|
186
192
|
|
|
187
193
|
data = api_v4.jsonify_response(resp)
|
|
188
194
|
|
|
@@ -276,16 +282,17 @@ def _prompt_login(
|
|
|
276
282
|
if user_password:
|
|
277
283
|
break
|
|
278
284
|
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
285
|
+
with api_v4.create_client_session() as session:
|
|
286
|
+
try:
|
|
287
|
+
resp = api_v4.get_upload_token(session, user_email, user_password)
|
|
288
|
+
except requests.HTTPError as ex:
|
|
289
|
+
if not _enabled:
|
|
290
|
+
raise ex
|
|
284
291
|
|
|
285
|
-
|
|
286
|
-
|
|
292
|
+
if _is_login_retryable(ex):
|
|
293
|
+
return _prompt_login()
|
|
287
294
|
|
|
288
|
-
|
|
295
|
+
raise ex
|
|
289
296
|
|
|
290
297
|
data = api_v4.jsonify_response(resp)
|
|
291
298
|
|
|
@@ -8,6 +8,7 @@ import requests
|
|
|
8
8
|
|
|
9
9
|
from .. import api_v4, constants, exceptions, VERSION
|
|
10
10
|
from ..upload import log_exception
|
|
11
|
+
from ..utils import configure_logger, get_app_name
|
|
11
12
|
from . import (
|
|
12
13
|
authenticate,
|
|
13
14
|
process,
|
|
@@ -31,8 +32,8 @@ mapillary_tools_commands = [
|
|
|
31
32
|
]
|
|
32
33
|
|
|
33
34
|
|
|
34
|
-
#
|
|
35
|
-
LOG = logging.getLogger(
|
|
35
|
+
# Root logger of mapillary_tools (not including third-party libraries)
|
|
36
|
+
LOG = logging.getLogger(get_app_name())
|
|
36
37
|
|
|
37
38
|
|
|
38
39
|
# Handle shared arguments/options here
|
|
@@ -79,13 +80,6 @@ def add_general_arguments(parser, command):
|
|
|
79
80
|
)
|
|
80
81
|
|
|
81
82
|
|
|
82
|
-
def configure_logger(logger: logging.Logger, stream=None) -> None:
|
|
83
|
-
formatter = logging.Formatter("%(asctime)s - %(levelname)-7s - %(message)s")
|
|
84
|
-
handler = logging.StreamHandler(stream)
|
|
85
|
-
handler.setFormatter(formatter)
|
|
86
|
-
logger.addHandler(handler)
|
|
87
|
-
|
|
88
|
-
|
|
89
83
|
def _log_params(argvars: dict) -> None:
|
|
90
84
|
MAX_ENTRIES = 5
|
|
91
85
|
|
|
@@ -152,9 +146,7 @@ def main():
|
|
|
152
146
|
|
|
153
147
|
args = parser.parse_args()
|
|
154
148
|
|
|
155
|
-
|
|
156
|
-
configure_logger(LOG, sys.stderr)
|
|
157
|
-
LOG.setLevel(log_level)
|
|
149
|
+
configure_logger(LOG, level=logging.DEBUG if args.verbose else logging.INFO)
|
|
158
150
|
|
|
159
151
|
LOG.debug("%s", version_text)
|
|
160
152
|
argvars = vars(args)
|
|
@@ -175,6 +167,10 @@ def main():
|
|
|
175
167
|
log_exception(ex)
|
|
176
168
|
sys.exit(ex.exit_code)
|
|
177
169
|
|
|
170
|
+
except KeyboardInterrupt:
|
|
171
|
+
LOG.info("Interrupted by user...")
|
|
172
|
+
sys.exit(130)
|
|
173
|
+
|
|
178
174
|
|
|
179
175
|
if __name__ == "__main__":
|
|
180
176
|
main()
|
|
@@ -23,6 +23,13 @@ class Command:
|
|
|
23
23
|
default=None,
|
|
24
24
|
required=False,
|
|
25
25
|
)
|
|
26
|
+
group.add_argument(
|
|
27
|
+
"--num_upload_workers",
|
|
28
|
+
help="Number of concurrent upload workers for uploading images. [default: %(default)s]",
|
|
29
|
+
default=constants.MAX_IMAGE_UPLOAD_WORKERS,
|
|
30
|
+
type=int,
|
|
31
|
+
required=False,
|
|
32
|
+
)
|
|
26
33
|
group.add_argument(
|
|
27
34
|
"--reupload",
|
|
28
35
|
help="Re-upload data that has already been uploaded.",
|
|
@@ -6,6 +6,8 @@ import sys
|
|
|
6
6
|
import typing as T
|
|
7
7
|
from typing import TypedDict
|
|
8
8
|
|
|
9
|
+
import jsonschema
|
|
10
|
+
|
|
9
11
|
if sys.version_info >= (3, 11):
|
|
10
12
|
from typing import Required
|
|
11
13
|
else:
|
|
@@ -50,6 +52,9 @@ UserItemSchema = {
|
|
|
50
52
|
}
|
|
51
53
|
|
|
52
54
|
|
|
55
|
+
UserItemSchemaValidator = jsonschema.Draft202012Validator(UserItemSchema)
|
|
56
|
+
|
|
57
|
+
|
|
53
58
|
def _load_config(config_path: str) -> configparser.ConfigParser:
|
|
54
59
|
config = configparser.ConfigParser()
|
|
55
60
|
# Override to not change option names (by default it will lower them)
|
|
@@ -154,16 +154,18 @@ UPLOAD_CACHE_DIR: str = os.getenv(
|
|
|
154
154
|
# The minimal upload speed is used to calculate the read timeout to avoid upload hanging:
|
|
155
155
|
# timeout = upload_size / MIN_UPLOAD_SPEED
|
|
156
156
|
MIN_UPLOAD_SPEED: int | None = _parse_filesize(
|
|
157
|
-
os.getenv(_ENV_PREFIX + "MIN_UPLOAD_SPEED", "50K") # 50
|
|
157
|
+
os.getenv(_ENV_PREFIX + "MIN_UPLOAD_SPEED", "50K") # 50 Kb/s
|
|
158
158
|
)
|
|
159
|
+
# Maximum number of parallel workers for uploading images within a single sequence.
|
|
160
|
+
# NOTE: Sequences themselves are uploaded sequentially, not in parallel.
|
|
159
161
|
MAX_IMAGE_UPLOAD_WORKERS: int = int(
|
|
160
|
-
os.getenv(_ENV_PREFIX + "MAX_IMAGE_UPLOAD_WORKERS",
|
|
162
|
+
os.getenv(_ENV_PREFIX + "MAX_IMAGE_UPLOAD_WORKERS", 4)
|
|
161
163
|
)
|
|
162
164
|
# The chunk size in MB (see chunked transfer encoding https://en.wikipedia.org/wiki/Chunked_transfer_encoding)
|
|
163
165
|
# for uploading data to MLY upload service.
|
|
164
166
|
# Changing this size does not change the number of requests nor affect upload performance,
|
|
165
167
|
# but it affects the responsiveness of the upload progress bar
|
|
166
|
-
UPLOAD_CHUNK_SIZE_MB: float = float(os.getenv(_ENV_PREFIX + "UPLOAD_CHUNK_SIZE_MB",
|
|
168
|
+
UPLOAD_CHUNK_SIZE_MB: float = float(os.getenv(_ENV_PREFIX + "UPLOAD_CHUNK_SIZE_MB", 2))
|
|
167
169
|
MAX_UPLOAD_RETRIES: int = int(os.getenv(_ENV_PREFIX + "MAX_UPLOAD_RETRIES", 200))
|
|
168
170
|
MAPILLARY__ENABLE_UPLOAD_HISTORY_FOR_DRY_RUN: bool = _yes_or_no(
|
|
169
171
|
os.getenv("MAPILLARY__ENABLE_UPLOAD_HISTORY_FOR_DRY_RUN", "NO")
|
|
@@ -871,7 +871,8 @@ class ExifRead(ExifReadFromEXIF):
|
|
|
871
871
|
|
|
872
872
|
def _xmp_with_reason(self, reason: str) -> ExifReadFromXMP | None:
|
|
873
873
|
if not self._xml_extracted:
|
|
874
|
-
|
|
874
|
+
# TODO Disabled because too verbose but still useful to know
|
|
875
|
+
# LOG.debug('Extracting XMP for "%s"', reason)
|
|
875
876
|
self._cached_xml = self._extract_xmp()
|
|
876
877
|
self._xml_extracted = True
|
|
877
878
|
|
|
@@ -46,7 +46,7 @@ class GeotagImagesFromGeneric(abc.ABC, T.Generic[TImageExtractor]):
|
|
|
46
46
|
map_results,
|
|
47
47
|
desc="Extracting images",
|
|
48
48
|
unit="images",
|
|
49
|
-
disable=LOG.
|
|
49
|
+
disable=LOG.isEnabledFor(logging.DEBUG),
|
|
50
50
|
total=len(extractors),
|
|
51
51
|
)
|
|
52
52
|
)
|
|
@@ -62,6 +62,8 @@ class GeotagImagesFromGeneric(abc.ABC, T.Generic[TImageExtractor]):
|
|
|
62
62
|
try:
|
|
63
63
|
return extractor.extract()
|
|
64
64
|
except exceptions.MapillaryDescriptionError as ex:
|
|
65
|
+
if LOG.isEnabledFor(logging.DEBUG):
|
|
66
|
+
LOG.error(f"{cls.__name__}({image_path.name}): {ex}")
|
|
65
67
|
return types.describe_error_metadata(
|
|
66
68
|
ex, image_path, filetype=types.FileType.IMAGE
|
|
67
69
|
)
|
|
@@ -112,7 +114,7 @@ class GeotagVideosFromGeneric(abc.ABC, T.Generic[TVideoExtractor]):
|
|
|
112
114
|
map_results,
|
|
113
115
|
desc="Extracting videos",
|
|
114
116
|
unit="videos",
|
|
115
|
-
disable=LOG.
|
|
117
|
+
disable=LOG.isEnabledFor(logging.DEBUG),
|
|
116
118
|
total=len(extractors),
|
|
117
119
|
)
|
|
118
120
|
)
|
|
@@ -128,6 +130,8 @@ class GeotagVideosFromGeneric(abc.ABC, T.Generic[TVideoExtractor]):
|
|
|
128
130
|
try:
|
|
129
131
|
return extractor.extract()
|
|
130
132
|
except exceptions.MapillaryDescriptionError as ex:
|
|
133
|
+
if LOG.isEnabledFor(logging.DEBUG):
|
|
134
|
+
LOG.error(f"{cls.__name__}({video_path.name}): {ex}")
|
|
131
135
|
return types.describe_error_metadata(
|
|
132
136
|
ex, video_path, filetype=types.FileType.VIDEO
|
|
133
137
|
)
|
|
@@ -67,7 +67,7 @@ def process(
|
|
|
67
67
|
reprocessable_paths = set(paths)
|
|
68
68
|
|
|
69
69
|
for idx, option in enumerate(options):
|
|
70
|
-
if LOG.
|
|
70
|
+
if LOG.isEnabledFor(logging.DEBUG):
|
|
71
71
|
LOG.info(
|
|
72
72
|
f"==> Processing {len(reprocessable_paths)} files with source {option}..."
|
|
73
73
|
)
|
|
@@ -121,6 +121,7 @@ def _is_reprocessable(metadata: types.MetadataOrError) -> bool:
|
|
|
121
121
|
exceptions.MapillaryGeoTaggingError,
|
|
122
122
|
exceptions.MapillaryVideoGPSNotFoundError,
|
|
123
123
|
exceptions.MapillaryExiftoolNotFoundError,
|
|
124
|
+
exceptions.MapillaryExifToolXMLNotFoundError,
|
|
124
125
|
),
|
|
125
126
|
):
|
|
126
127
|
return True
|
|
@@ -93,7 +93,7 @@ class GeotagImagesFromExifToolRunner(GeotagImagesFromGeneric):
|
|
|
93
93
|
LOG.warning(
|
|
94
94
|
"Failed to parse ExifTool XML: %s",
|
|
95
95
|
str(ex),
|
|
96
|
-
exc_info=LOG.
|
|
96
|
+
exc_info=LOG.isEnabledFor(logging.DEBUG),
|
|
97
97
|
)
|
|
98
98
|
rdf_by_path = {}
|
|
99
99
|
else:
|
|
@@ -61,21 +61,42 @@ class GeotagVideosFromExifToolXML(GeotagVideosFromGeneric):
|
|
|
61
61
|
def find_rdf_by_path(
|
|
62
62
|
cls, option: options.SourcePathOption, paths: T.Iterable[Path]
|
|
63
63
|
) -> dict[str, ET.Element]:
|
|
64
|
+
# Find RDF descriptions by path in RDF description
|
|
65
|
+
# Sources are matched based on the paths in "rdf:about" in XML elements
|
|
66
|
+
# {"source_path": "/path/to/exiftool.xml"}
|
|
67
|
+
# {"source_path": "/path/to/exiftool_xmls/"}
|
|
64
68
|
if option.source_path is not None:
|
|
65
69
|
return index_rdf_description_by_path([option.source_path])
|
|
66
70
|
|
|
67
|
-
|
|
71
|
+
# Find RDF descriptions by pattern matching
|
|
72
|
+
# i.e. "video.mp4" matches "/path/to/video.xml" regardless of "rdf:about"
|
|
73
|
+
# {"pattern": "/path/to/%g.xml"}
|
|
74
|
+
if option.pattern is not None:
|
|
68
75
|
rdf_by_path = {}
|
|
69
76
|
for path in paths:
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
77
|
+
canonical_path = exiftool_read.canonical_path(path)
|
|
78
|
+
|
|
79
|
+
# Skip non-existent resolved source paths to avoid verbose warnings
|
|
80
|
+
resolved_source_path = option.resolve(path)
|
|
81
|
+
if not resolved_source_path.exists():
|
|
82
|
+
continue
|
|
83
|
+
|
|
84
|
+
rdf_by_about = index_rdf_description_by_path([resolved_source_path])
|
|
85
|
+
if not rdf_by_about:
|
|
86
|
+
continue
|
|
87
|
+
|
|
88
|
+
rdf = rdf_by_about.get(canonical_path)
|
|
89
|
+
if rdf is None:
|
|
90
|
+
about, rdf = list(rdf_by_about.items())[0]
|
|
91
|
+
if len(rdf_by_about) > 1:
|
|
92
|
+
LOG.warning(
|
|
93
|
+
f"Found {len(rdf_by_about)} RDFs in the XML source {resolved_source_path}. Using the first RDF (with rdf:about={about}) for {path}"
|
|
94
|
+
)
|
|
95
|
+
rdf_by_path[canonical_path] = rdf
|
|
96
|
+
|
|
75
97
|
return rdf_by_path
|
|
76
98
|
|
|
77
|
-
|
|
78
|
-
assert False, "Either source_path or pattern must be provided"
|
|
99
|
+
raise AssertionError("Either source_path or pattern must be provided")
|
|
79
100
|
|
|
80
101
|
|
|
81
102
|
class GeotagVideosFromExifToolRunner(GeotagVideosFromGeneric):
|
|
@@ -110,7 +131,7 @@ class GeotagVideosFromExifToolRunner(GeotagVideosFromGeneric):
|
|
|
110
131
|
LOG.warning(
|
|
111
132
|
"Failed to parse ExifTool XML: %s",
|
|
112
133
|
str(ex),
|
|
113
|
-
exc_info=LOG.
|
|
134
|
+
exc_info=LOG.isEnabledFor(logging.DEBUG),
|
|
114
135
|
)
|
|
115
136
|
rdf_by_path = {}
|
|
116
137
|
else:
|
|
@@ -173,8 +173,11 @@ SourceOptionSchema = {
|
|
|
173
173
|
}
|
|
174
174
|
|
|
175
175
|
|
|
176
|
+
SourceOptionSchemaValidator = jsonschema.Draft202012Validator(SourceOptionSchema)
|
|
177
|
+
|
|
178
|
+
|
|
176
179
|
def validate_option(instance):
|
|
177
|
-
|
|
180
|
+
SourceOptionSchemaValidator.validate(instance=instance)
|
|
178
181
|
|
|
179
182
|
|
|
180
183
|
if __name__ == "__main__":
|
|
@@ -37,26 +37,23 @@ def parse_gpx(gpx_file: Path) -> list[Track]:
|
|
|
37
37
|
return tracks
|
|
38
38
|
|
|
39
39
|
|
|
40
|
-
def index_rdf_description_by_path(
|
|
41
|
-
|
|
42
|
-
) -> dict[str, ET.Element]:
|
|
43
|
-
rdf_description_by_path: dict[str, ET.Element] = {}
|
|
40
|
+
def index_rdf_description_by_path(xml_paths: T.Sequence[Path]) -> dict[str, ET.Element]:
|
|
41
|
+
rdf_by_path: dict[str, ET.Element] = {}
|
|
44
42
|
|
|
45
43
|
for xml_path in utils.find_xml_files(xml_paths):
|
|
46
44
|
try:
|
|
47
45
|
etree = ET.parse(xml_path)
|
|
48
|
-
except
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
LOG.
|
|
52
|
-
|
|
53
|
-
LOG.warning("Failed to parse %s: %s", xml_path, ex)
|
|
46
|
+
except Exception as ex:
|
|
47
|
+
LOG.warning(
|
|
48
|
+
f"Failed to parse {xml_path}: {ex}",
|
|
49
|
+
exc_info=LOG.isEnabledFor(logging.DEBUG),
|
|
50
|
+
)
|
|
54
51
|
continue
|
|
55
52
|
|
|
56
|
-
|
|
53
|
+
rdf_by_path.update(
|
|
57
54
|
exiftool_read.index_rdf_description_by_path_from_xml_element(
|
|
58
55
|
etree.getroot()
|
|
59
56
|
)
|
|
60
57
|
)
|
|
61
58
|
|
|
62
|
-
return
|
|
59
|
+
return rdf_by_path
|
{mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/geotag/video_extractors/gpx.py
RENAMED
|
@@ -12,7 +12,7 @@ if sys.version_info >= (3, 12):
|
|
|
12
12
|
else:
|
|
13
13
|
from typing_extensions import override
|
|
14
14
|
|
|
15
|
-
from ... import exceptions, geo, telemetry, types
|
|
15
|
+
from ... import exceptions, geo, telemetry, types, utils
|
|
16
16
|
from ..utils import parse_gpx
|
|
17
17
|
from .base import BaseVideoExtractor
|
|
18
18
|
from .native import NativeVideoExtractor
|
|
@@ -59,6 +59,7 @@ class GPXVideoExtractor(BaseVideoExtractor):
|
|
|
59
59
|
self._rebase_times(gpx_points)
|
|
60
60
|
return types.VideoMetadata(
|
|
61
61
|
filename=self.video_path,
|
|
62
|
+
filesize=utils.get_file_size(self.video_path),
|
|
62
63
|
filetype=types.FileType.VIDEO,
|
|
63
64
|
points=gpx_points,
|
|
64
65
|
)
|