mapillary-tools 0.14.0__tar.gz → 0.14.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/PKG-INFO +1 -1
  2. mapillary_tools-0.14.4/mapillary_tools/__init__.py +1 -0
  3. mapillary_tools-0.14.4/mapillary_tools/api_v4.py +176 -0
  4. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/authenticate.py +41 -34
  5. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/commands/__main__.py +8 -12
  6. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/commands/upload.py +7 -0
  7. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/config.py +5 -0
  8. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/constants.py +5 -3
  9. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/exif_read.py +2 -1
  10. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/geotag/base.py +6 -2
  11. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/geotag/factory.py +2 -1
  12. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/geotag/geotag_images_from_exiftool.py +1 -1
  13. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/geotag/geotag_videos_from_exiftool.py +30 -9
  14. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/geotag/options.py +4 -1
  15. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/geotag/utils.py +9 -12
  16. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/geotag/video_extractors/gpx.py +2 -1
  17. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/history.py +88 -53
  18. mapillary_tools-0.14.4/mapillary_tools/http.py +211 -0
  19. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/process_geotag_properties.py +12 -14
  20. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/sample_video.py +1 -2
  21. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/serializer/description.py +12 -2
  22. mapillary_tools-0.14.4/mapillary_tools/store.py +128 -0
  23. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/upload.py +47 -47
  24. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/upload_api_v4.py +8 -20
  25. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/uploader.py +576 -171
  26. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/utils.py +50 -5
  27. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools.egg-info/PKG-INFO +1 -1
  28. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools.egg-info/SOURCES.txt +2 -0
  29. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/pyproject.toml +1 -0
  30. mapillary_tools-0.14.0/mapillary_tools/__init__.py +0 -1
  31. mapillary_tools-0.14.0/mapillary_tools/api_v4.py +0 -416
  32. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/LICENSE +0 -0
  33. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/README.md +0 -0
  34. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/blackvue_parser.py +0 -0
  35. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/camm/camm_builder.py +0 -0
  36. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/camm/camm_parser.py +0 -0
  37. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/commands/__init__.py +0 -0
  38. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/commands/authenticate.py +0 -0
  39. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/commands/process.py +0 -0
  40. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/commands/process_and_upload.py +0 -0
  41. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/commands/sample_video.py +0 -0
  42. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/commands/video_process.py +0 -0
  43. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/commands/video_process_and_upload.py +0 -0
  44. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/commands/zip.py +0 -0
  45. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/exceptions.py +0 -0
  46. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/exif_write.py +0 -0
  47. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/exiftool_read.py +0 -0
  48. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/exiftool_read_video.py +0 -0
  49. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/exiftool_runner.py +0 -0
  50. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/ffmpeg.py +0 -0
  51. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/geo.py +0 -0
  52. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/geotag/__init__.py +0 -0
  53. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/geotag/geotag_images_from_exif.py +0 -0
  54. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/geotag/geotag_images_from_gpx.py +0 -0
  55. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/geotag/geotag_images_from_gpx_file.py +0 -0
  56. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/geotag/geotag_images_from_nmea_file.py +0 -0
  57. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/geotag/geotag_images_from_video.py +0 -0
  58. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/geotag/geotag_videos_from_gpx.py +0 -0
  59. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/geotag/geotag_videos_from_video.py +0 -0
  60. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/geotag/image_extractors/base.py +0 -0
  61. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/geotag/image_extractors/exif.py +0 -0
  62. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/geotag/image_extractors/exiftool.py +0 -0
  63. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/geotag/video_extractors/base.py +0 -0
  64. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/geotag/video_extractors/exiftool.py +0 -0
  65. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/geotag/video_extractors/native.py +0 -0
  66. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/gpmf/gpmf_gps_filter.py +0 -0
  67. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/gpmf/gpmf_parser.py +0 -0
  68. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/gpmf/gps_filter.py +0 -0
  69. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/ipc.py +0 -0
  70. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/mp4/__init__.py +0 -0
  71. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/mp4/construct_mp4_parser.py +0 -0
  72. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/mp4/io_utils.py +0 -0
  73. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/mp4/mp4_sample_parser.py +0 -0
  74. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/mp4/simple_mp4_builder.py +0 -0
  75. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/mp4/simple_mp4_parser.py +0 -0
  76. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/process_sequence_properties.py +0 -0
  77. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/serializer/gpx.py +0 -0
  78. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/telemetry.py +0 -0
  79. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools/types.py +0 -0
  80. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools.egg-info/dependency_links.txt +0 -0
  81. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools.egg-info/entry_points.txt +0 -0
  82. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools.egg-info/requires.txt +0 -0
  83. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/mapillary_tools.egg-info/top_level.txt +0 -0
  84. {mapillary_tools-0.14.0 → mapillary_tools-0.14.4}/setup.cfg +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: mapillary_tools
3
- Version: 0.14.0
3
+ Version: 0.14.4
4
4
  Summary: Mapillary Image/Video Import Pipeline
5
5
  Author-email: Mapillary <support@mapillary.com>
6
6
  License: BSD
@@ -0,0 +1 @@
1
+ VERSION = "0.14.4"
@@ -0,0 +1,176 @@
1
+ from __future__ import annotations
2
+
3
+ import enum
4
+ import logging
5
+ import os
6
+ import typing as T
7
+
8
+ import requests
9
+
10
+ from . import http
11
+
12
+ LOG = logging.getLogger(__name__)
13
+ MAPILLARY_CLIENT_TOKEN = os.getenv(
14
+ "MAPILLARY_CLIENT_TOKEN", "MLY|5675152195860640|6b02c72e6e3c801e5603ab0495623282"
15
+ )
16
+ MAPILLARY_GRAPH_API_ENDPOINT = os.getenv(
17
+ "MAPILLARY_GRAPH_API_ENDPOINT", "https://graph.mapillary.com"
18
+ )
19
+ REQUESTS_TIMEOUT: float = 60 # 1 minutes
20
+
21
+
22
+ class HTTPContentError(Exception):
23
+ """
24
+ Raised when the HTTP response is ok (200) but the content is not as expected
25
+ e.g. not JSON or not a valid response.
26
+ """
27
+
28
+ def __init__(self, message: str, response: requests.Response):
29
+ self.response = response
30
+ super().__init__(message)
31
+
32
+
33
+ class ClusterFileType(enum.Enum):
34
+ ZIP = "zip"
35
+ BLACKVUE = "mly_blackvue_video"
36
+ CAMM = "mly_camm_video"
37
+ MLY_BUNDLE_MANIFEST = "mly_bundle_manifest"
38
+
39
+
40
+ def create_user_session(user_access_token: str) -> requests.Session:
41
+ session = http.Session()
42
+ session.headers["Authorization"] = f"OAuth {user_access_token}"
43
+ return session
44
+
45
+
46
+ def create_client_session(disable_logging: bool = False) -> requests.Session:
47
+ session = http.Session()
48
+ session.headers["Authorization"] = f"OAuth {MAPILLARY_CLIENT_TOKEN}"
49
+ if disable_logging:
50
+ session.disable_logging_request = True
51
+ session.disable_logging_response = True
52
+ return session
53
+
54
+
55
+ def is_auth_error(resp: requests.Response) -> bool:
56
+ if resp.status_code in [401, 403]:
57
+ return True
58
+
59
+ if resp.status_code in [400]:
60
+ try:
61
+ error_body = resp.json()
62
+ except Exception:
63
+ error_body = {}
64
+
65
+ type = error_body.get("debug_info", {}).get("type")
66
+ if type in ["NotAuthorizedError"]:
67
+ return True
68
+
69
+ return False
70
+
71
+
72
+ def extract_auth_error_message(resp: requests.Response) -> str:
73
+ assert is_auth_error(resp), "has to be an auth error"
74
+
75
+ try:
76
+ error_body = resp.json()
77
+ except Exception:
78
+ error_body = {}
79
+
80
+ # from Graph APIs
81
+ message = error_body.get("error", {}).get("message")
82
+ if message is not None:
83
+ return str(message)
84
+
85
+ # from upload service
86
+ message = error_body.get("debug_info", {}).get("message")
87
+ if message is not None:
88
+ return str(message)
89
+
90
+ return resp.text
91
+
92
+
93
+ def get_upload_token(
94
+ client_session: requests.Session, email: str, password: str
95
+ ) -> requests.Response:
96
+ url = f"{MAPILLARY_GRAPH_API_ENDPOINT}/login"
97
+ json_data = {"email": email, "password": password, "locale": "en_US"}
98
+
99
+ resp = client_session.post(url, json=json_data, timeout=REQUESTS_TIMEOUT)
100
+ resp.raise_for_status()
101
+
102
+ return resp
103
+
104
+
105
+ def fetch_organization(
106
+ user_session: requests.Session, organization_id: int | str
107
+ ) -> requests.Response:
108
+ url = f"{MAPILLARY_GRAPH_API_ENDPOINT}/{organization_id}"
109
+ params = {"fields": ",".join(["slug", "description", "name"])}
110
+
111
+ resp = user_session.get(url, params=params, timeout=REQUESTS_TIMEOUT)
112
+ resp.raise_for_status()
113
+
114
+ return resp
115
+
116
+
117
+ def fetch_user_or_me(
118
+ user_session: requests.Session, user_id: int | str | None = None
119
+ ) -> requests.Response:
120
+ if user_id is None:
121
+ url = f"{MAPILLARY_GRAPH_API_ENDPOINT}/me"
122
+ else:
123
+ url = f"{MAPILLARY_GRAPH_API_ENDPOINT}/{user_id}"
124
+ params = {"fields": ",".join(["id", "username"])}
125
+
126
+ resp = user_session.get(url, params=params, timeout=REQUESTS_TIMEOUT)
127
+ resp.raise_for_status()
128
+
129
+ return resp
130
+
131
+
132
+ ActionType = T.Literal[
133
+ "upload_started_upload", "upload_finished_upload", "upload_failed_upload"
134
+ ]
135
+
136
+
137
+ def log_event(
138
+ client_session: requests.Session, action_type: ActionType, properties: dict
139
+ ) -> requests.Response:
140
+ url = f"{MAPILLARY_GRAPH_API_ENDPOINT}/logging"
141
+ json_data = {"action_type": action_type, "properties": properties}
142
+
143
+ resp = client_session.post(url, json=json_data, timeout=REQUESTS_TIMEOUT)
144
+ resp.raise_for_status()
145
+
146
+ return resp
147
+
148
+
149
+ def finish_upload(
150
+ user_session: requests.Session,
151
+ file_handle: str,
152
+ cluster_filetype: ClusterFileType,
153
+ organization_id: int | str | None = None,
154
+ ) -> requests.Response:
155
+ url = f"{MAPILLARY_GRAPH_API_ENDPOINT}/finish_upload"
156
+ json_data: dict[str, str | int] = {
157
+ "file_handle": file_handle,
158
+ "file_type": cluster_filetype.value,
159
+ }
160
+ if organization_id is not None:
161
+ json_data["organization_id"] = organization_id
162
+
163
+ resp = user_session.post(url, json=json_data, timeout=REQUESTS_TIMEOUT)
164
+ resp.raise_for_status()
165
+
166
+ return resp
167
+
168
+
169
+ def jsonify_response(resp: requests.Response) -> T.Any:
170
+ """
171
+ Convert the response to JSON, raising HTTPContentError if the response is not JSON.
172
+ """
173
+ try:
174
+ return resp.json()
175
+ except requests.JSONDecodeError as ex:
176
+ raise HTTPContentError("Invalid JSON response", resp) from ex
@@ -10,7 +10,7 @@ import jsonschema
10
10
 
11
11
  import requests
12
12
 
13
- from . import api_v4, config, constants, exceptions
13
+ from . import api_v4, config, constants, exceptions, http
14
14
 
15
15
 
16
16
  LOG = logging.getLogger(__name__)
@@ -77,11 +77,11 @@ def authenticate(
77
77
  # TODO: print more user information
78
78
  if profile_name in all_user_items:
79
79
  LOG.info(
80
- 'Profile "%s" updated: %s', profile_name, api_v4._sanitize(user_items)
80
+ 'Profile "%s" updated: %s', profile_name, http._sanitize(user_items)
81
81
  )
82
82
  else:
83
83
  LOG.info(
84
- 'Profile "%s" created: %s', profile_name, api_v4._sanitize(user_items)
84
+ 'Profile "%s" created: %s', profile_name, http._sanitize(user_items)
85
85
  )
86
86
 
87
87
 
@@ -127,21 +127,28 @@ def fetch_user_items(
127
127
 
128
128
  assert profile_name is not None, "profile_name should be set"
129
129
 
130
- user_items = _verify_user_auth(_validate_profile(user_items))
131
-
132
- LOG.info(
133
- f'Uploading to profile "{profile_name}": {user_items.get("MAPSettingsUsername")} (ID: {user_items.get("MAPSettingsUserKey")})'
134
- )
130
+ try:
131
+ LOG.info(f'Verifying profile "{profile_name}"...')
132
+ user_items = _verify_user_auth(_validate_profile(user_items))
135
133
 
136
- if organization_key is not None:
137
- resp = api_v4.fetch_organization(
138
- user_items["user_upload_token"], organization_key
139
- )
140
- data = api_v4.jsonify_response(resp)
141
134
  LOG.info(
142
- f"Uploading to organization: {data.get('name')} (ID: {data.get('id')})"
135
+ f'Uploading to profile "{profile_name}": {user_items.get("MAPSettingsUsername")} (ID: {user_items.get("MAPSettingsUserKey")})'
143
136
  )
144
- user_items["MAPOrganizationKey"] = data.get("id")
137
+
138
+ if organization_key is not None:
139
+ with api_v4.create_user_session(user_items["user_upload_token"]) as session:
140
+ resp = api_v4.fetch_organization(session, organization_key)
141
+ data = api_v4.jsonify_response(resp)
142
+ LOG.info(
143
+ f"Uploading to organization: {data.get('name')} (ID: {data.get('id')})"
144
+ )
145
+ user_items["MAPOrganizationKey"] = data.get("id")
146
+
147
+ except requests.Timeout as ex:
148
+ raise exceptions.MapillaryUploadTimeoutError(str(ex)) from ex
149
+
150
+ except requests.ConnectionError as ex:
151
+ raise exceptions.MapillaryUploadConnectionError(str(ex)) from ex
145
152
 
146
153
  return user_items
147
154
 
@@ -158,7 +165,7 @@ def _prompt(message: str) -> str:
158
165
 
159
166
  def _validate_profile(user_items: config.UserItem) -> config.UserItem:
160
167
  try:
161
- jsonschema.validate(user_items, config.UserItemSchema)
168
+ config.UserItemSchemaValidator.validate(user_items)
162
169
  except jsonschema.ValidationError as ex:
163
170
  raise exceptions.MapillaryBadParameterError(
164
171
  f"Invalid profile format: {ex.message}"
@@ -173,16 +180,15 @@ def _verify_user_auth(user_items: config.UserItem) -> config.UserItem:
173
180
  if constants._AUTH_VERIFICATION_DISABLED:
174
181
  return user_items
175
182
 
176
- try:
177
- resp = api_v4.fetch_user_or_me(
178
- user_access_token=user_items["user_upload_token"]
179
- )
180
- except requests.HTTPError as ex:
181
- if api_v4.is_auth_error(ex.response):
182
- message = api_v4.extract_auth_error_message(ex.response)
183
- raise exceptions.MapillaryUploadUnauthorizedError(message)
184
- else:
185
- raise ex
183
+ with api_v4.create_user_session(user_items["user_upload_token"]) as session:
184
+ try:
185
+ resp = api_v4.fetch_user_or_me(session)
186
+ except requests.HTTPError as ex:
187
+ if api_v4.is_auth_error(ex.response):
188
+ message = api_v4.extract_auth_error_message(ex.response)
189
+ raise exceptions.MapillaryUploadUnauthorizedError(message)
190
+ else:
191
+ raise ex
186
192
 
187
193
  data = api_v4.jsonify_response(resp)
188
194
 
@@ -276,16 +282,17 @@ def _prompt_login(
276
282
  if user_password:
277
283
  break
278
284
 
279
- try:
280
- resp = api_v4.get_upload_token(user_email, user_password)
281
- except requests.HTTPError as ex:
282
- if not _enabled:
283
- raise ex
285
+ with api_v4.create_client_session() as session:
286
+ try:
287
+ resp = api_v4.get_upload_token(session, user_email, user_password)
288
+ except requests.HTTPError as ex:
289
+ if not _enabled:
290
+ raise ex
284
291
 
285
- if _is_login_retryable(ex):
286
- return _prompt_login()
292
+ if _is_login_retryable(ex):
293
+ return _prompt_login()
287
294
 
288
- raise ex
295
+ raise ex
289
296
 
290
297
  data = api_v4.jsonify_response(resp)
291
298
 
@@ -8,6 +8,7 @@ import requests
8
8
 
9
9
  from .. import api_v4, constants, exceptions, VERSION
10
10
  from ..upload import log_exception
11
+ from ..utils import configure_logger, get_app_name
11
12
  from . import (
12
13
  authenticate,
13
14
  process,
@@ -31,8 +32,8 @@ mapillary_tools_commands = [
31
32
  ]
32
33
 
33
34
 
34
- # do not use __name__ here is because if you run tools as a module, __name__ will be "__main__"
35
- LOG = logging.getLogger("mapillary_tools")
35
+ # Root logger of mapillary_tools (not including third-party libraries)
36
+ LOG = logging.getLogger(get_app_name())
36
37
 
37
38
 
38
39
  # Handle shared arguments/options here
@@ -79,13 +80,6 @@ def add_general_arguments(parser, command):
79
80
  )
80
81
 
81
82
 
82
- def configure_logger(logger: logging.Logger, stream=None) -> None:
83
- formatter = logging.Formatter("%(asctime)s - %(levelname)-7s - %(message)s")
84
- handler = logging.StreamHandler(stream)
85
- handler.setFormatter(formatter)
86
- logger.addHandler(handler)
87
-
88
-
89
83
  def _log_params(argvars: dict) -> None:
90
84
  MAX_ENTRIES = 5
91
85
 
@@ -152,9 +146,7 @@ def main():
152
146
 
153
147
  args = parser.parse_args()
154
148
 
155
- log_level = logging.DEBUG if args.verbose else logging.INFO
156
- configure_logger(LOG, sys.stderr)
157
- LOG.setLevel(log_level)
149
+ configure_logger(LOG, level=logging.DEBUG if args.verbose else logging.INFO)
158
150
 
159
151
  LOG.debug("%s", version_text)
160
152
  argvars = vars(args)
@@ -175,6 +167,10 @@ def main():
175
167
  log_exception(ex)
176
168
  sys.exit(ex.exit_code)
177
169
 
170
+ except KeyboardInterrupt:
171
+ LOG.info("Interrupted by user...")
172
+ sys.exit(130)
173
+
178
174
 
179
175
  if __name__ == "__main__":
180
176
  main()
@@ -23,6 +23,13 @@ class Command:
23
23
  default=None,
24
24
  required=False,
25
25
  )
26
+ group.add_argument(
27
+ "--num_upload_workers",
28
+ help="Number of concurrent upload workers for uploading images. [default: %(default)s]",
29
+ default=constants.MAX_IMAGE_UPLOAD_WORKERS,
30
+ type=int,
31
+ required=False,
32
+ )
26
33
  group.add_argument(
27
34
  "--reupload",
28
35
  help="Re-upload data that has already been uploaded.",
@@ -6,6 +6,8 @@ import sys
6
6
  import typing as T
7
7
  from typing import TypedDict
8
8
 
9
+ import jsonschema
10
+
9
11
  if sys.version_info >= (3, 11):
10
12
  from typing import Required
11
13
  else:
@@ -50,6 +52,9 @@ UserItemSchema = {
50
52
  }
51
53
 
52
54
 
55
+ UserItemSchemaValidator = jsonschema.Draft202012Validator(UserItemSchema)
56
+
57
+
53
58
  def _load_config(config_path: str) -> configparser.ConfigParser:
54
59
  config = configparser.ConfigParser()
55
60
  # Override to not change option names (by default it will lower them)
@@ -154,16 +154,18 @@ UPLOAD_CACHE_DIR: str = os.getenv(
154
154
  # The minimal upload speed is used to calculate the read timeout to avoid upload hanging:
155
155
  # timeout = upload_size / MIN_UPLOAD_SPEED
156
156
  MIN_UPLOAD_SPEED: int | None = _parse_filesize(
157
- os.getenv(_ENV_PREFIX + "MIN_UPLOAD_SPEED", "50K") # 50 KiB/s
157
+ os.getenv(_ENV_PREFIX + "MIN_UPLOAD_SPEED", "50K") # 50 Kb/s
158
158
  )
159
+ # Maximum number of parallel workers for uploading images within a single sequence.
160
+ # NOTE: Sequences themselves are uploaded sequentially, not in parallel.
159
161
  MAX_IMAGE_UPLOAD_WORKERS: int = int(
160
- os.getenv(_ENV_PREFIX + "MAX_IMAGE_UPLOAD_WORKERS", 64)
162
+ os.getenv(_ENV_PREFIX + "MAX_IMAGE_UPLOAD_WORKERS", 4)
161
163
  )
162
164
  # The chunk size in MB (see chunked transfer encoding https://en.wikipedia.org/wiki/Chunked_transfer_encoding)
163
165
  # for uploading data to MLY upload service.
164
166
  # Changing this size does not change the number of requests nor affect upload performance,
165
167
  # but it affects the responsiveness of the upload progress bar
166
- UPLOAD_CHUNK_SIZE_MB: float = float(os.getenv(_ENV_PREFIX + "UPLOAD_CHUNK_SIZE_MB", 1))
168
+ UPLOAD_CHUNK_SIZE_MB: float = float(os.getenv(_ENV_PREFIX + "UPLOAD_CHUNK_SIZE_MB", 2))
167
169
  MAX_UPLOAD_RETRIES: int = int(os.getenv(_ENV_PREFIX + "MAX_UPLOAD_RETRIES", 200))
168
170
  MAPILLARY__ENABLE_UPLOAD_HISTORY_FOR_DRY_RUN: bool = _yes_or_no(
169
171
  os.getenv("MAPILLARY__ENABLE_UPLOAD_HISTORY_FOR_DRY_RUN", "NO")
@@ -871,7 +871,8 @@ class ExifRead(ExifReadFromEXIF):
871
871
 
872
872
  def _xmp_with_reason(self, reason: str) -> ExifReadFromXMP | None:
873
873
  if not self._xml_extracted:
874
- LOG.debug('Extracting XMP for "%s"', reason)
874
+ # TODO Disabled because too verbose but still useful to know
875
+ # LOG.debug('Extracting XMP for "%s"', reason)
875
876
  self._cached_xml = self._extract_xmp()
876
877
  self._xml_extracted = True
877
878
 
@@ -46,7 +46,7 @@ class GeotagImagesFromGeneric(abc.ABC, T.Generic[TImageExtractor]):
46
46
  map_results,
47
47
  desc="Extracting images",
48
48
  unit="images",
49
- disable=LOG.getEffectiveLevel() <= logging.DEBUG,
49
+ disable=LOG.isEnabledFor(logging.DEBUG),
50
50
  total=len(extractors),
51
51
  )
52
52
  )
@@ -62,6 +62,8 @@ class GeotagImagesFromGeneric(abc.ABC, T.Generic[TImageExtractor]):
62
62
  try:
63
63
  return extractor.extract()
64
64
  except exceptions.MapillaryDescriptionError as ex:
65
+ if LOG.isEnabledFor(logging.DEBUG):
66
+ LOG.error(f"{cls.__name__}({image_path.name}): {ex}")
65
67
  return types.describe_error_metadata(
66
68
  ex, image_path, filetype=types.FileType.IMAGE
67
69
  )
@@ -112,7 +114,7 @@ class GeotagVideosFromGeneric(abc.ABC, T.Generic[TVideoExtractor]):
112
114
  map_results,
113
115
  desc="Extracting videos",
114
116
  unit="videos",
115
- disable=LOG.getEffectiveLevel() <= logging.DEBUG,
117
+ disable=LOG.isEnabledFor(logging.DEBUG),
116
118
  total=len(extractors),
117
119
  )
118
120
  )
@@ -128,6 +130,8 @@ class GeotagVideosFromGeneric(abc.ABC, T.Generic[TVideoExtractor]):
128
130
  try:
129
131
  return extractor.extract()
130
132
  except exceptions.MapillaryDescriptionError as ex:
133
+ if LOG.isEnabledFor(logging.DEBUG):
134
+ LOG.error(f"{cls.__name__}({video_path.name}): {ex}")
131
135
  return types.describe_error_metadata(
132
136
  ex, video_path, filetype=types.FileType.VIDEO
133
137
  )
@@ -67,7 +67,7 @@ def process(
67
67
  reprocessable_paths = set(paths)
68
68
 
69
69
  for idx, option in enumerate(options):
70
- if LOG.getEffectiveLevel() <= logging.DEBUG:
70
+ if LOG.isEnabledFor(logging.DEBUG):
71
71
  LOG.info(
72
72
  f"==> Processing {len(reprocessable_paths)} files with source {option}..."
73
73
  )
@@ -121,6 +121,7 @@ def _is_reprocessable(metadata: types.MetadataOrError) -> bool:
121
121
  exceptions.MapillaryGeoTaggingError,
122
122
  exceptions.MapillaryVideoGPSNotFoundError,
123
123
  exceptions.MapillaryExiftoolNotFoundError,
124
+ exceptions.MapillaryExifToolXMLNotFoundError,
124
125
  ),
125
126
  ):
126
127
  return True
@@ -93,7 +93,7 @@ class GeotagImagesFromExifToolRunner(GeotagImagesFromGeneric):
93
93
  LOG.warning(
94
94
  "Failed to parse ExifTool XML: %s",
95
95
  str(ex),
96
- exc_info=LOG.getEffectiveLevel() <= logging.DEBUG,
96
+ exc_info=LOG.isEnabledFor(logging.DEBUG),
97
97
  )
98
98
  rdf_by_path = {}
99
99
  else:
@@ -61,21 +61,42 @@ class GeotagVideosFromExifToolXML(GeotagVideosFromGeneric):
61
61
  def find_rdf_by_path(
62
62
  cls, option: options.SourcePathOption, paths: T.Iterable[Path]
63
63
  ) -> dict[str, ET.Element]:
64
+ # Find RDF descriptions by path in RDF description
65
+ # Sources are matched based on the paths in "rdf:about" in XML elements
66
+ # {"source_path": "/path/to/exiftool.xml"}
67
+ # {"source_path": "/path/to/exiftool_xmls/"}
64
68
  if option.source_path is not None:
65
69
  return index_rdf_description_by_path([option.source_path])
66
70
 
67
- elif option.pattern is not None:
71
+ # Find RDF descriptions by pattern matching
72
+ # i.e. "video.mp4" matches "/path/to/video.xml" regardless of "rdf:about"
73
+ # {"pattern": "/path/to/%g.xml"}
74
+ if option.pattern is not None:
68
75
  rdf_by_path = {}
69
76
  for path in paths:
70
- source_path = option.resolve(path)
71
- r = index_rdf_description_by_path([source_path])
72
- rdfs = list(r.values())
73
- if rdfs:
74
- rdf_by_path[exiftool_read.canonical_path(path)] = rdfs[0]
77
+ canonical_path = exiftool_read.canonical_path(path)
78
+
79
+ # Skip non-existent resolved source paths to avoid verbose warnings
80
+ resolved_source_path = option.resolve(path)
81
+ if not resolved_source_path.exists():
82
+ continue
83
+
84
+ rdf_by_about = index_rdf_description_by_path([resolved_source_path])
85
+ if not rdf_by_about:
86
+ continue
87
+
88
+ rdf = rdf_by_about.get(canonical_path)
89
+ if rdf is None:
90
+ about, rdf = list(rdf_by_about.items())[0]
91
+ if len(rdf_by_about) > 1:
92
+ LOG.warning(
93
+ f"Found {len(rdf_by_about)} RDFs in the XML source {resolved_source_path}. Using the first RDF (with rdf:about={about}) for {path}"
94
+ )
95
+ rdf_by_path[canonical_path] = rdf
96
+
75
97
  return rdf_by_path
76
98
 
77
- else:
78
- assert False, "Either source_path or pattern must be provided"
99
+ raise AssertionError("Either source_path or pattern must be provided")
79
100
 
80
101
 
81
102
  class GeotagVideosFromExifToolRunner(GeotagVideosFromGeneric):
@@ -110,7 +131,7 @@ class GeotagVideosFromExifToolRunner(GeotagVideosFromGeneric):
110
131
  LOG.warning(
111
132
  "Failed to parse ExifTool XML: %s",
112
133
  str(ex),
113
- exc_info=LOG.getEffectiveLevel() <= logging.DEBUG,
134
+ exc_info=LOG.isEnabledFor(logging.DEBUG),
114
135
  )
115
136
  rdf_by_path = {}
116
137
  else:
@@ -173,8 +173,11 @@ SourceOptionSchema = {
173
173
  }
174
174
 
175
175
 
176
+ SourceOptionSchemaValidator = jsonschema.Draft202012Validator(SourceOptionSchema)
177
+
178
+
176
179
  def validate_option(instance):
177
- jsonschema.validate(instance=instance, schema=SourceOptionSchema)
180
+ SourceOptionSchemaValidator.validate(instance=instance)
178
181
 
179
182
 
180
183
  if __name__ == "__main__":
@@ -37,26 +37,23 @@ def parse_gpx(gpx_file: Path) -> list[Track]:
37
37
  return tracks
38
38
 
39
39
 
40
- def index_rdf_description_by_path(
41
- xml_paths: T.Sequence[Path],
42
- ) -> dict[str, ET.Element]:
43
- rdf_description_by_path: dict[str, ET.Element] = {}
40
+ def index_rdf_description_by_path(xml_paths: T.Sequence[Path]) -> dict[str, ET.Element]:
41
+ rdf_by_path: dict[str, ET.Element] = {}
44
42
 
45
43
  for xml_path in utils.find_xml_files(xml_paths):
46
44
  try:
47
45
  etree = ET.parse(xml_path)
48
- except ET.ParseError as ex:
49
- verbose = LOG.getEffectiveLevel() <= logging.DEBUG
50
- if verbose:
51
- LOG.warning("Failed to parse %s", xml_path, exc_info=True)
52
- else:
53
- LOG.warning("Failed to parse %s: %s", xml_path, ex)
46
+ except Exception as ex:
47
+ LOG.warning(
48
+ f"Failed to parse {xml_path}: {ex}",
49
+ exc_info=LOG.isEnabledFor(logging.DEBUG),
50
+ )
54
51
  continue
55
52
 
56
- rdf_description_by_path.update(
53
+ rdf_by_path.update(
57
54
  exiftool_read.index_rdf_description_by_path_from_xml_element(
58
55
  etree.getroot()
59
56
  )
60
57
  )
61
58
 
62
- return rdf_description_by_path
59
+ return rdf_by_path
@@ -12,7 +12,7 @@ if sys.version_info >= (3, 12):
12
12
  else:
13
13
  from typing_extensions import override
14
14
 
15
- from ... import exceptions, geo, telemetry, types
15
+ from ... import exceptions, geo, telemetry, types, utils
16
16
  from ..utils import parse_gpx
17
17
  from .base import BaseVideoExtractor
18
18
  from .native import NativeVideoExtractor
@@ -59,6 +59,7 @@ class GPXVideoExtractor(BaseVideoExtractor):
59
59
  self._rebase_times(gpx_points)
60
60
  return types.VideoMetadata(
61
61
  filename=self.video_path,
62
+ filesize=utils.get_file_size(self.video_path),
62
63
  filetype=types.FileType.VIDEO,
63
64
  points=gpx_points,
64
65
  )