mapillary-tools 0.14.1__py3-none-any.whl → 0.14.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1 +1 @@
1
- VERSION = "0.14.1"
1
+ VERSION = "0.14.3"
@@ -165,7 +165,7 @@ def _prompt(message: str) -> str:
165
165
 
166
166
  def _validate_profile(user_items: config.UserItem) -> config.UserItem:
167
167
  try:
168
- jsonschema.validate(user_items, config.UserItemSchema)
168
+ config.UserItemSchemaValidator.validate(user_items)
169
169
  except jsonschema.ValidationError as ex:
170
170
  raise exceptions.MapillaryBadParameterError(
171
171
  f"Invalid profile format: {ex.message}"
mapillary_tools/config.py CHANGED
@@ -6,6 +6,8 @@ import sys
6
6
  import typing as T
7
7
  from typing import TypedDict
8
8
 
9
+ import jsonschema
10
+
9
11
  if sys.version_info >= (3, 11):
10
12
  from typing import Required
11
13
  else:
@@ -50,6 +52,9 @@ UserItemSchema = {
50
52
  }
51
53
 
52
54
 
55
+ UserItemSchemaValidator = jsonschema.Draft202012Validator(UserItemSchema)
56
+
57
+
53
58
  def _load_config(config_path: str) -> configparser.ConfigParser:
54
59
  config = configparser.ConfigParser()
55
60
  # Override to not change option names (by default it will lower them)
@@ -173,8 +173,11 @@ SourceOptionSchema = {
173
173
  }
174
174
 
175
175
 
176
+ SourceOptionSchemaValidator = jsonschema.Draft202012Validator(SourceOptionSchema)
177
+
178
+
176
179
  def validate_option(instance):
177
- jsonschema.validate(instance=instance, schema=SourceOptionSchema)
180
+ SourceOptionSchemaValidator.validate(instance=instance)
178
181
 
179
182
 
180
183
  if __name__ == "__main__":
@@ -162,6 +162,11 @@ class PersistentCache:
162
162
 
163
163
  return expired_keys
164
164
 
165
+ def keys(self):
166
+ with self._lock:
167
+ with dbm.open(self._file, flag="c") as db:
168
+ return db.keys()
169
+
165
170
  def _is_expired(self, payload: JSONDict) -> bool:
166
171
  expires_at = payload.get("expires_at")
167
172
  if isinstance(expires_at, (int, float)):
@@ -304,19 +304,12 @@ def _validate_metadatas(
304
304
  # TypeError: __init__() missing 3 required positional arguments: 'image_time', 'gpx_start_time', and 'gpx_end_time'
305
305
  # See https://stackoverflow.com/a/61432070
306
306
  good_metadatas, error_metadatas = types.separate_errors(metadatas)
307
- map_results = utils.mp_map_maybe(
308
- validate_and_fail_metadata,
309
- T.cast(T.Iterable[types.Metadata], good_metadatas),
310
- num_processes=num_processes,
311
- )
312
307
 
313
308
  validated_metadatas = list(
314
- tqdm(
315
- map_results,
316
- desc="Validating metadatas",
317
- unit="metadata",
318
- disable=LOG.getEffectiveLevel() <= logging.DEBUG,
319
- total=len(good_metadatas),
309
+ utils.mp_map_maybe(
310
+ validate_and_fail_metadata,
311
+ T.cast(T.Iterable[types.Metadata], good_metadatas),
312
+ num_processes=num_processes,
320
313
  )
321
314
  )
322
315
 
@@ -259,6 +259,11 @@ ImageDescriptionFileSchema = _merge_schema(
259
259
  )
260
260
 
261
261
 
262
+ ImageDescriptionFileSchemaValidator = jsonschema.Draft202012Validator(
263
+ ImageDescriptionFileSchema
264
+ )
265
+
266
+
262
267
  VideoDescriptionFileSchema = _merge_schema(
263
268
  VideoDescriptionSchema,
264
269
  {
@@ -295,6 +300,11 @@ VideoDescriptionFileSchema = _merge_schema(
295
300
  )
296
301
 
297
302
 
303
+ VideoDescriptionFileSchemaValidator = jsonschema.Draft202012Validator(
304
+ VideoDescriptionFileSchema
305
+ )
306
+
307
+
298
308
  ImageVideoDescriptionFileSchema = {
299
309
  "oneOf": [VideoDescriptionFileSchema, ImageDescriptionFileSchema]
300
310
  }
@@ -520,7 +530,7 @@ def parse_capture_time(time: str) -> datetime.datetime:
520
530
 
521
531
  def validate_image_desc(desc: T.Any) -> None:
522
532
  try:
523
- jsonschema.validate(instance=desc, schema=ImageDescriptionFileSchema)
533
+ ImageDescriptionFileSchemaValidator.validate(desc)
524
534
  except jsonschema.ValidationError as ex:
525
535
  # do not use str(ex) which is more verbose
526
536
  raise exceptions.MapillaryMetadataValidationError(ex.message) from ex
@@ -533,7 +543,7 @@ def validate_image_desc(desc: T.Any) -> None:
533
543
 
534
544
  def validate_video_desc(desc: T.Any) -> None:
535
545
  try:
536
- jsonschema.validate(instance=desc, schema=VideoDescriptionFileSchema)
546
+ VideoDescriptionFileSchemaValidator.validate(desc)
537
547
  except jsonschema.ValidationError as ex:
538
548
  # do not use str(ex) which is more verbose
539
549
  raise exceptions.MapillaryMetadataValidationError(ex.message) from ex
mapillary_tools/upload.py CHANGED
@@ -10,7 +10,6 @@ import uuid
10
10
  from pathlib import Path
11
11
 
12
12
  import humanize
13
- import jsonschema
14
13
  import requests
15
14
  from tqdm import tqdm
16
15
 
@@ -57,7 +56,7 @@ def upload(
57
56
 
58
57
  metadatas = _load_descs(_metadatas_from_process, import_paths, desc_path)
59
58
 
60
- jsonschema.validate(instance=user_items, schema=config.UserItemSchema)
59
+ config.UserItemSchemaValidator.validate(user_items)
61
60
 
62
61
  # Setup the emitter -- the order matters here
63
62
 
@@ -2,6 +2,9 @@ from __future__ import annotations
2
2
 
3
3
  import concurrent.futures
4
4
  import dataclasses
5
+ import datetime
6
+ import email.utils
7
+ import hashlib
5
8
  import io
6
9
  import json
7
10
  import logging
@@ -36,6 +39,7 @@ from . import (
36
39
  types,
37
40
  upload_api_v4,
38
41
  utils,
42
+ VERSION,
39
43
  )
40
44
  from .camm import camm_builder, camm_parser
41
45
  from .gpmf import gpmf_parser
@@ -55,6 +59,9 @@ class UploadOptions:
55
59
  user_items: config.UserItem
56
60
  chunk_size: int = int(constants.UPLOAD_CHUNK_SIZE_MB * 1024 * 1024)
57
61
  num_upload_workers: int = constants.MAX_IMAGE_UPLOAD_WORKERS
62
+ # When set, upload cache will be read/write there
63
+ # This option is exposed for testing purpose. In PROD, the path is calculated based on envvar and user_items
64
+ upload_cache_path: Path | None = None
58
65
  dry_run: bool = False
59
66
  nofinish: bool = False
60
67
  noresume: bool = False
@@ -470,7 +477,7 @@ class ZipUploader:
470
477
  # Arcname should be unique, the name does not matter
471
478
  arcname = f"{idx}.jpg"
472
479
  zipinfo = zipfile.ZipInfo(arcname, date_time=(1980, 1, 1, 0, 0, 0))
473
- zipf.writestr(zipinfo, SingleImageUploader.dump_image_bytes(metadata))
480
+ zipf.writestr(zipinfo, CachedImageUploader.dump_image_bytes(metadata))
474
481
  assert len(sequence) == len(set(zipf.namelist()))
475
482
  zipf.comment = json.dumps(
476
483
  {"sequence_md5sum": sequence_md5sum},
@@ -536,6 +543,13 @@ class ImageSequenceUploader:
536
543
  def __init__(self, upload_options: UploadOptions, emitter: EventEmitter):
537
544
  self.upload_options = upload_options
538
545
  self.emitter = emitter
546
+ # Create a single shared SingleImageUploader instance that will be used across all uploads
547
+ cache = _maybe_create_persistent_cache_instance(self.upload_options)
548
+ if cache:
549
+ cache.clear_expired()
550
+ self.cached_image_uploader = CachedImageUploader(
551
+ self.upload_options, cache=cache
552
+ )
539
553
 
540
554
  def upload_images(
541
555
  self, image_metadatas: T.Sequence[types.ImageMetadata]
@@ -687,10 +701,6 @@ class ImageSequenceUploader:
687
701
  with api_v4.create_user_session(
688
702
  self.upload_options.user_items["user_upload_token"]
689
703
  ) as user_session:
690
- single_image_uploader = SingleImageUploader(
691
- self.upload_options, user_session=user_session
692
- )
693
-
694
704
  while True:
695
705
  # Assert that all images are already pushed into the queue
696
706
  try:
@@ -709,8 +719,8 @@ class ImageSequenceUploader:
709
719
  }
710
720
 
711
721
  # image_progress will be updated during uploading
712
- file_handle = single_image_uploader.upload(
713
- image_metadata, image_progress
722
+ file_handle = self.cached_image_uploader.upload(
723
+ user_session, image_metadata, image_progress
714
724
  )
715
725
 
716
726
  # Update chunk_size (it was constant if set)
@@ -730,24 +740,27 @@ class ImageSequenceUploader:
730
740
  return indexed_file_handles
731
741
 
732
742
 
733
- class SingleImageUploader:
743
+ class CachedImageUploader:
734
744
  def __init__(
735
745
  self,
736
746
  upload_options: UploadOptions,
737
- user_session: requests.Session | None = None,
747
+ cache: history.PersistentCache | None = None,
738
748
  ):
739
749
  self.upload_options = upload_options
740
- self.user_session = user_session
741
- self.cache = self._maybe_create_persistent_cache_instance(
742
- self.upload_options.user_items, upload_options
743
- )
750
+ self.cache = cache
751
+ if self.cache:
752
+ self.cache.clear_expired()
744
753
 
754
+ # Thread-safe
745
755
  def upload(
746
- self, image_metadata: types.ImageMetadata, image_progress: dict[str, T.Any]
756
+ self,
757
+ user_session: requests.Session,
758
+ image_metadata: types.ImageMetadata,
759
+ image_progress: dict[str, T.Any],
747
760
  ) -> str:
748
761
  image_bytes = self.dump_image_bytes(image_metadata)
749
762
 
750
- uploader = Uploader(self.upload_options, user_session=self.user_session)
763
+ uploader = Uploader(self.upload_options, user_session=user_session)
751
764
 
752
765
  session_key = uploader._gen_session_key(io.BytesIO(image_bytes), image_progress)
753
766
 
@@ -785,44 +798,7 @@ class SingleImageUploader:
785
798
  f"Failed to dump EXIF bytes: {ex}", metadata.filename
786
799
  ) from ex
787
800
 
788
- @classmethod
789
- def _maybe_create_persistent_cache_instance(
790
- cls, user_items: config.UserItem, upload_options: UploadOptions
791
- ) -> history.PersistentCache | None:
792
- if not constants.UPLOAD_CACHE_DIR:
793
- LOG.debug(
794
- "Upload cache directory is set empty, skipping caching upload file handles"
795
- )
796
- return None
797
-
798
- if upload_options.dry_run:
799
- LOG.debug("Dry-run mode enabled, skipping caching upload file handles")
800
- return None
801
-
802
- cache_path_dir = (
803
- Path(constants.UPLOAD_CACHE_DIR)
804
- .joinpath(api_v4.MAPILLARY_CLIENT_TOKEN.replace("|", "_"))
805
- .joinpath(
806
- user_items.get("MAPSettingsUserKey", user_items["user_upload_token"])
807
- )
808
- )
809
- cache_path_dir.mkdir(parents=True, exist_ok=True)
810
- cache_path = cache_path_dir.joinpath("cached_file_handles")
811
-
812
- # Sanitize sensitive segments for logging
813
- sanitized_cache_path = (
814
- Path(constants.UPLOAD_CACHE_DIR)
815
- .joinpath("***")
816
- .joinpath("***")
817
- .joinpath("cached_file_handles")
818
- )
819
- LOG.debug(f"File handle cache path: {sanitized_cache_path}")
820
-
821
- cache = history.PersistentCache(str(cache_path.resolve()))
822
- cache.clear_expired()
823
-
824
- return cache
825
-
801
+ # Thread-safe
826
802
  def _get_cached_file_handle(self, key: str) -> str | None:
827
803
  if self.cache is None:
828
804
  return None
@@ -832,6 +808,7 @@ class SingleImageUploader:
832
808
 
833
809
  return self.cache.get(key)
834
810
 
811
+ # Thread-safe
835
812
  def _set_file_handle_cache(self, key: str, value: str) -> None:
836
813
  if self.cache is None:
837
814
  return
@@ -971,27 +948,33 @@ class Uploader:
971
948
  begin_offset = progress.get("begin_offset")
972
949
  offset = progress.get("offset")
973
950
 
974
- if retries <= constants.MAX_UPLOAD_RETRIES and _is_retriable_exception(ex):
975
- self.emitter.emit("upload_retrying", progress)
951
+ LOG.warning(
952
+ f"Error uploading {self._upload_name(progress)} at {offset=} since {begin_offset=}: {ex.__class__.__name__}: {ex}"
953
+ )
976
954
 
977
- LOG.warning(
978
- f"Error uploading {self._upload_name(progress)} at {offset=} since {begin_offset=}: {ex.__class__.__name__}: {ex}"
979
- )
955
+ if retries <= constants.MAX_UPLOAD_RETRIES:
956
+ retriable, retry_after_sec = _is_retriable_exception(ex)
957
+ if retriable:
958
+ self.emitter.emit("upload_retrying", progress)
980
959
 
981
- # Keep things immutable here. Will increment retries in the caller
982
- retries += 1
983
- if _is_immediate_retriable_exception(ex):
984
- sleep_for = 0
985
- else:
986
- sleep_for = min(2**retries, 16)
987
- LOG.info(
988
- f"Retrying in {sleep_for} seconds ({retries}/{constants.MAX_UPLOAD_RETRIES})"
989
- )
990
- if sleep_for:
991
- time.sleep(sleep_for)
992
- else:
993
- self.emitter.emit("upload_failed", progress)
994
- raise ex
960
+ # Keep things immutable here. Will increment retries in the caller
961
+ retries += 1
962
+ if _is_immediate_retriable_exception(ex):
963
+ sleep_for = 0
964
+ else:
965
+ sleep_for = min(2**retries, 16)
966
+ sleep_for += retry_after_sec
967
+
968
+ LOG.info(
969
+ f"Retrying in {sleep_for} seconds ({retries}/{constants.MAX_UPLOAD_RETRIES})"
970
+ )
971
+ if sleep_for:
972
+ time.sleep(sleep_for)
973
+
974
+ return
975
+
976
+ self.emitter.emit("upload_failed", progress)
977
+ raise ex
995
978
 
996
979
  @classmethod
997
980
  def _upload_name(cls, progress: UploaderProgress):
@@ -1108,23 +1091,188 @@ def _is_immediate_retriable_exception(ex: BaseException) -> bool:
1108
1091
  return False
1109
1092
 
1110
1093
 
1111
- def _is_retriable_exception(ex: BaseException) -> bool:
1094
+ def _is_retriable_exception(ex: BaseException) -> tuple[bool, int]:
1095
+ """
1096
+ Determine if an exception should be retried and how long to wait.
1097
+
1098
+ Args:
1099
+ ex: Exception to check for retryability
1100
+
1101
+ Returns:
1102
+ Tuple of (retriable, retry_after_sec) where:
1103
+ - retriable: True if the exception should be retried
1104
+ - retry_after_sec: Seconds to wait before retry (>= 0)
1105
+
1106
+ Examples:
1107
+ >>> resp = requests.Response()
1108
+ >>> resp._content = b"foo"
1109
+ >>> resp.status_code = 400
1110
+ >>> ex = requests.HTTPError("error", response=resp)
1111
+ >>> _is_retriable_exception(ex)
1112
+ (False, 0)
1113
+ >>> resp._content = b'{"backoff": 13000, "debug_info": {"retriable": false, "type": "RequestRateLimitedError", "message": "Request rate limit has been exceeded"}}'
1114
+ >>> resp.status_code = 400
1115
+ >>> ex = requests.HTTPError("error", response=resp)
1116
+ >>> _is_retriable_exception(ex)
1117
+ (True, 13)
1118
+ >>> resp._content = b'{"backoff": "foo", "debug_info": {"retriable": false, "type": "RequestRateLimitedError", "message": "Request rate limit has been exceeded"}}'
1119
+ >>> resp.status_code = 400
1120
+ >>> ex = requests.HTTPError("error", response=resp)
1121
+ >>> _is_retriable_exception(ex)
1122
+ (True, 10)
1123
+ >>> resp._content = b'{"debug_info": {"retriable": false, "type": "RequestRateLimitedError", "message": "Request rate limit has been exceeded"}}'
1124
+ >>> resp.status_code = 400
1125
+ >>> ex = requests.HTTPError("error", response=resp)
1126
+ >>> _is_retriable_exception(ex)
1127
+ (True, 10)
1128
+ >>> resp._content = b"foo"
1129
+ >>> resp.status_code = 429
1130
+ >>> ex = requests.HTTPError("error", response=resp)
1131
+ >>> _is_retriable_exception(ex)
1132
+ (True, 10)
1133
+ >>> resp._content = b"foo"
1134
+ >>> resp.status_code = 429
1135
+ >>> ex = requests.HTTPError("error", response=resp)
1136
+ >>> _is_retriable_exception(ex)
1137
+ (True, 10)
1138
+ >>> resp._content = b'{"backoff": 12000, "debug_info": {"retriable": false, "type": "RequestRateLimitedError", "message": "Request rate limit has been exceeded"}}'
1139
+ >>> resp.status_code = 429
1140
+ >>> ex = requests.HTTPError("error", response=resp)
1141
+ >>> _is_retriable_exception(ex)
1142
+ (True, 12)
1143
+ >>> resp._content = b'{"backoff": 12000, "debug_info": {"retriable": false, "type": "RequestRateLimitedError", "message": "Request rate limit has been exceeded"}}'
1144
+ >>> resp.headers = {"Retry-After": "1"}
1145
+ >>> resp.status_code = 503
1146
+ >>> ex = requests.HTTPError("error", response=resp)
1147
+ >>> _is_retriable_exception(ex)
1148
+ (True, 1)
1149
+ """
1150
+
1151
+ DEFAULT_RETRY_AFTER_RATE_LIMIT_SEC = 10
1152
+
1112
1153
  if isinstance(ex, (requests.ConnectionError, requests.Timeout)):
1113
- return True
1154
+ return True, 0
1114
1155
 
1115
1156
  if isinstance(ex, requests.HTTPError) and isinstance(
1116
1157
  ex.response, requests.Response
1117
1158
  ):
1118
- if 400 <= ex.response.status_code < 500:
1159
+ status_code = ex.response.status_code
1160
+
1161
+ # Always retry with some delay
1162
+ if status_code == 429:
1163
+ retry_after_sec = (
1164
+ _parse_retry_after_from_header(ex.response)
1165
+ or DEFAULT_RETRY_AFTER_RATE_LIMIT_SEC
1166
+ )
1167
+
1119
1168
  try:
1120
- resp = ex.response.json()
1121
- except json.JSONDecodeError:
1122
- return False
1123
- return resp.get("debug_info", {}).get("retriable", False)
1124
- else:
1125
- return True
1169
+ data = ex.response.json()
1170
+ except requests.JSONDecodeError:
1171
+ return True, retry_after_sec
1126
1172
 
1127
- return False
1173
+ backoff_ms = _parse_backoff(data.get("backoff"))
1174
+ if backoff_ms is None:
1175
+ return True, retry_after_sec
1176
+ else:
1177
+ return True, max(0, int(int(backoff_ms) / 1000))
1178
+
1179
+ if 400 <= status_code < 500:
1180
+ try:
1181
+ data = ex.response.json()
1182
+ except requests.JSONDecodeError:
1183
+ return False, (_parse_retry_after_from_header(ex.response) or 0)
1184
+
1185
+ debug_info = data.get("debug_info", {})
1186
+
1187
+ if isinstance(debug_info, dict):
1188
+ error_type = debug_info.get("type")
1189
+ else:
1190
+ error_type = None
1191
+
1192
+ # The server may respond 429 RequestRateLimitedError but with retryable=False
1193
+ # We should retry for this case regardless
1194
+ # e.g. HTTP 429 {"backoff": 10000, "debug_info": {"retriable": false, "type": "RequestRateLimitedError", "message": "Request rate limit has been exceeded"}}
1195
+ if error_type == "RequestRateLimitedError":
1196
+ backoff_ms = _parse_backoff(data.get("backoff"))
1197
+ if backoff_ms is None:
1198
+ return True, (
1199
+ _parse_retry_after_from_header(ex.response)
1200
+ or DEFAULT_RETRY_AFTER_RATE_LIMIT_SEC
1201
+ )
1202
+ else:
1203
+ return True, max(0, int(int(backoff_ms) / 1000))
1204
+
1205
+ return debug_info.get("retriable", False), 0
1206
+
1207
+ if 500 <= status_code < 600:
1208
+ return True, (_parse_retry_after_from_header(ex.response) or 0)
1209
+
1210
+ return False, 0
1211
+
1212
+
1213
+ def _parse_backoff(backoff: T.Any) -> int | None:
1214
+ if backoff is not None:
1215
+ try:
1216
+ backoff_ms = int(backoff)
1217
+ except (ValueError, TypeError):
1218
+ backoff_ms = None
1219
+ else:
1220
+ backoff_ms = None
1221
+ return backoff_ms
1222
+
1223
+
1224
+ def _parse_retry_after_from_header(resp: requests.Response) -> int | None:
1225
+ """
1226
+ Parse Retry-After header from HTTP response.
1227
+ See See https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Retry-After
1228
+
1229
+ Args:
1230
+ resp: HTTP response object with headers
1231
+
1232
+ Returns:
1233
+ Number of seconds to wait (>= 0) or None if header missing/invalid.
1234
+
1235
+ Examples:
1236
+ >>> resp = requests.Response()
1237
+ >>> resp.headers = {"Retry-After": "1"}
1238
+ >>> _parse_retry_after_from_header(resp)
1239
+ 1
1240
+ >>> resp.headers = {"Retry-After": "-1"}
1241
+ >>> _parse_retry_after_from_header(resp)
1242
+ 0
1243
+ >>> resp.headers = {"Retry-After": "Wed, 21 Oct 2015 07:28:00 GMT"}
1244
+ >>> _parse_retry_after_from_header(resp)
1245
+ 0
1246
+ >>> resp.headers = {"Retry-After": "Wed, 21 Oct 2315 07:28:00"}
1247
+ >>> _parse_retry_after_from_header(resp)
1248
+ """
1249
+
1250
+ value = resp.headers.get("Retry-After")
1251
+ if value is None:
1252
+ return None
1253
+
1254
+ try:
1255
+ return max(0, int(value))
1256
+ except (ValueError, TypeError):
1257
+ pass
1258
+
1259
+ # e.g. "Wed, 21 Oct 2015 07:28:00 GMT"
1260
+ try:
1261
+ dt = email.utils.parsedate_to_datetime(value)
1262
+ except (ValueError, TypeError):
1263
+ dt = None
1264
+
1265
+ if dt is None:
1266
+ LOG.warning(f"Error parsing Retry-After: {value}")
1267
+ return None
1268
+
1269
+ try:
1270
+ delta = dt - datetime.datetime.now(datetime.timezone.utc)
1271
+ except (TypeError, ValueError):
1272
+ # e.g. TypeError: can't subtract offset-naive and offset-aware datetimes
1273
+ return None
1274
+
1275
+ return max(0, int(delta.total_seconds()))
1128
1276
 
1129
1277
 
1130
1278
  _SUFFIX_MAP: dict[api_v4.ClusterFileType | types.FileType, str] = {
@@ -1160,3 +1308,57 @@ def _prefixed_uuid4():
1160
1308
 
1161
1309
  def _is_uuid(key: str) -> bool:
1162
1310
  return key.startswith("uuid_") or key.startswith("mly_tools_uuid_")
1311
+
1312
+
1313
+ def _build_upload_cache_path(upload_options: UploadOptions) -> Path:
1314
+ # Different python/CLI versions use different cache (dbm) formats.
1315
+ # Separate them to avoid conflicts
1316
+ py_version_parts = [str(part) for part in sys.version_info[:3]]
1317
+ version = f"py_{'_'.join(py_version_parts)}_{VERSION}"
1318
+ # File handles are not sharable between different users
1319
+ user_id = str(
1320
+ upload_options.user_items.get(
1321
+ "MAPSettingsUserKey", upload_options.user_items["user_upload_token"]
1322
+ )
1323
+ )
1324
+ # Use hash to avoid log sensitive data
1325
+ user_fingerprint = utils.md5sum_fp(
1326
+ io.BytesIO((api_v4.MAPILLARY_CLIENT_TOKEN + user_id).encode("utf-8")),
1327
+ md5=hashlib.sha256(),
1328
+ ).hexdigest()[:24]
1329
+
1330
+ cache_path = (
1331
+ Path(constants.UPLOAD_CACHE_DIR)
1332
+ .joinpath(version)
1333
+ .joinpath(user_fingerprint)
1334
+ .joinpath("cached_file_handles")
1335
+ )
1336
+
1337
+ return cache_path
1338
+
1339
+
1340
+ def _maybe_create_persistent_cache_instance(
1341
+ upload_options: UploadOptions,
1342
+ ) -> history.PersistentCache | None:
1343
+ """Create a persistent cache instance if caching is enabled."""
1344
+
1345
+ if upload_options.dry_run:
1346
+ LOG.debug("Dry-run mode enabled, skipping caching upload file handles")
1347
+ return None
1348
+
1349
+ if upload_options.upload_cache_path is None:
1350
+ if not constants.UPLOAD_CACHE_DIR:
1351
+ LOG.debug(
1352
+ "Upload cache directory is set empty, skipping caching upload file handles"
1353
+ )
1354
+ return None
1355
+
1356
+ cache_path = _build_upload_cache_path(upload_options)
1357
+ else:
1358
+ cache_path = upload_options.upload_cache_path
1359
+
1360
+ LOG.debug(f"File handle cache path: {cache_path}")
1361
+
1362
+ cache_path.parent.mkdir(parents=True, exist_ok=True)
1363
+
1364
+ return history.PersistentCache(str(cache_path.resolve()))
mapillary_tools/utils.py CHANGED
@@ -247,8 +247,8 @@ def configure_logger(
247
247
  try:
248
248
  # Disable globally for now. TODO Disable it in non-interactive mode only
249
249
  raise ImportError
250
- from rich.console import Console # type: ignore
251
- from rich.logging import RichHandler # type: ignore
250
+ from rich.console import Console # type: ignore[import]
251
+ from rich.logging import RichHandler # type: ignore[import]
252
252
  except ImportError:
253
253
  formatter = logging.Formatter(
254
254
  "%(asctime)s.%(msecs)03d - %(levelname)-7s - %(message)s",
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: mapillary_tools
3
- Version: 0.14.1
3
+ Version: 0.14.3
4
4
  Summary: Mapillary Image/Video Import Pipeline
5
5
  Author-email: Mapillary <support@mapillary.com>
6
6
  License: BSD
@@ -1,8 +1,8 @@
1
- mapillary_tools/__init__.py,sha256=91RNgd9bTuNlioixZy3CssJc8LG26WWwcNDinmjqx20,19
1
+ mapillary_tools/__init__.py,sha256=-BdvXvwpHU687pG7vjnbfxs01J_K3Vy-_CnU53ctcY0,19
2
2
  mapillary_tools/api_v4.py,sha256=bckAU_atUs0pSuqySeY4W0Rs011a21ClJHo_mbbcXXw,4864
3
- mapillary_tools/authenticate.py,sha256=JXmoTPcAyv9difceKIupXz7vyuLcNZ_AnxLrUVdtPPs,11745
3
+ mapillary_tools/authenticate.py,sha256=mmaOwjQ444DcX4lRw2ms3naBg5Y_xwIJAIWeVdsQfqM,11742
4
4
  mapillary_tools/blackvue_parser.py,sha256=ea2JtU9MWU6yB0bQlF970_Of0bJVofSTRq1P30WKW-0,5623
5
- mapillary_tools/config.py,sha256=97zsyPnZXGincEkM__c6UNWH25EMTtRKZGsp6vBpHy8,3326
5
+ mapillary_tools/config.py,sha256=LDxQoL2StjLGPefCN0y67nhIDN3xSE6Qn8G-tWC-oGA,3421
6
6
  mapillary_tools/constants.py,sha256=fk5HBczTBGyDOKQy-grzlf0SafiGwggdF8Ota13Rk0k,6235
7
7
  mapillary_tools/exceptions.py,sha256=uxTgBEfXgGxT0XNGRIAZ5mjtdqsCYfP6gnaXAK_ewBM,2483
8
8
  mapillary_tools/exif_read.py,sha256=nAbZDYAIBx3g4n6QIGKXX3s-A3SkfuvZQBInDrXMlKk,32220
@@ -12,18 +12,18 @@ mapillary_tools/exiftool_read_video.py,sha256=23O_bjUOVq6j7i3xMz6fY-XIEsjinsCejK
12
12
  mapillary_tools/exiftool_runner.py,sha256=g4gSyqeh3D6EnMJ-c3s-RnO2EP_jD354Qkaz0Y-4D04,1658
13
13
  mapillary_tools/ffmpeg.py,sha256=akpvvsjAR-Iiv-hOrUoJvPM9vUU3JqMQ5HJL1_NgwB8,22908
14
14
  mapillary_tools/geo.py,sha256=mWaESfDf_zHmyvnt5aVFro4FGrjiULNsuZ6HfGUWvSA,11009
15
- mapillary_tools/history.py,sha256=LP6e0zEYVBwRGUbFaGoE_AaBIEdpB4XrZsg9qwJVvRI,5344
15
+ mapillary_tools/history.py,sha256=zyXYXB8pO9Buffn-8-Ien4s74hGD7fyPr2QpBeZwEWw,5478
16
16
  mapillary_tools/http.py,sha256=-df_oGyImO2AOmPnXcKMcztlL4LOZLArE6ki81NMGUA,6411
17
17
  mapillary_tools/ipc.py,sha256=DwWQb9hNshx0bg0Fo5NjY0mXjs-FkbR6tIQmjMgMtmg,1089
18
- mapillary_tools/process_geotag_properties.py,sha256=lF2Po83BAGWlLTtS5AFf6MLLTm-kiImD8SW-tWIbyZ0,14419
18
+ mapillary_tools/process_geotag_properties.py,sha256=3EaVvjfKB-O38OjopBcxeEdP6qI5IPIxqmO6isjcXKM,14205
19
19
  mapillary_tools/process_sequence_properties.py,sha256=n4VjQHrgVjksIr3WoBviRhrQIBBDHGXMClolfyz6tu4,24057
20
20
  mapillary_tools/sample_video.py,sha256=pKSj1Vc8e5p1XGjykBuKY9XieTOskc-9L3F4L407jDM,13935
21
21
  mapillary_tools/telemetry.py,sha256=lL6qQbtOZft4DZZrCNK3njlwHT_30zLyYS_YRN5pgHY,1568
22
22
  mapillary_tools/types.py,sha256=pIU2wcxiOUWT5Pd05pgNzY9EVEDlwoldtlF2IIYYvE0,5909
23
- mapillary_tools/upload.py,sha256=yLunE66mZqTOiefN47KwO6auuyL5augbODyf5EafKeA,24084
23
+ mapillary_tools/upload.py,sha256=XejAgmVW4Y33MiQ2g-shvHZA_zXTekEsOUHUHNx2AE4,24047
24
24
  mapillary_tools/upload_api_v4.py,sha256=VgOf7RhfUuzmlSBUp5CpekKIJ0xQrC0r-r0Ds9-wU4I,7344
25
- mapillary_tools/uploader.py,sha256=gsXMyNriaFPFzCmtw9wpFSh6tDe4nrYe1agzwiVVehE,39484
26
- mapillary_tools/utils.py,sha256=cP9idKt4EJqfC0qqOGneSoPNpPiYhaW8VjQ9CLYjESc,8092
25
+ mapillary_tools/uploader.py,sha256=4bd2YGIAJOK5Jx3ZLIzkLAAfBtU2F708_lTtatJvVas,46642
26
+ mapillary_tools/utils.py,sha256=HjTZ01GQv_UNGySaTZ_Mc1Gn_Y0x3knQf7Vh17whDFw,8108
27
27
  mapillary_tools/camm/camm_builder.py,sha256=ub6Z9ijep8zAo1NOlU51Gxk95kQ2vfN58YgVCLmNMRk,9211
28
28
  mapillary_tools/camm/camm_parser.py,sha256=aNHP65hNXYQBWBTfhaj_S5XYzmAHhjwcAfGhbm83__o,18043
29
29
  mapillary_tools/commands/__init__.py,sha256=41CFrPLGlG3566uhxssEF3TGAtSpADFPPcDMHbViU0E,171
@@ -48,7 +48,7 @@ mapillary_tools/geotag/geotag_images_from_video.py,sha256=3NV3-NfSkxT0n_n8Ajqjab
48
48
  mapillary_tools/geotag/geotag_videos_from_exiftool.py,sha256=Splhtv21JvrbFPVuKycf5wen0wOJ0zqOWk8d4aSw-ys,5345
49
49
  mapillary_tools/geotag/geotag_videos_from_gpx.py,sha256=IoV7asxl7dojF1lftvohm1jK_LboFg_mBz25GiV_CsY,1653
50
50
  mapillary_tools/geotag/geotag_videos_from_video.py,sha256=T8XS4lVF2Wz4eDXNi5Vt076M5dxjxJXibVrWhqVvErs,863
51
- mapillary_tools/geotag/options.py,sha256=0CSQUuhVE-WdAVQXWXjwMtxujW1yB6yxRkFmVKH8LkQ,5092
51
+ mapillary_tools/geotag/options.py,sha256=AgINCSBlxT9Etfu05zuzunWgcCHVWdFjWVL7oBduL4g,5166
52
52
  mapillary_tools/geotag/utils.py,sha256=tixXiN3uda2HuMnuXVu4xapgoSzZ86kNlJJQ66QERk0,1588
53
53
  mapillary_tools/geotag/image_extractors/base.py,sha256=XoNrLCbJtd-MN-snbhv6zyr6zBfJRoJkntV0ptrh6qg,358
54
54
  mapillary_tools/geotag/image_extractors/exif.py,sha256=cCUegbFqWxjAP4oOP1nZmwoJISWeKgjGO8h_t7nucHs,2079
@@ -66,11 +66,11 @@ mapillary_tools/mp4/io_utils.py,sha256=KZaJTlgFS27Oh3pcA5MKXYFoCifqgFaEZJyU6lb1j
66
66
  mapillary_tools/mp4/mp4_sample_parser.py,sha256=0ILTq8M6mXFTI3agKgljpvO9uYa7HXGUGZpdHT8a6ac,11547
67
67
  mapillary_tools/mp4/simple_mp4_builder.py,sha256=9TUGk1hzI6mQFN1P30jwHL3dCYz3Zz7rsm8UBvMAqMc,12734
68
68
  mapillary_tools/mp4/simple_mp4_parser.py,sha256=g3vvPhBoNu7anhVzC5_XQCV7IwfRWro1vJ6d6GyDkHE,6315
69
- mapillary_tools/serializer/description.py,sha256=FB-DawgHVW5M2e81Uo921bY7DQv9fogXF3QMilpmEhQ,18473
69
+ mapillary_tools/serializer/description.py,sha256=ECnQxC-1LOgkAKE5qFi9Y2KuCeH8KPUjjNFDiwebjvo,18647
70
70
  mapillary_tools/serializer/gpx.py,sha256=_xx6gHjaWHrlXaUpB5GGBrbRKzbExFyIzWWAH-CvksI,4383
71
- mapillary_tools-0.14.1.dist-info/licenses/LICENSE,sha256=l2D8cKfFmmJq_wcVq_JElPJrlvWQOzNWx7gMLINucxc,1292
72
- mapillary_tools-0.14.1.dist-info/METADATA,sha256=NmnszTprFtSwwR7VYc2kwUl3SYOiE9ClmQ5TWMPyA80,22200
73
- mapillary_tools-0.14.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
74
- mapillary_tools-0.14.1.dist-info/entry_points.txt,sha256=A3f3LP-BO_P-U8Y29QfpT4jx6Mjk3sXjTi2Yew4bvj8,75
75
- mapillary_tools-0.14.1.dist-info/top_level.txt,sha256=FbDkMgOrt1S70ho1WSBrOwzKOSkJFDwwqFOoY5-527s,16
76
- mapillary_tools-0.14.1.dist-info/RECORD,,
71
+ mapillary_tools-0.14.3.dist-info/licenses/LICENSE,sha256=l2D8cKfFmmJq_wcVq_JElPJrlvWQOzNWx7gMLINucxc,1292
72
+ mapillary_tools-0.14.3.dist-info/METADATA,sha256=f-tqdyREvL0ZXxfm_Mao2KdWkLsWhHzglP6S6SYMjTU,22200
73
+ mapillary_tools-0.14.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
74
+ mapillary_tools-0.14.3.dist-info/entry_points.txt,sha256=A3f3LP-BO_P-U8Y29QfpT4jx6Mjk3sXjTi2Yew4bvj8,75
75
+ mapillary_tools-0.14.3.dist-info/top_level.txt,sha256=FbDkMgOrt1S70ho1WSBrOwzKOSkJFDwwqFOoY5-527s,16
76
+ mapillary_tools-0.14.3.dist-info/RECORD,,