mapillary-tools 0.11.0b4__tar.gz → 0.11.2__tar.gz

{mapillary_tools-0.11.0b4 → mapillary_tools-0.11.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mapillary_tools
-Version: 0.11.0b4
+Version: 0.11.2
 Summary: Mapillary Image/Video Import Pipeline
 Home-page: https://github.com/mapillary/mapillary_tools
 Author: Mapillary
@@ -13,7 +13,7 @@ Requires-Dist: construct<3.0.0,>=2.10.0
 Requires-Dist: exifread==2.3.2
 Requires-Dist: piexif==1.1.3
 Requires-Dist: gpxpy<1.6.0,>=1.5.0
-Requires-Dist: pynmea2==1.12.0
+Requires-Dist: pynmea2<2.0.0,>=1.12.0
 Requires-Dist: requests<3.0.0,>=2.20.0
 Requires-Dist: tqdm<5.0,>=4.0
 Requires-Dist: typing_extensions
@@ -130,7 +130,7 @@ A command line program such as Termux is required. Installation can be done with
 commands will install Python 3, pip3, git, and all required libraries for mapillary_tools on Termux:
 
 ```sh
-pkg install python git build-essential libgeos openssl libjpeg-turbo
+pkg install python git build-essential libgeos openssl libjpeg-turbo libexpat libexpat-static
 pip install --upgrade pip wheel
 pip install --upgrade mapillary_tools
 ```

{mapillary_tools-0.11.0b4 → mapillary_tools-0.11.2}/README.md

@@ -108,7 +108,7 @@ A command line program such as Termux is required. Installation can be done with
 commands will install Python 3, pip3, git, and all required libraries for mapillary_tools on Termux:
 
 ```sh
-pkg install python git build-essential libgeos openssl libjpeg-turbo
+pkg install python git build-essential libgeos openssl libjpeg-turbo libexpat libexpat-static
 pip install --upgrade pip wheel
 pip install --upgrade mapillary_tools
 ```

mapillary_tools-0.11.2/mapillary_tools/__init__.py

@@ -0,0 +1 @@
+VERSION = "0.11.2"

{mapillary_tools-0.11.0b4 → mapillary_tools-0.11.2}/mapillary_tools/api_v4.py

@@ -9,6 +9,10 @@ MAPILLARY_CLIENT_TOKEN = os.getenv(
 MAPILLARY_GRAPH_API_ENDPOINT = os.getenv(
     "MAPILLARY_GRAPH_API_ENDPOINT", "https://graph.mapillary.com"
 )
+# https://requests.readthedocs.io/en/latest/user/advanced/#ssl-cert-verification
+MAPILLARY__DISABLE_VERIFYING_SSL = (
+    os.getenv("MAPILLARY__DISABLE_VERIFYING_SSL") == "TRUE"
+)
 REQUESTS_TIMEOUT = 60  # 1 minutes
 
 
@@ -18,6 +22,7 @@ def get_upload_token(email: str, password: str) -> requests.Response:
         params={"access_token": MAPILLARY_CLIENT_TOKEN},
         json={"email": email, "password": password, "locale": "en_US"},
         timeout=REQUESTS_TIMEOUT,
+        verify=not MAPILLARY__DISABLE_VERIFYING_SSL,
     )
     resp.raise_for_status()
     return resp
@@ -35,6 +40,7 @@ def fetch_organization(
             "Authorization": f"OAuth {user_access_token}",
         },
         timeout=REQUESTS_TIMEOUT,
+        verify=not MAPILLARY__DISABLE_VERIFYING_SSL,
     )
     resp.raise_for_status()
     return resp
@@ -56,6 +62,7 @@ def logging(action_type: ActionType, properties: T.Dict) -> requests.Response:
             "Authorization": f"OAuth {MAPILLARY_CLIENT_TOKEN}",
         },
         timeout=REQUESTS_TIMEOUT,
+        verify=not MAPILLARY__DISABLE_VERIFYING_SSL,
     )
     resp.raise_for_status()
     return resp

{mapillary_tools-0.11.0b4 → mapillary_tools-0.11.2}/mapillary_tools/authenticate.py

@@ -51,7 +51,11 @@ def prompt_user_for_user_items(user_name: str) -> types.UserItem:
     try:
         resp = api_v4.get_upload_token(user_email, user_password)
     except requests.HTTPError as ex:
-        if 400 <= ex.response.status_code < 500:
+        if (
+            isinstance(ex, requests.HTTPError)
+            and isinstance(ex.response, requests.Response)
+            and 400 <= ex.response.status_code < 500
+        ):
             r = ex.response.json()
             subcode = r.get("error", {}).get("error_subcode")
             if subcode in [1348028, 1348092, 3404005, 1348131]:

{mapillary_tools-0.11.0b4 → mapillary_tools-0.11.2}/mapillary_tools/exif_read.py

@@ -399,7 +399,7 @@ class ExifReadFromXMP(ExifReadABC):
 
     def extract_capture_time(self) -> T.Optional[datetime.datetime]:
         dt = self.extract_gps_datetime()
-        if dt is not None:
+        if dt is not None and dt.date() != datetime.date(1970, 1, 1):
             return dt
 
         dt = self.extract_exif_datetime()
@@ -554,7 +554,7 @@ class ExifReadFromEXIF(ExifReadABC):
             return None
 
         dt = strptime_alternative_formats(gpsdate, ["%Y:%m:%d", "%Y-%m-%d"])
-        if dt is None:
+        if dt is None or dt == datetime.date(1970, 1, 1):
             return None
 
         gpstimestamp = self.tags.get("GPS GPSTimeStamp")
@@ -632,7 +632,7 @@ class ExifReadFromEXIF(ExifReadABC):
             gps_dt = self.extract_gps_datetime()
         except (ValueError, TypeError, ZeroDivisionError):
             gps_dt = None
-        if gps_dt is not None:
+        if gps_dt is not None and gps_dt.date() != datetime.date(1970, 1, 1):
             return gps_dt
 
         dt = self.extract_exif_datetime()

{mapillary_tools-0.11.0b4 → mapillary_tools-0.11.2}/mapillary_tools/exiftool_read.py

@@ -266,14 +266,14 @@ class ExifToolRead(exif_read.ExifReadABC):
             dt = self.extract_gps_datetime()
         except (ValueError, TypeError, ZeroDivisionError):
             dt = None
-        if dt is not None:
+        if dt is not None and dt.date() != datetime.date(1970, 1, 1):
             return dt
 
         try:
             dt = self.extract_gps_datetime_from_xmp()
         except (ValueError, TypeError, ZeroDivisionError):
             dt = None
-        if dt is not None:
+        if dt is not None and dt.date() != datetime.date(1970, 1, 1):
             return dt
 
         dt = self.extract_exif_datetime()

{mapillary_tools-0.11.0b4 → mapillary_tools-0.11.2}/mapillary_tools/ffmpeg.py

@@ -252,12 +252,18 @@ class FFMPEG:
                     *[
                         *["-filter_script:v", select_file.name],
                         # Each frame is passed with its timestamp from the demuxer to the muxer
-                        # vsync is deprecated but -fps_mode is not avaliable on some versions ;(
                         *["-vsync", "0"],
+                        # vsync is deprecated by fps_mode,
+                        # but fps_mode is not avaliable on some older versions ;(
                         # *[f"-fps_mode:{stream_specifier}", "passthrough"],
                         # Set the number of video frames to output
                         *[f"-frames:{stream_specifier}", str(len(frame_indices))],
-                        *["-frame_pts", "1"],
+                        # Disabled because it doesn't always name the sample images as expected
+                        # For example "select(n\,1)" we expected the first sample to be IMG_001.JPG
+                        # but it could be IMG_005.JPG
+                        # https://www.ffmpeg.org/ffmpeg-formats.html#Options-21
+                        # If set to 1, expand the filename with pts from pkt->pts. Default value is 0.
+                        # *["-frame_pts", "1"],
                     ],
                     # video quality level (or the alias -q:v)
                     *[f"-qscale:{stream_specifier}", "2"],

{mapillary_tools-0.11.0b4 → mapillary_tools-0.11.2}/mapillary_tools/process_geotag_properties.py

@@ -289,6 +289,7 @@ def _process_videos_beta(vars_args: T.Dict):
         "num_processes": vars_args["num_processes"],
         "device_make": vars_args["device_make"],
         "device_model": vars_args["device_model"],
+        "check_file_suffix": len(vars_args["filetypes"]) > 1,
     }
     extractor = VideoDataExtractor(options)
     return extractor.process()

{mapillary_tools-0.11.0b4 → mapillary_tools-0.11.2}/mapillary_tools/process_sequence_properties.py

@@ -195,9 +195,11 @@ def _interpolate_subsecs_for_sorting(sequence: PointSequence) -> None:
 
         t = sequence[gidx].time
         nt = min(
-            sequence[gidx + len(group)].time
-            if gidx + len(group) < len(sequence)
-            else math.floor(t + 1.0),
+            (
+                sequence[gidx + len(group)].time
+                if gidx + len(group) < len(sequence)
+                else math.floor(t + 1.0)
+            ),
             math.floor(t + 1.0),
         )
         assert t <= nt, f"expect sorted but got {t} > {nt}"

{mapillary_tools-0.11.0b4 → mapillary_tools-0.11.2}/mapillary_tools/sample_video.py

@@ -320,35 +320,37 @@ def _sample_single_video_by_distance(
     sample_points_by_frame_idx = _sample_video_stream_by_distance(
         video_metadata.points, video_track_parser, sample_distance
     )
+    sorted_sample_indices = sorted(sample_points_by_frame_idx.keys())
 
     with wip_dir_context(wip_sample_dir(sample_dir), sample_dir) as wip_dir:
         ffmpeg.extract_specified_frames(
             video_path,
             wip_dir,
-            frame_indices=set(sample_points_by_frame_idx.keys()),
+            frame_indices=set(sorted_sample_indices),
             stream_idx=video_stream_idx,
         )
 
         frame_samples = ffmpeglib.sort_selected_samples(
             wip_dir, video_path, [video_stream_idx]
         )
-        # extract_specified_frames() produces 0-based frame indices
-        for frame_idx_0based, sample_paths in frame_samples:
-            assert len(sample_paths) == 1
-            if sample_paths[0] is None:
-                continue
-
-            sample_point = sample_points_by_frame_idx.get(frame_idx_0based)
-            if sample_point is None:
-                # this should not happen
-                LOG.warning(
-                    "The specified frame index %d was not extracted from stream index %d",
-                    frame_idx_0based,
-                    video_stream_idx,
+        if len(frame_samples) != len(sorted_sample_indices):
+            raise exceptions.MapillaryVideoError(
+                f"Expect {len(sorted_sample_indices)} samples but extracted {len(frame_samples)} samples"
+            )
+        for idx, (frame_idx_1based, sample_paths) in enumerate(frame_samples):
+            assert (
+                len(sample_paths) == 1
+            ), "Expect 1 sample path at {frame_idx_1based} but got {sample_paths}"
+            if idx + 1 != frame_idx_1based:
+                raise exceptions.MapillaryVideoError(
+                    f"Expect {sample_paths[0]} to be {idx + 1}th sample but got {frame_idx_1based}"
                 )
+
+        for (_, sample_paths), sample_idx in zip(frame_samples, sorted_sample_indices):
+            if sample_paths[0] is None:
                 continue
 
-            video_sample, interp = sample_point
+            video_sample, interp = sample_points_by_frame_idx[sample_idx]
             assert (
                 interp.time == video_sample.composition_time_offset
             ), f"interpolated time {interp.time} should match the video sample time {video_sample.composition_time_offset}"

{mapillary_tools-0.11.0b4 → mapillary_tools-0.11.2}/mapillary_tools/types.py

@@ -456,18 +456,15 @@ def map_capture_time_to_datetime(time: str) -> datetime.datetime:
 
 
 @T.overload
-def as_desc(metadata: ImageMetadata) -> ImageDescription:
-    ...
+def as_desc(metadata: ImageMetadata) -> ImageDescription: ...
 
 
 @T.overload
-def as_desc(metadata: ErrorMetadata) -> ImageDescriptionError:
-    ...
+def as_desc(metadata: ErrorMetadata) -> ImageDescriptionError: ...
 
 
 @T.overload
-def as_desc(metadata: VideoMetadata) -> VideoDescription:
-    ...
+def as_desc(metadata: VideoMetadata) -> VideoDescription: ...
 
 
 def as_desc(metadata):
@@ -524,13 +521,11 @@ def _as_image_desc(metadata: ImageMetadata) -> ImageDescription:
 
 
 @T.overload
-def from_desc(metadata: ImageDescription) -> ImageMetadata:
-    ...
+def from_desc(metadata: ImageDescription) -> ImageMetadata: ...
 
 
 @T.overload
-def from_desc(metadata: VideoDescription) -> VideoMetadata:
-    ...
+def from_desc(metadata: VideoDescription) -> VideoMetadata: ...
 
 
 def from_desc(desc):

{mapillary_tools-0.11.0b4 → mapillary_tools-0.11.2}/mapillary_tools/upload.py

@@ -62,12 +62,17 @@ class DirectUploadFileType(enum.Enum):
 
 
 def wrap_http_exception(ex: requests.HTTPError):
+    req = ex.request
     resp = ex.response
-    lines = [
-        f"{ex.request.method} {resp.url}",
-        f"> HTTP Status: {ex.response.status_code}",
-        str(resp.content),
-    ]
+    if isinstance(resp, requests.Response) and isinstance(req, requests.Request):
+        lines = [
+            f"{req.method} {resp.url}",
+            f"> HTTP Status: {resp.status_code}",
+            str(resp.content),
+        ]
+    else:
+        lines = []
+
     return UploadHTTPError("\n".join(lines))
 
 
@@ -713,7 +718,9 @@ def upload(
         if isinstance(inner_ex, requests.Timeout):
             raise exceptions.MapillaryUploadTimeoutError(str(inner_ex)) from inner_ex
 
-        if isinstance(inner_ex, requests.HTTPError):
+        if isinstance(inner_ex, requests.HTTPError) and isinstance(
+            inner_ex.response, requests.Response
+        ):
             if inner_ex.response.status_code in [400, 401]:
                 try:
                     error_body = inner_ex.response.json()

{mapillary_tools-0.11.0b4 → mapillary_tools-0.11.2}/mapillary_tools/upload_api_v4.py

@@ -15,6 +15,10 @@ MAPILLARY_UPLOAD_ENDPOINT = os.getenv(
 MAPILLARY_GRAPH_API_ENDPOINT = os.getenv(
     "MAPILLARY_GRAPH_API_ENDPOINT", "https://graph.mapillary.com"
 )
+# https://requests.readthedocs.io/en/latest/user/advanced/#ssl-cert-verification
+MAPILLARY__DISABLE_VERIFYING_SSL = (
+    os.getenv("MAPILLARY__DISABLE_VERIFYING_SSL") == "TRUE"
+)
 DEFAULT_CHUNK_SIZE = 1024 * 1024 * 16  # 16MB
 # According to the docs, UPLOAD_REQUESTS_TIMEOUT can be a tuple of
 # (connection_timeout, read_timeout): https://requests.readthedocs.io/en/latest/user/advanced/#timeouts
@@ -97,6 +101,7 @@ class UploadService:
             url,
             headers=headers,
             timeout=REQUESTS_TIMEOUT,
+            verify=not MAPILLARY__DISABLE_VERIFYING_SSL,
         )
         LOG.debug("HTTP response %s: %s", resp.status_code, resp.content)
         resp.raise_for_status()
@@ -139,6 +144,7 @@ class UploadService:
                 headers=headers,
                 data=chunk,
                 timeout=UPLOAD_REQUESTS_TIMEOUT,
+                verify=not MAPILLARY__DISABLE_VERIFYING_SSL,
             )
             LOG.debug(
                 "HTTP response %s: %s", resp.status_code, _truncate_end(resp.content)
@@ -185,6 +191,7 @@ class UploadService:
             headers=headers,
             json=data,
             timeout=REQUESTS_TIMEOUT,
+            verify=not MAPILLARY__DISABLE_VERIFYING_SSL,
         )
         LOG.debug("HTTP response %s: %s", resp.status_code, _truncate_end(resp.content))
 

{mapillary_tools-0.11.0b4 → mapillary_tools-0.11.2}/mapillary_tools/uploader.py

@@ -321,7 +321,11 @@ def _extract_upload_md5sum(fp: T.IO[bytes]) -> T.Optional[str]:
 
 
 def _is_immediate_retry(ex: Exception):
-    if isinstance(ex, requests.HTTPError) and ex.response.status_code == 412:
+    if (
+        isinstance(ex, requests.HTTPError)
+        and isinstance(ex.response, requests.Response)
+        and ex.response.status_code == 412
+    ):
         try:
             resp = ex.response.json()
         except json.JSONDecodeError:
@@ -334,7 +338,9 @@ def _is_retriable_exception(ex: Exception):
     if isinstance(ex, (requests.ConnectionError, requests.Timeout)):
         return True
 
-    if isinstance(ex, requests.HTTPError):
+    if isinstance(ex, requests.HTTPError) and isinstance(
+        ex.response, requests.Response
+    ):
         if 400 <= ex.response.status_code < 500:
             try:
                 resp = ex.response.json()

{mapillary_tools-0.11.0b4 → mapillary_tools-0.11.2}/mapillary_tools/video_data_extraction/cli_options.py

@@ -20,3 +20,4 @@ class CliOptions(T.TypedDict, total=False):
     num_processes: int
     device_make: T.Optional[str]
     device_model: T.Optional[str]
+    check_file_suffix: bool

{mapillary_tools-0.11.0b4 → mapillary_tools-0.11.2}/mapillary_tools/video_data_extraction/extract_video_data.py

@@ -33,7 +33,9 @@ class VideoDataExtractor:
     def process(self) -> T.List[MetadataOrError]:
         paths = self.options["paths"]
         self._check_paths(paths)
-        video_files = utils.find_videos(paths)
+        video_files = utils.find_videos(
+            paths, check_file_suffix=self.options["check_file_suffix"]
+        )
         self._check_sources_cardinality(video_files)
 
         num_processes = self.options["num_processes"] or None
@@ -102,10 +104,12 @@ class VideoDataExtractor:
         else:
             return ErrorMetadata(
                 filename=file,
-                error=ex
-                if ex
-                else exceptions.MapillaryVideoGPSNotFoundError(
-                    "No GPS data found from the video"
+                error=(
+                    ex
+                    if ex
+                    else exceptions.MapillaryVideoGPSNotFoundError(
+                        "No GPS data found from the video"
+                    )
                 ),
                 filetype=FileType.VIDEO,
             )

{mapillary_tools-0.11.0b4 → mapillary_tools-0.11.2}/mapillary_tools/video_data_extraction/extractors/exiftool_runtime_parser.py

@@ -30,13 +30,17 @@ class ExiftoolRuntimeParser(BaseParser):
     ):
         super().__init__(video_path, options, parser_options)
         exiftool_path = shutil.which(constants.EXIFTOOL_PATH)
+
         if not exiftool_path:
             raise exceptions.MapillaryExiftoolNotFoundError(
                 "Cannot execute exiftool. Please install it from https://exiftool.org/ or you package manager, or set the environment variable MAPILLARY_TOOLS_EXIFTOOL_PATH"
             )
-
         if not self.geotag_source_path:
             return
+
+        # To handle non-latin1 filenames under Windows, we pass the path
+        # via stdin. See https://exiftool.org/faq.html#Q18
+        stdin = str(self.geotag_source_path)
         args = [
             exiftool_path,
             "-q",
@@ -46,13 +50,18 @@ class ExiftoolRuntimeParser(BaseParser):
             "-api",
             "LargeFileSupport=1",
             "-X",
-            str(self.geotag_source_path),
+            "-charset",
+            "filename=utf8",
+            "-@",
+            "-",
         ]
 
-        xml_content = subprocess.run(args, capture_output=True, text=True).stdout
+        process = subprocess.run(
+            args, capture_output=True, text=True, input=stdin, encoding="utf-8"
+        )
 
         self.exiftoolXmlParser = ExiftoolXmlParser(
-            video_path, options, parser_options, xml_content
+            video_path, options, parser_options, process.stdout
         )
 
     def extract_points(self) -> T.Sequence[geo.Point]:

{mapillary_tools-0.11.0b4 → mapillary_tools-0.11.2}/mapillary_tools.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
-Name: mapillary-tools
-Version: 0.11.0b4
+Name: mapillary_tools
+Version: 0.11.2
 Summary: Mapillary Image/Video Import Pipeline
 Home-page: https://github.com/mapillary/mapillary_tools
 Author: Mapillary
@@ -13,7 +13,7 @@ Requires-Dist: construct<3.0.0,>=2.10.0
 Requires-Dist: exifread==2.3.2
 Requires-Dist: piexif==1.1.3
 Requires-Dist: gpxpy<1.6.0,>=1.5.0
-Requires-Dist: pynmea2==1.12.0
+Requires-Dist: pynmea2<2.0.0,>=1.12.0
 Requires-Dist: requests<3.0.0,>=2.20.0
 Requires-Dist: tqdm<5.0,>=4.0
 Requires-Dist: typing_extensions
@@ -130,7 +130,7 @@ A command line program such as Termux is required. Installation can be done with
 commands will install Python 3, pip3, git, and all required libraries for mapillary_tools on Termux:
 
 ```sh
-pkg install python git build-essential libgeos openssl libjpeg-turbo
+pkg install python git build-essential libgeos openssl libjpeg-turbo libexpat libexpat-static
 pip install --upgrade pip wheel
 pip install --upgrade mapillary_tools
 ```

{mapillary_tools-0.11.0b4 → mapillary_tools-0.11.2}/mapillary_tools.egg-info/requires.txt

@@ -3,7 +3,7 @@ construct<3.0.0,>=2.10.0
 exifread==2.3.2
 piexif==1.1.3
 gpxpy<1.6.0,>=1.5.0
-pynmea2==1.12.0
+pynmea2<2.0.0,>=1.12.0
 requests<3.0.0,>=2.20.0
 tqdm<5.0,>=4.0
 typing_extensions

{mapillary_tools-0.11.0b4 → mapillary_tools-0.11.2}/requirements.txt

@@ -3,7 +3,7 @@ construct>=2.10.0,<3.0.0
 exifread==2.3.2
 piexif==1.1.3
 gpxpy>=1.5.0,<1.6.0
-pynmea2==1.12.0
+pynmea2>=1.12.0,<2.0.0
 requests>=2.20.0,<3.0.0
 tqdm>=4.0,<5.0
 typing_extensions

mapillary_tools-0.11.0b4/mapillary_tools/__init__.py

@@ -1 +0,0 @@
-VERSION = "0.11.0b4"