mapillary-tools 0.11.1__tar.gz → 0.11.2__tar.gz

{mapillary_tools-0.11.1 → mapillary_tools-0.11.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mapillary_tools
-Version: 0.11.1
+Version: 0.11.2
 Summary: Mapillary Image/Video Import Pipeline
 Home-page: https://github.com/mapillary/mapillary_tools
 Author: Mapillary
@@ -13,7 +13,7 @@ Requires-Dist: construct<3.0.0,>=2.10.0
 Requires-Dist: exifread==2.3.2
 Requires-Dist: piexif==1.1.3
 Requires-Dist: gpxpy<1.6.0,>=1.5.0
-Requires-Dist: pynmea2==1.12.0
+Requires-Dist: pynmea2<2.0.0,>=1.12.0
 Requires-Dist: requests<3.0.0,>=2.20.0
 Requires-Dist: tqdm<5.0,>=4.0
 Requires-Dist: typing_extensions
@@ -130,7 +130,7 @@ A command line program such as Termux is required. Installation can be done with
 commands will install Python 3, pip3, git, and all required libraries for mapillary_tools on Termux:
 
 ```sh
-pkg install python git build-essential libgeos openssl libjpeg-turbo
+pkg install python git build-essential libgeos openssl libjpeg-turbo libexpat libexpat-static
 pip install --upgrade pip wheel
 pip install --upgrade mapillary_tools
 ```

{mapillary_tools-0.11.1 → mapillary_tools-0.11.2}/README.md

@@ -108,7 +108,7 @@ A command line program such as Termux is required. Installation can be done with
 commands will install Python 3, pip3, git, and all required libraries for mapillary_tools on Termux:
 
 ```sh
-pkg install python git build-essential libgeos openssl libjpeg-turbo
+pkg install python git build-essential libgeos openssl libjpeg-turbo libexpat libexpat-static
 pip install --upgrade pip wheel
 pip install --upgrade mapillary_tools
 ```

mapillary_tools-0.11.2/mapillary_tools/__init__.py

@@ -0,0 +1 @@
+VERSION = "0.11.2"

{mapillary_tools-0.11.1 → mapillary_tools-0.11.2}/mapillary_tools/api_v4.py

@@ -9,6 +9,10 @@ MAPILLARY_CLIENT_TOKEN = os.getenv(
 MAPILLARY_GRAPH_API_ENDPOINT = os.getenv(
     "MAPILLARY_GRAPH_API_ENDPOINT", "https://graph.mapillary.com"
 )
+# https://requests.readthedocs.io/en/latest/user/advanced/#ssl-cert-verification
+MAPILLARY__DISABLE_VERIFYING_SSL = (
+    os.getenv("MAPILLARY__DISABLE_VERIFYING_SSL") == "TRUE"
+)
 REQUESTS_TIMEOUT = 60  # 1 minutes
 
 
@@ -18,6 +22,7 @@ def get_upload_token(email: str, password: str) -> requests.Response:
         params={"access_token": MAPILLARY_CLIENT_TOKEN},
         json={"email": email, "password": password, "locale": "en_US"},
         timeout=REQUESTS_TIMEOUT,
+        verify=not MAPILLARY__DISABLE_VERIFYING_SSL,
     )
     resp.raise_for_status()
     return resp
@@ -35,6 +40,7 @@ def fetch_organization(
             "Authorization": f"OAuth {user_access_token}",
         },
         timeout=REQUESTS_TIMEOUT,
+        verify=not MAPILLARY__DISABLE_VERIFYING_SSL,
     )
     resp.raise_for_status()
     return resp
@@ -56,6 +62,7 @@ def logging(action_type: ActionType, properties: T.Dict) -> requests.Response:
             "Authorization": f"OAuth {MAPILLARY_CLIENT_TOKEN}",
         },
         timeout=REQUESTS_TIMEOUT,
+        verify=not MAPILLARY__DISABLE_VERIFYING_SSL,
     )
     resp.raise_for_status()
     return resp

{mapillary_tools-0.11.1 → mapillary_tools-0.11.2}/mapillary_tools/exif_read.py

@@ -399,7 +399,7 @@ class ExifReadFromXMP(ExifReadABC):
 
     def extract_capture_time(self) -> T.Optional[datetime.datetime]:
         dt = self.extract_gps_datetime()
-        if dt is not None:
+        if dt is not None and dt.date() != datetime.date(1970, 1, 1):
             return dt
 
         dt = self.extract_exif_datetime()
@@ -554,7 +554,7 @@ class ExifReadFromEXIF(ExifReadABC):
             return None
 
         dt = strptime_alternative_formats(gpsdate, ["%Y:%m:%d", "%Y-%m-%d"])
-        if dt is None:
+        if dt is None or dt == datetime.date(1970, 1, 1):
             return None
 
         gpstimestamp = self.tags.get("GPS GPSTimeStamp")
@@ -632,7 +632,7 @@ class ExifReadFromEXIF(ExifReadABC):
             gps_dt = self.extract_gps_datetime()
         except (ValueError, TypeError, ZeroDivisionError):
             gps_dt = None
-        if gps_dt is not None:
+        if gps_dt is not None and gps_dt.date() != datetime.date(1970, 1, 1):
             return gps_dt
 
         dt = self.extract_exif_datetime()

{mapillary_tools-0.11.1 → mapillary_tools-0.11.2}/mapillary_tools/exiftool_read.py

@@ -266,14 +266,14 @@ class ExifToolRead(exif_read.ExifReadABC):
             dt = self.extract_gps_datetime()
         except (ValueError, TypeError, ZeroDivisionError):
             dt = None
-        if dt is not None:
+        if dt is not None and dt.date() != datetime.date(1970, 1, 1):
             return dt
 
         try:
             dt = self.extract_gps_datetime_from_xmp()
         except (ValueError, TypeError, ZeroDivisionError):
             dt = None
-        if dt is not None:
+        if dt is not None and dt.date() != datetime.date(1970, 1, 1):
             return dt
 
         dt = self.extract_exif_datetime()

{mapillary_tools-0.11.1 → mapillary_tools-0.11.2}/mapillary_tools/ffmpeg.py

@@ -252,12 +252,18 @@ class FFMPEG:
                     *[
                         *["-filter_script:v", select_file.name],
                         # Each frame is passed with its timestamp from the demuxer to the muxer
-                        # vsync is deprecated but -fps_mode is not avaliable on some versions ;(
                         *["-vsync", "0"],
+                        # vsync is deprecated by fps_mode,
+                        # but fps_mode is not avaliable on some older versions ;(
                         # *[f"-fps_mode:{stream_specifier}", "passthrough"],
                         # Set the number of video frames to output
                         *[f"-frames:{stream_specifier}", str(len(frame_indices))],
-                        *["-frame_pts", "1"],
+                        # Disabled because it doesn't always name the sample images as expected
+                        # For example "select(n\,1)" we expected the first sample to be IMG_001.JPG
+                        # but it could be IMG_005.JPG
+                        # https://www.ffmpeg.org/ffmpeg-formats.html#Options-21
+                        # If set to 1, expand the filename with pts from pkt->pts. Default value is 0.
+                        # *["-frame_pts", "1"],
                     ],
                     # video quality level (or the alias -q:v)
                     *[f"-qscale:{stream_specifier}", "2"],

{mapillary_tools-0.11.1 → mapillary_tools-0.11.2}/mapillary_tools/process_sequence_properties.py

@@ -195,9 +195,11 @@ def _interpolate_subsecs_for_sorting(sequence: PointSequence) -> None:
 
         t = sequence[gidx].time
         nt = min(
-            sequence[gidx + len(group)].time
-            if gidx + len(group) < len(sequence)
-            else math.floor(t + 1.0),
+            (
+                sequence[gidx + len(group)].time
+                if gidx + len(group) < len(sequence)
+                else math.floor(t + 1.0)
+            ),
             math.floor(t + 1.0),
         )
         assert t <= nt, f"expect sorted but got {t} > {nt}"

{mapillary_tools-0.11.1 → mapillary_tools-0.11.2}/mapillary_tools/sample_video.py

@@ -320,35 +320,37 @@ def _sample_single_video_by_distance(
     sample_points_by_frame_idx = _sample_video_stream_by_distance(
         video_metadata.points, video_track_parser, sample_distance
     )
+    sorted_sample_indices = sorted(sample_points_by_frame_idx.keys())
 
     with wip_dir_context(wip_sample_dir(sample_dir), sample_dir) as wip_dir:
         ffmpeg.extract_specified_frames(
             video_path,
             wip_dir,
-            frame_indices=set(sample_points_by_frame_idx.keys()),
+            frame_indices=set(sorted_sample_indices),
             stream_idx=video_stream_idx,
         )
 
         frame_samples = ffmpeglib.sort_selected_samples(
             wip_dir, video_path, [video_stream_idx]
         )
-        # extract_specified_frames() produces 0-based frame indices
-        for frame_idx_0based, sample_paths in frame_samples:
-            assert len(sample_paths) == 1
-            if sample_paths[0] is None:
-                continue
-
-            sample_point = sample_points_by_frame_idx.get(frame_idx_0based)
-            if sample_point is None:
-                # this should not happen
-                LOG.warning(
-                    "The specified frame index %d was not extracted from stream index %d",
-                    frame_idx_0based,
-                    video_stream_idx,
+        if len(frame_samples) != len(sorted_sample_indices):
+            raise exceptions.MapillaryVideoError(
+                f"Expect {len(sorted_sample_indices)} samples but extracted {len(frame_samples)} samples"
+            )
+        for idx, (frame_idx_1based, sample_paths) in enumerate(frame_samples):
+            assert (
+                len(sample_paths) == 1
+            ), "Expect 1 sample path at {frame_idx_1based} but got {sample_paths}"
+            if idx + 1 != frame_idx_1based:
+                raise exceptions.MapillaryVideoError(
+                    f"Expect {sample_paths[0]} to be {idx + 1}th sample but got {frame_idx_1based}"
                 )
+
+        for (_, sample_paths), sample_idx in zip(frame_samples, sorted_sample_indices):
+            if sample_paths[0] is None:
                 continue
 
-            video_sample, interp = sample_point
+            video_sample, interp = sample_points_by_frame_idx[sample_idx]
             assert (
                 interp.time == video_sample.composition_time_offset
             ), f"interpolated time {interp.time} should match the video sample time {video_sample.composition_time_offset}"

{mapillary_tools-0.11.1 → mapillary_tools-0.11.2}/mapillary_tools/types.py

@@ -456,18 +456,15 @@ def map_capture_time_to_datetime(time: str) -> datetime.datetime:
 
 
 @T.overload
-def as_desc(metadata: ImageMetadata) -> ImageDescription:
-    ...
+def as_desc(metadata: ImageMetadata) -> ImageDescription: ...
 
 
 @T.overload
-def as_desc(metadata: ErrorMetadata) -> ImageDescriptionError:
-    ...
+def as_desc(metadata: ErrorMetadata) -> ImageDescriptionError: ...
 
 
 @T.overload
-def as_desc(metadata: VideoMetadata) -> VideoDescription:
-    ...
+def as_desc(metadata: VideoMetadata) -> VideoDescription: ...
 
 
 def as_desc(metadata):
@@ -524,13 +521,11 @@ def _as_image_desc(metadata: ImageMetadata) -> ImageDescription:
 
 
 @T.overload
-def from_desc(metadata: ImageDescription) -> ImageMetadata:
-    ...
+def from_desc(metadata: ImageDescription) -> ImageMetadata: ...
 
 
 @T.overload
-def from_desc(metadata: VideoDescription) -> VideoMetadata:
-    ...
+def from_desc(metadata: VideoDescription) -> VideoMetadata: ...
 
 
 def from_desc(desc):

{mapillary_tools-0.11.1 → mapillary_tools-0.11.2}/mapillary_tools/upload_api_v4.py

@@ -15,6 +15,10 @@ MAPILLARY_UPLOAD_ENDPOINT = os.getenv(
 MAPILLARY_GRAPH_API_ENDPOINT = os.getenv(
     "MAPILLARY_GRAPH_API_ENDPOINT", "https://graph.mapillary.com"
 )
+# https://requests.readthedocs.io/en/latest/user/advanced/#ssl-cert-verification
+MAPILLARY__DISABLE_VERIFYING_SSL = (
+    os.getenv("MAPILLARY__DISABLE_VERIFYING_SSL") == "TRUE"
+)
 DEFAULT_CHUNK_SIZE = 1024 * 1024 * 16  # 16MB
 # According to the docs, UPLOAD_REQUESTS_TIMEOUT can be a tuple of
 # (connection_timeout, read_timeout): https://requests.readthedocs.io/en/latest/user/advanced/#timeouts
@@ -97,6 +101,7 @@ class UploadService:
             url,
             headers=headers,
             timeout=REQUESTS_TIMEOUT,
+            verify=not MAPILLARY__DISABLE_VERIFYING_SSL,
         )
         LOG.debug("HTTP response %s: %s", resp.status_code, resp.content)
         resp.raise_for_status()
@@ -139,6 +144,7 @@ class UploadService:
                 headers=headers,
                 data=chunk,
                 timeout=UPLOAD_REQUESTS_TIMEOUT,
+                verify=not MAPILLARY__DISABLE_VERIFYING_SSL,
             )
             LOG.debug(
                 "HTTP response %s: %s", resp.status_code, _truncate_end(resp.content)
@@ -185,6 +191,7 @@ class UploadService:
             headers=headers,
             json=data,
             timeout=REQUESTS_TIMEOUT,
+            verify=not MAPILLARY__DISABLE_VERIFYING_SSL,
         )
         LOG.debug("HTTP response %s: %s", resp.status_code, _truncate_end(resp.content))
 

{mapillary_tools-0.11.1 → mapillary_tools-0.11.2}/mapillary_tools/video_data_extraction/extract_video_data.py

@@ -104,10 +104,12 @@ class VideoDataExtractor:
         else:
             return ErrorMetadata(
                 filename=file,
-                error=ex
-                if ex
-                else exceptions.MapillaryVideoGPSNotFoundError(
-                    "No GPS data found from the video"
+                error=(
+                    ex
+                    if ex
+                    else exceptions.MapillaryVideoGPSNotFoundError(
+                        "No GPS data found from the video"
+                    )
                 ),
                 filetype=FileType.VIDEO,
             )

{mapillary_tools-0.11.1 → mapillary_tools-0.11.2}/mapillary_tools.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
-Name: mapillary-tools
-Version: 0.11.1
+Name: mapillary_tools
+Version: 0.11.2
 Summary: Mapillary Image/Video Import Pipeline
 Home-page: https://github.com/mapillary/mapillary_tools
 Author: Mapillary
@@ -13,7 +13,7 @@ Requires-Dist: construct<3.0.0,>=2.10.0
 Requires-Dist: exifread==2.3.2
 Requires-Dist: piexif==1.1.3
 Requires-Dist: gpxpy<1.6.0,>=1.5.0
-Requires-Dist: pynmea2==1.12.0
+Requires-Dist: pynmea2<2.0.0,>=1.12.0
 Requires-Dist: requests<3.0.0,>=2.20.0
 Requires-Dist: tqdm<5.0,>=4.0
 Requires-Dist: typing_extensions
@@ -130,7 +130,7 @@ A command line program such as Termux is required. Installation can be done with
 commands will install Python 3, pip3, git, and all required libraries for mapillary_tools on Termux:
 
 ```sh
-pkg install python git build-essential libgeos openssl libjpeg-turbo
+pkg install python git build-essential libgeos openssl libjpeg-turbo libexpat libexpat-static
 pip install --upgrade pip wheel
 pip install --upgrade mapillary_tools
 ```

{mapillary_tools-0.11.1 → mapillary_tools-0.11.2}/mapillary_tools.egg-info/requires.txt

@@ -3,7 +3,7 @@ construct<3.0.0,>=2.10.0
 exifread==2.3.2
 piexif==1.1.3
 gpxpy<1.6.0,>=1.5.0
-pynmea2==1.12.0
+pynmea2<2.0.0,>=1.12.0
 requests<3.0.0,>=2.20.0
 tqdm<5.0,>=4.0
 typing_extensions

{mapillary_tools-0.11.1 → mapillary_tools-0.11.2}/requirements.txt

@@ -3,7 +3,7 @@ construct>=2.10.0,<3.0.0
 exifread==2.3.2
 piexif==1.1.3
 gpxpy>=1.5.0,<1.6.0
-pynmea2==1.12.0
+pynmea2>=1.12.0,<2.0.0
 requests>=2.20.0,<3.0.0
 tqdm>=4.0,<5.0
 typing_extensions

mapillary_tools-0.11.1/mapillary_tools/__init__.py

@@ -1 +0,0 @@
-VERSION = "0.11.1"