mapillary-tools 0.13.3__py3-none-any.whl → 0.14.0__py3-none-any.whl

mapillary_tools/authenticate.py

@@ -1,102 +1,364 @@
+from __future__ import annotations
+
 import getpass
 import logging
+import re
+import sys
 import typing as T
 
 import jsonschema
+
 import requests
 
-from . import api_v4, config, types
+from . import api_v4, config, constants, exceptions
 
 
 LOG = logging.getLogger(__name__)
 
 
 def authenticate(
-    user_name: T.Optional[str] = None,
-    user_email: T.Optional[str] = None,
-    user_password: T.Optional[str] = None,
-    jwt: T.Optional[str] = None,
+    user_name: str | None = None,
+    user_email: str | None = None,
+    user_password: str | None = None,
+    jwt: str | None = None,
+    delete: bool = False,
 ):
-    if user_name:
-        user_name = user_name.strip()
+    """
+    Prompt for authentication information and save it to the config file
+    """
+
+    # We still have to accept --user_name for the back compatibility
+    profile_name = user_name
 
-    while not user_name:
-        user_name = input(
-            "Enter the Mapillary username you would like to (re)authenticate: "
+    all_user_items = config.list_all_users()
+    if all_user_items:
+        _list_all_profiles(all_user_items)
+    else:
+        _welcome()
+
+    # Make sure profile name either validated or existed
+    if profile_name is not None:
+        profile_name = profile_name.strip()
+    else:
+        if not _prompt_enabled():
+            raise exceptions.MapillaryBadParameterError(
+                "Profile name is required, please specify one with --user_name"
+            )
+        profile_name = _prompt_choose_profile_name(
+            list(all_user_items.keys()), must_exist=delete
         )
-        user_name = user_name.strip()
 
-    if jwt:
-        user_items: types.UserItem = {
-            "user_upload_token": jwt,
-        }
-    elif user_email and user_password:
-        resp = api_v4.get_upload_token(user_email, user_password)
-        data = resp.json()
-        user_items = {
-            "MAPSettingsUserKey": data["user_id"],
-            "user_upload_token": data["access_token"],
-        }
+    assert profile_name is not None, "profile_name should be set"
+
+    if delete:
+        config.remove_config(profile_name)
+        LOG.info('Profile "%s" deleted', profile_name)
+    else:
+        if profile_name in all_user_items:
+            LOG.warning(
+                'The profile "%s" already exists and will be overridden',
+                profile_name,
+            )
+        else:
+            LOG.info('Creating new profile: "%s"', profile_name)
+
+        if jwt:
+            user_items: config.UserItem = {"user_upload_token": jwt}
+            user_items = _verify_user_auth(_validate_profile(user_items))
+        else:
+            user_items = _prompt_login(
+                user_email=user_email, user_password=user_password
+            )
+            _validate_profile(user_items)
+
+        # Update the config with the new user items
+        config.update_config(profile_name, user_items)
+
+        # TODO: print more user information
+        if profile_name in all_user_items:
+            LOG.info(
+                'Profile "%s" updated: %s', profile_name, api_v4._sanitize(user_items)
+            )
+        else:
+            LOG.info(
+                'Profile "%s" created: %s', profile_name, api_v4._sanitize(user_items)
+            )
+
+
+def fetch_user_items(
+    user_name: str | None = None, organization_key: str | None = None
+) -> config.UserItem:
+    """
+    Read user information from the config file,
+    or prompt the user to authenticate if the specified profile does not exist
+    """
+
+    # we still have to accept --user_name for the back compatibility
+    profile_name = user_name
+
+    all_user_items = config.list_all_users()
+    if not all_user_items:
+        authenticate(user_name=profile_name)
+
+    # Fetch user information only here
+    all_user_items = config.list_all_users()
+    assert len(all_user_items) >= 1, "should have at least 1 profile"
+    if profile_name is None:
+        if len(all_user_items) > 1:
+            if not _prompt_enabled():
+                raise exceptions.MapillaryBadParameterError(
+                    "Multiple user profiles found, please choose one with --user_name"
+                )
+            _list_all_profiles(all_user_items)
+            profile_name = _prompt_choose_profile_name(
+                list(all_user_items.keys()), must_exist=True
+            )
+            user_items = all_user_items[profile_name]
+        else:
+            profile_name, user_items = list(all_user_items.items())[0]
     else:
-        user_items = prompt_user_for_user_items(user_name)
+        if profile_name in all_user_items:
+            user_items = all_user_items[profile_name]
+        else:
+            _list_all_profiles(all_user_items)
+            raise exceptions.MapillaryBadParameterError(
+                f'Profile "{profile_name}" not found'
+            )
 
-    config.update_config(user_name, user_items)
+    assert profile_name is not None, "profile_name should be set"
 
+    user_items = _verify_user_auth(_validate_profile(user_items))
 
-def prompt_user_for_user_items(user_name: str) -> types.UserItem:
-    print(f"Sign in for user {user_name}")
-    user_email = input("Enter your Mapillary user email: ")
-    user_password = getpass.getpass("Enter Mapillary user password: ")
+    LOG.info(
+        f'Uploading to profile "{profile_name}": {user_items.get("MAPSettingsUsername")} (ID: {user_items.get("MAPSettingsUserKey")})'
+    )
 
+    if organization_key is not None:
+        resp = api_v4.fetch_organization(
+            user_items["user_upload_token"], organization_key
+        )
+        data = api_v4.jsonify_response(resp)
+        LOG.info(
+            f"Uploading to organization: {data.get('name')} (ID: {data.get('id')})"
+        )
+        user_items["MAPOrganizationKey"] = data.get("id")
+
+    return user_items
+
+
+def _echo(*args, **kwargs):
+    print(*args, **kwargs, file=sys.stderr)
+
+
+def _prompt(message: str) -> str:
+    """Display prompt on stderr and get input from stdin"""
+    print(message, end="", file=sys.stderr, flush=True)
+    return input()
+
+
+def _validate_profile(user_items: config.UserItem) -> config.UserItem:
     try:
-        resp = api_v4.get_upload_token(user_email, user_password)
+        jsonschema.validate(user_items, config.UserItemSchema)
+    except jsonschema.ValidationError as ex:
+        raise exceptions.MapillaryBadParameterError(
+            f"Invalid profile format: {ex.message}"
+        )
+    return user_items
+
+
+def _verify_user_auth(user_items: config.UserItem) -> config.UserItem:
+    """
+    Verify that the user access token is valid
+    """
+    if constants._AUTH_VERIFICATION_DISABLED:
+        return user_items
+
+    try:
+        resp = api_v4.fetch_user_or_me(
+            user_access_token=user_items["user_upload_token"]
+        )
     except requests.HTTPError as ex:
-        if (
-            isinstance(ex, requests.HTTPError)
-            and isinstance(ex.response, requests.Response)
-            and 400 <= ex.response.status_code < 500
-        ):
-            r = ex.response.json()
-            subcode = r.get("error", {}).get("error_subcode")
-            if subcode in [1348028, 1348092, 3404005, 1348131]:
-                title = r.get("error", {}).get("error_user_title")
-                message = r.get("error", {}).get("error_user_msg")
-                LOG.error("%s: %s", title, message)
-                return prompt_user_for_user_items(user_name)
-            else:
-                raise ex
+        if api_v4.is_auth_error(ex.response):
+            message = api_v4.extract_auth_error_message(ex.response)
+            raise exceptions.MapillaryUploadUnauthorizedError(message)
         else:
             raise ex
 
-    data = resp.json()
-    upload_token = T.cast(str, data.get("access_token"))
-    user_key = T.cast(str, data.get("user_id"))
-    if not isinstance(upload_token, str) or not isinstance(user_key, (str, int)):
-        raise RuntimeError(
-            f"Error extracting user_key or token from the login response: {data}"
+    data = api_v4.jsonify_response(resp)
+
+    return {
+        **user_items,
+        "MAPSettingsUsername": data.get("username"),
+        "MAPSettingsUserKey": data.get("id"),
+    }
+
+
+def _validate_profile_name(profile_name: str):
+    if not (2 <= len(profile_name) <= 32):
+        raise exceptions.MapillaryBadParameterError(
+            "Profile name must be between 2 and 32 characters long"
         )
 
-    if isinstance(user_key, int):
-        user_key = str(user_key)
+    pattern = re.compile(r"^[a-zA-Z]+[a-zA-Z0-9_-]*$")
+    if not bool(pattern.match(profile_name)):
+        raise exceptions.MapillaryBadParameterError(
+            "Invalid profile name. Use only letters, numbers, hyphens and underscores"
+        )
 
-    return {
-        "MAPSettingsUserKey": user_key,
-        "user_upload_token": upload_token,
+
+def _list_all_profiles(profiles: dict[str, config.UserItem]) -> None:
+    _echo("Existing Mapillary profiles:")
+
+    # Header
+    _echo(f"{'':>5}  {'Profile name':<32} {'User ID':>16} {'Username':>32}")
+
+    # List all profiles
+    for idx, name in enumerate(profiles, 1):
+        items = profiles[name]
+        user_id = items.get("MAPSettingsUserKey", "N/A")
+        username = items.get("MAPSettingsUsername", "N/A")
+        _echo(f"{idx:>5}. {name:<32} {user_id:>16} {username:>32}")
+
+
+def _is_interactive():
+    # Check if stdout is connected to a terminal
+    stdout_interactive = sys.stdout.isatty() if hasattr(sys.stdout, "isatty") else False
+
+    # Optionally, also check stdin and stderr
+    stdin_interactive = sys.stdin.isatty() if hasattr(sys.stdin, "isatty") else False
+    stderr_interactive = sys.stderr.isatty() if hasattr(sys.stderr, "isatty") else False
+
+    # Return True if any stream is interactive
+    return stdout_interactive or stdin_interactive or stderr_interactive
+
+
+def _prompt_enabled() -> bool:
+    if constants.PROMPT_DISABLED:
+        return False
+
+    if not _is_interactive():
+        return False
+
+    return True
+
+
+def _is_login_retryable(ex: requests.HTTPError) -> bool:
+    if 400 <= ex.response.status_code < 500:
+        r = ex.response.json()
+        subcode = r.get("error", {}).get("error_subcode")
+        if subcode in [1348028, 1348092, 3404005, 1348131]:
+            title = r.get("error", {}).get("error_user_title")
+            message = r.get("error", {}).get("error_user_msg")
+            LOG.error("%s: %s", title, message)
+            return True
+    return False
+
+
+def _prompt_login(
+    user_email: str | None = None,
+    user_password: str | None = None,
+) -> config.UserItem:
+    _enabled = _prompt_enabled()
+
+    if user_email is None:
+        if not _enabled:
+            raise exceptions.MapillaryBadParameterError("user_email is required")
+        while not user_email:
+            user_email = _prompt("Enter Mapillary user email: ").strip()
+    else:
+        user_email = user_email.strip()
+
+    if user_password is None:
+        if not _enabled:
+            raise exceptions.MapillaryBadParameterError("user_password is required")
+        while True:
+            user_password = getpass.getpass("Enter Mapillary user password: ")
+            if user_password:
+                break
+
+    try:
+        resp = api_v4.get_upload_token(user_email, user_password)
+    except requests.HTTPError as ex:
+        if not _enabled:
+            raise ex
+
+        if _is_login_retryable(ex):
+            return _prompt_login()
+
+        raise ex
+
+    data = api_v4.jsonify_response(resp)
+
+    user_items: config.UserItem = {
+        "user_upload_token": str(data["access_token"]),
+        "MAPSettingsUserKey": str(data["user_id"]),
     }
 
+    return user_items
+
+
+def _prompt_choose_profile_name(
+    existing_profile_names: T.Sequence[str], must_exist: bool = False
+) -> str:
+    assert _prompt_enabled(), "should not get here if prompting is disabled"
+
+    existed = set(existing_profile_names)
+
+    while True:
+        if must_exist:
+            prompt = "Enter an existing profile: "
+        else:
+            prompt = "Enter an existing profile or create a new one: "
+
+        profile_name = _prompt(prompt).strip()
 
-def authenticate_user(user_name: str) -> types.UserItem:
-    user_items = config.load_user(user_name)
-    if user_items is not None:
+        if not profile_name:
+            continue
+
+        # Exit if it's found
+        if profile_name in existed:
+            break
+
+        # Try to find by index
         try:
-            jsonschema.validate(user_items, types.UserItemSchema)
-        except jsonschema.ValidationError:
+            profile_name = existing_profile_names[int(profile_name) - 1]
+        except (ValueError, IndexError):
             pass
         else:
-            return user_items
+            # Exit if it's found
+            break
 
-    user_items = prompt_user_for_user_items(user_name)
-    jsonschema.validate(user_items, types.UserItemSchema)
-    config.update_config(user_name, user_items)
+        assert profile_name not in existed, (
+            f"Profile {profile_name} must not exist here"
+        )
 
-    return user_items
+        if must_exist:
+            LOG.error('Profile "%s" not found', profile_name)
+        else:
+            try:
+                _validate_profile_name(profile_name)
+            except exceptions.MapillaryBadParameterError as ex:
+                LOG.error("Error validating profile name: %s", ex)
+                profile_name = ""
+            else:
+                break
+
+    if must_exist:
+        assert profile_name in existed, f"Profile {profile_name} must exist"
+
+    return profile_name
+
+
+def _welcome():
+    _echo(
+        """
+================================================================================
+                             Welcome to Mapillary!
+================================================================================
+  If you haven't registered yet, please visit https://www.mapillary.com/signup
+  to create your account first.
+
+  Once registered, proceed here to sign in.
+================================================================================
+    """
+    )

mapillary_tools/blackvue_parser.py

@@ -0,0 +1,195 @@
+from __future__ import annotations
+
+import dataclasses
+
+import json
+import logging
+import re
+import typing as T
+
+import pynmea2
+
+from . import geo
+from .mp4 import simple_mp4_parser as sparser
+
+
+LOG = logging.getLogger(__name__)
+NMEA_LINE_REGEX = re.compile(
+    rb"""
+    ^\s*
+    \[(\d+)\] # Timestamp
+    \s*
+    (\$\w{5}.*) # NMEA message
+    \s*
+    (\[\d+\])? # Strange timestamp
+    \s*$
+    """,
+    re.X,
+)
+
+
+@dataclasses.dataclass
+class BlackVueInfo:
+    # None and [] are equivalent here. Use None as default because:
+    # ValueError: mutable default <class 'list'> for field gps is not allowed: use default_factory
+    gps: list[geo.Point] | None = None
+    make: str = "BlackVue"
+    model: str = ""
+
+
+def extract_blackvue_info(fp: T.BinaryIO) -> BlackVueInfo | None:
+    try:
+        gps_data = sparser.parse_mp4_data_first(fp, [b"free", b"gps "])
+    except sparser.ParsingError:
+        gps_data = None
+
+    if gps_data is None:
+        return None
+
+    points = _parse_gps_box(gps_data)
+    points.sort(key=lambda p: p.time)
+
+    if points:
+        first_point_time = points[0].time
+        for p in points:
+            p.time = (p.time - first_point_time) / 1000
+
+    # Camera model
+    try:
+        cprt_bytes = sparser.parse_mp4_data_first(fp, [b"free", b"cprt"])
+    except sparser.ParsingError:
+        cprt_bytes = None
+        model = ""
+
+    if cprt_bytes is None:
+        model = ""
+    else:
+        model = _extract_camera_model_from_cprt(cprt_bytes)
+
+    return BlackVueInfo(model=model, gps=points)
+
+
+def extract_camera_model(fp: T.BinaryIO) -> str:
+    try:
+        cprt_bytes = sparser.parse_mp4_data_first(fp, [b"free", b"cprt"])
+    except sparser.ParsingError:
+        return ""
+
+    if cprt_bytes is None:
+        return ""
+
+    return _extract_camera_model_from_cprt(cprt_bytes)
+
+
+def _extract_camera_model_from_cprt(cprt_bytes: bytes) -> str:
+    """
+    >>> _extract_camera_model_from_cprt(b' {"model":"DR900X Plus","ver":0.918,"lang":"English","direct":1,"psn":"","temp":34,"GPS":1}')
+    'DR900X Plus'
+    >>> _extract_camera_model_from_cprt(b' Pittasoft Co., Ltd.;DR900S-1CH;1.008;English;1;D90SS1HAE00661;T69;')
+    'DR900S-1CH'
+    """
+    cprt_bytes = cprt_bytes.strip().strip(b"\x00")
+
+    try:
+        cprt_str = cprt_bytes.decode("utf8")
+    except UnicodeDecodeError:
+        return ""
+
+    try:
+        cprt_json = json.loads(cprt_str)
+    except json.JSONDecodeError:
+        cprt_json = None
+
+    if cprt_json is not None:
+        return str(cprt_json.get("model", "")).strip()
+
+    fields = cprt_str.split(";")
+    if 2 <= len(fields):
+        model = fields[1]
+        if model:
+            return model.strip()
+        else:
+            return ""
+    else:
+        return ""
+
+
+def _parse_gps_box(gps_data: bytes) -> list[geo.Point]:
+    """
+    >>> list(_parse_gps_box(b"[1623057074211]$GPGGA,202530.00,5109.0262,N,11401.8407,W,5,40,0.5,1097.36,M,-17.00,M,18,TSTR*61"))
+    [Point(time=1623057074211, lat=51.150436666666664, lon=-114.03067833333333, alt=1097.36, angle=None)]
+
+    >>> list(_parse_gps_box(b"[1629874404069]$GNGGA,175322.00,3244.53126,N,11710.97811,W,1,12,0.84,17.4,M,-34.0,M,,*45"))
+    [Point(time=1629874404069, lat=32.742187666666666, lon=-117.1829685, alt=17.4, angle=None)]
+
+    >>> list(_parse_gps_box(b"[1629874404069]$GNGLL,4404.14012,N,12118.85993,W,001037.00,A,A*67"))
+    [Point(time=1629874404069, lat=44.069002, lon=-121.31433216666667, alt=None, angle=None)]
+
+    >>> list(_parse_gps_box(b"[1629874404069]$GNRMC,001031.00,A,4404.13993,N,12118.86023,W,0.146,,100117,,,A*7B"))
+    [Point(time=1629874404069, lat=44.06899883333333, lon=-121.31433716666666, alt=None, angle=None)]
+
+    >>> list(_parse_gps_box(b"[1623057074211]$GPVTG,,T,,M,0.078,N,0.144,K,D*28[1623057075215]"))
+    []
+    """
+    points_by_sentence_type: dict[str, list[geo.Point]] = {}
+
+    for line_bytes in gps_data.splitlines():
+        match = NMEA_LINE_REGEX.match(line_bytes)
+        if match is None:
+            continue
+        nmea_line_bytes = match.group(2)
+
+        if not nmea_line_bytes:
+            continue
+
+        try:
+            nmea_line = nmea_line_bytes.decode("utf8")
+        except UnicodeDecodeError:
+            continue
+
+        if not nmea_line:
+            continue
+
+        try:
+            message = pynmea2.parse(nmea_line)
+        except pynmea2.nmea.ParseError:
+            continue
+
+        epoch_ms = int(match.group(1))
+
+        # https://tavotech.com/gps-nmea-sentence-structure/
+        if message.sentence_type in ["GGA"]:
+            if not message.is_valid:
+                continue
+            point = geo.Point(
+                time=epoch_ms,
+                lat=message.latitude,
+                lon=message.longitude,
+                alt=message.altitude,
+                angle=None,
+            )
+            points_by_sentence_type.setdefault(message.sentence_type, []).append(point)
+
+        elif message.sentence_type in ["RMC", "GLL"]:
+            if not message.is_valid:
+                continue
+            point = geo.Point(
+                time=epoch_ms,
+                lat=message.latitude,
+                lon=message.longitude,
+                alt=None,
+                angle=None,
+            )
+            points_by_sentence_type.setdefault(message.sentence_type, []).append(point)
+
+    # This is the extraction order in exiftool
+    if "RMC" in points_by_sentence_type:
+        return points_by_sentence_type["RMC"]
+
+    if "GGA" in points_by_sentence_type:
+        return points_by_sentence_type["GGA"]
+
+    if "GLL" in points_by_sentence_type:
+        return points_by_sentence_type["GLL"]
+
+    return []