mapillary-downloader 0.1.0__tar.gz

mapillary_downloader-0.1.0/LICENSE.md

@@ -0,0 +1,7 @@
+# WTFPL + Warranty
+
+Licensed under the WTFPL with one additional clause:
+
+1. Don't blame me.
+
+Do whatever the fuck you want, just don't blame me.

mapillary_downloader-0.1.0/PKG-INFO

@@ -0,0 +1,54 @@
+Metadata-Version: 2.4
+Name: mapillary_downloader
+Version: 0.1.0
+Summary: Download your Mapillary data before it's gone
+Author-email: Gareth Davidson <gaz@bitplane.net>
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE.md
+Requires-Dist: requests>=2.31.0
+Requires-Dist: piexif>=1.1.3
+Requires-Dist: pre-commit ; extra == "dev"
+Requires-Dist: pytest ; extra == "dev"
+Requires-Dist: coverage ; extra == "dev"
+Requires-Dist: pytest-cov ; extra == "dev"
+Requires-Dist: build ; extra == "dev"
+Requires-Dist: twine ; extra == "dev"
+Requires-Dist: ruff ; extra == "dev"
+Requires-Dist: mkdocs ; extra == "dev"
+Requires-Dist: mkdocs-material ; extra == "dev"
+Requires-Dist: pydoc-markdown ; extra == "dev"
+Requires-Dist: Pillow ; extra == "dev"
+Provides-Extra: dev
+
+# Mapillary Downloader
+
+Download your Mapillary data before it's gone.
+
+## Installation
+
+```bash
+make install
+```
+
+## Usage
+
+First, get your Mapillary API access token from https://www.mapillary.com/dashboard/developers
+
+```bash
+source .venv/bin/activate
+python -m mapillary_downloader --token YOUR_TOKEN --username YOUR_USERNAME --output ./downloads
+```
+
+## Development
+
+```bash
+make dev      # Setup dev environment
+make test     # Run tests
+make coverage # Run tests with coverage
+```
+
+## License
+
+WTFPL + Warranty (Don't blame me)
+

mapillary_downloader-0.1.0/README.md

@@ -0,0 +1,30 @@
+# Mapillary Downloader
+
+Download your Mapillary data before it's gone.
+
+## Installation
+
+```bash
+make install
+```
+
+## Usage
+
+First, get your Mapillary API access token from https://www.mapillary.com/dashboard/developers
+
+```bash
+source .venv/bin/activate
+python -m mapillary_downloader --token YOUR_TOKEN --username YOUR_USERNAME --output ./downloads
+```
+
+## Development
+
+```bash
+make dev      # Setup dev environment
+make test     # Run tests
+make coverage # Run tests with coverage
+```
+
+## License
+
+WTFPL + Warranty (Don't blame me)

mapillary_downloader-0.1.0/pyproject.toml

@@ -0,0 +1,40 @@
+[project]
+name = "mapillary_downloader"
+description = "Download your Mapillary data before it's gone"
+version = "0.1.0"
+authors = [
+    { name = "Gareth Davidson", email = "gaz@bitplane.net" }
+]
+readme = "README.md"
+requires-python = ">=3.10"
+
+dependencies = [
+    "requests>=2.31.0",
+    "piexif>=1.1.3",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pre-commit",
+    "pytest",
+    "coverage",
+    "pytest-cov",
+    "build",
+    "twine",
+    "ruff",
+    "mkdocs",
+    "mkdocs-material",
+    "pydoc-markdown",
+    "Pillow",
+]
+
+[build-system]
+build-backend = "flit_core.buildapi"
+requires = ["flit_core >=3.2,<4"]
+
+[tool.ruff]
+line-length = 120
+target-version = "py310"
+
+[tool.ruff.format]
+docstring-code-format = true

mapillary_downloader-0.1.0/src/mapillary_downloader/__init__.py

@@ -0,0 +1,5 @@
+"""Mapillary data downloader."""
+
+from importlib.metadata import version
+
+__version__ = version("mapillary_downloader")

mapillary_downloader-0.1.0/src/mapillary_downloader/__main__.py

@@ -0,0 +1,65 @@
+"""CLI entry point."""
+
+import argparse
+import sys
+from mapillary_downloader.client import MapillaryClient
+from mapillary_downloader.downloader import MapillaryDownloader
+
+
+def main():
+    """Main CLI entry point."""
+    parser = argparse.ArgumentParser(
+        description="Download your Mapillary data before it's gone"
+    )
+    parser.add_argument(
+        "--token",
+        required=True,
+        help="Mapillary API access token"
+    )
+    parser.add_argument(
+        "--username",
+        required=True,
+        help="Your Mapillary username"
+    )
+    parser.add_argument(
+        "--output",
+        default="./mapillary_data",
+        help="Output directory (default: ./mapillary_data)"
+    )
+    parser.add_argument(
+        "--quality",
+        choices=["256", "1024", "2048", "original"],
+        default="original",
+        help="Image quality to download (default: original)"
+    )
+    parser.add_argument(
+        "--bbox",
+        help="Bounding box: west,south,east,north"
+    )
+
+    args = parser.parse_args()
+
+    bbox = None
+    if args.bbox:
+        try:
+            bbox = [float(x) for x in args.bbox.split(",")]
+            if len(bbox) != 4:
+                raise ValueError
+        except ValueError:
+            print("Error: bbox must be four comma-separated numbers")
+            sys.exit(1)
+
+    try:
+        client = MapillaryClient(args.token)
+        downloader = MapillaryDownloader(client, args.output)
+        downloader.download_user_data(args.username, args.quality, bbox)
+    except KeyboardInterrupt:
+        print("\nInterrupted by user")
+        sys.exit(1)
+    except Exception as e:
+        print(f"Error: {e}")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

mapillary_downloader-0.1.0/src/mapillary_downloader/client.py

@@ -0,0 +1,107 @@
+"""Mapillary API client."""
+
+import time
+import requests
+
+
+class MapillaryClient:
+    """Client for interacting with Mapillary API v4."""
+
+    def __init__(self, access_token):
+        """Initialize the client with an access token.
+
+        Args:
+            access_token: Mapillary API access token
+        """
+        self.access_token = access_token
+        self.base_url = "https://graph.mapillary.com"
+        self.session = requests.Session()
+        self.session.headers.update({"Authorization": f"OAuth {access_token}"})
+
+    def get_user_images(self, username, bbox=None, limit=2000):
+        """Get images uploaded by a specific user.
+
+        Args:
+            username: Mapillary username
+            bbox: Optional bounding box [west, south, east, north]
+            limit: Number of results per page (max 2000)
+
+        Yields:
+            Image data dictionaries
+        """
+        params = {
+            "creator_username": username,
+            "limit": limit,
+            "fields": ",".join(
+                [
+                    "id",
+                    "captured_at",
+                    "compass_angle",
+                    "computed_compass_angle",
+                    "geometry",
+                    "computed_geometry",
+                    "altitude",
+                    "computed_altitude",
+                    "is_pano",
+                    "sequence",
+                    "camera_type",
+                    "camera_parameters",
+                    "make",
+                    "model",
+                    "exif_orientation",
+                    "computed_rotation",
+                    "height",
+                    "width",
+                    "thumb_256_url",
+                    "thumb_1024_url",
+                    "thumb_2048_url",
+                    "thumb_original_url",
+                ]
+            ),
+        }
+
+        if bbox:
+            params["bbox"] = ",".join(map(str, bbox))
+
+        url = f"{self.base_url}/images"
+
+        while url:
+            response = self.session.get(url, params=params)
+            response.raise_for_status()
+
+            data = response.json()
+
+            for image in data.get("data", []):
+                yield image
+
+            # Get next page URL
+            url = data.get("paging", {}).get("next")
+            params = None  # Don't send params on subsequent requests, URL has them
+
+            # Rate limiting: 10,000 requests/minute for search API
+            time.sleep(0.01)
+
+    def download_image(self, image_url, output_path):
+        """Download an image from a URL.
+
+        Args:
+            image_url: URL of the image to download
+            output_path: Path to save the image
+
+        Returns:
+            Number of bytes downloaded if successful, 0 otherwise
+        """
+        try:
+            response = self.session.get(image_url, stream=True)
+            response.raise_for_status()
+
+            total_bytes = 0
+            with open(output_path, "wb") as f:
+                for chunk in response.iter_content(chunk_size=8192):
+                    f.write(chunk)
+                    total_bytes += len(chunk)
+
+            return total_bytes
+        except Exception as e:
+            print(f"Error downloading {image_url}: {e}")
+            return 0

mapillary_downloader-0.1.0/src/mapillary_downloader/downloader.py

@@ -0,0 +1,119 @@
+"""Main downloader logic."""
+
+import json
+import os
+from pathlib import Path
+from mapillary_downloader.exif_writer import write_exif_to_image
+
+
+def format_bytes(bytes_count):
+    """Format bytes as human-readable string."""
+    if bytes_count < 1024:
+        return f"{bytes_count} B"
+    if bytes_count < 1024 * 1024:
+        return f"{bytes_count / 1024:.3f} KB"
+    if bytes_count < 1024 * 1024 * 1024:
+        return f"{bytes_count / (1024 * 1024):.3f} MB"
+    return f"{bytes_count / (1024 * 1024 * 1024):.3f} GB"
+
+
+class MapillaryDownloader:
+    """Handles downloading Mapillary data for a user."""
+
+    def __init__(self, client, output_dir):
+        """Initialize the downloader.
+
+        Args:
+            client: MapillaryClient instance
+            output_dir: Directory to save downloads
+        """
+        self.client = client
+        self.output_dir = Path(output_dir)
+        self.output_dir.mkdir(parents=True, exist_ok=True)
+
+        self.metadata_file = self.output_dir / "metadata.jsonl"
+        self.progress_file = self.output_dir / "progress.json"
+        self.downloaded = self._load_progress()
+
+    def _load_progress(self):
+        """Load previously downloaded image IDs."""
+        if self.progress_file.exists():
+            with open(self.progress_file) as f:
+                return set(json.load(f).get("downloaded", []))
+        return set()
+
+    def _save_progress(self):
+        """Save progress to disk atomically."""
+        temp_file = self.progress_file.with_suffix(".json.tmp")
+        with open(temp_file, "w") as f:
+            json.dump({"downloaded": list(self.downloaded)}, f)
+            f.flush()
+            os.fsync(f.fileno())
+        temp_file.replace(self.progress_file)
+
+    def download_user_data(self, username, quality="original", bbox=None):
+        """Download all images for a user.
+
+        Args:
+            username: Mapillary username
+            quality: Image quality to download (256, 1024, 2048, original)
+            bbox: Optional bounding box [west, south, east, north]
+        """
+        quality_field = f"thumb_{quality}_url"
+
+        print(f"Downloading images for user: {username}")
+        print(f"Output directory: {self.output_dir}")
+        print(f"Quality: {quality}")
+
+        processed = 0
+        downloaded_count = 0
+        skipped = 0
+        total_bytes = 0
+
+        with open(self.metadata_file, "a") as meta_f:
+            for image in self.client.get_user_images(username, bbox=bbox):
+                image_id = image["id"]
+                processed += 1
+
+                if image_id in self.downloaded:
+                    skipped += 1
+                    continue
+
+                # Save metadata
+                meta_f.write(json.dumps(image) + "\n")
+                meta_f.flush()
+
+                # Download image
+                image_url = image.get(quality_field)
+                if not image_url:
+                    print(f"No {quality} URL for image {image_id}")
+                    continue
+
+                # Use sequence ID for organization
+                sequence_id = image.get("sequence")
+                if sequence_id:
+                    img_dir = self.output_dir / sequence_id
+                    img_dir.mkdir(exist_ok=True)
+                else:
+                    img_dir = self.output_dir
+
+                output_path = img_dir / f"{image_id}.jpg"
+
+                bytes_downloaded = self.client.download_image(image_url, output_path)
+                if bytes_downloaded:
+                    # Write EXIF metadata to the downloaded image
+                    write_exif_to_image(output_path, image)
+
+                    self.downloaded.add(image_id)
+                    downloaded_count += 1
+                    total_bytes += bytes_downloaded
+                    print(f"Processed: {processed}, Downloaded: {downloaded_count} ({format_bytes(total_bytes)})")
+
+                    # Save progress every 10 images
+                    if downloaded_count % 10 == 0:
+                        self._save_progress()
+
+        self._save_progress()
+        print(
+            f"\nComplete! Processed {processed} images, downloaded {downloaded_count} ({format_bytes(total_bytes)}), skipped {skipped}"
+        )

mapillary_downloader-0.1.0/src/mapillary_downloader/exif_writer.py

@@ -0,0 +1,122 @@
+"""EXIF metadata writer for Mapillary images."""
+
+import piexif
+from datetime import datetime
+
+
+def decimal_to_dms(decimal):
+    """Convert decimal degrees to degrees, minutes, seconds format for EXIF.
+
+    Args:
+        decimal: Decimal degrees (can be negative)
+
+    Returns:
+        Tuple of ((degrees, 1), (minutes, 1), (seconds, 100)) as rational numbers
+    """
+    decimal = abs(decimal)
+    degrees = int(decimal)
+    minutes_float = (decimal - degrees) * 60
+    minutes = int(minutes_float)
+    seconds = (minutes_float - minutes) * 60
+    seconds_rational = (int(seconds * 100), 100)
+
+    return ((degrees, 1), (minutes, 1), seconds_rational)
+
+
+def timestamp_to_exif_datetime(timestamp):
+    """Convert Unix timestamp to EXIF datetime string.
+
+    Args:
+        timestamp: Unix timestamp in milliseconds
+
+    Returns:
+        String in format "YYYY:MM:DD HH:MM:SS"
+    """
+    dt = datetime.fromtimestamp(timestamp / 1000.0)
+    return dt.strftime("%Y:%m:%d %H:%M:%S")
+
+
+def write_exif_to_image(image_path, metadata):
+    """Write EXIF metadata from Mapillary API to downloaded image.
+
+    Args:
+        image_path: Path to the downloaded image file
+        metadata: Dictionary of metadata from Mapillary API
+
+    Returns:
+        True if successful, False otherwise
+    """
+    try:
+        # Load existing EXIF data if any
+        try:
+            exif_dict = piexif.load(str(image_path))
+        except Exception:
+            # No existing EXIF data, start fresh
+            exif_dict = {"0th": {}, "Exif": {}, "GPS": {}, "1st": {}, "thumbnail": None}
+
+        # Ensure all IFDs exist
+        for ifd in ["0th", "Exif", "GPS", "1st"]:
+            if ifd not in exif_dict:
+                exif_dict[ifd] = {}
+
+        # Basic image info (0th IFD)
+        if "make" in metadata and metadata["make"]:
+            exif_dict["0th"][piexif.ImageIFD.Make] = metadata["make"].encode("utf-8")
+
+        if "model" in metadata and metadata["model"]:
+            exif_dict["0th"][piexif.ImageIFD.Model] = metadata["model"].encode("utf-8")
+
+        if "exif_orientation" in metadata and metadata["exif_orientation"]:
+            exif_dict["0th"][piexif.ImageIFD.Orientation] = metadata["exif_orientation"]
+
+        if "width" in metadata and metadata["width"]:
+            exif_dict["0th"][piexif.ImageIFD.ImageWidth] = metadata["width"]
+
+        if "height" in metadata and metadata["height"]:
+            exif_dict["0th"][piexif.ImageIFD.ImageLength] = metadata["height"]
+
+        # Datetime tags
+        if "captured_at" in metadata and metadata["captured_at"]:
+            datetime_str = timestamp_to_exif_datetime(metadata["captured_at"])
+            datetime_bytes = datetime_str.encode("utf-8")
+            exif_dict["0th"][piexif.ImageIFD.DateTime] = datetime_bytes
+            exif_dict["Exif"][piexif.ExifIFD.DateTimeOriginal] = datetime_bytes
+            exif_dict["Exif"][piexif.ExifIFD.DateTimeDigitized] = datetime_bytes
+
+        # GPS data - prefer computed_geometry over geometry
+        geometry = metadata.get("computed_geometry") or metadata.get("geometry")
+        if geometry and "coordinates" in geometry:
+            lon, lat = geometry["coordinates"]
+
+            # GPS Latitude
+            exif_dict["GPS"][piexif.GPSIFD.GPSLatitude] = decimal_to_dms(lat)
+            exif_dict["GPS"][piexif.GPSIFD.GPSLatitudeRef] = b"N" if lat >= 0 else b"S"
+
+            # GPS Longitude
+            exif_dict["GPS"][piexif.GPSIFD.GPSLongitude] = decimal_to_dms(lon)
+            exif_dict["GPS"][piexif.GPSIFD.GPSLongitudeRef] = b"E" if lon >= 0 else b"W"
+
+        # GPS Altitude - prefer computed_altitude over altitude
+        altitude = metadata.get("computed_altitude") or metadata.get("altitude")
+        if altitude is not None:
+            exif_dict["GPS"][piexif.GPSIFD.GPSAltitude] = (int(abs(altitude) * 100), 100)
+            exif_dict["GPS"][piexif.GPSIFD.GPSAltitudeRef] = 1 if altitude < 0 else 0
+
+        # GPS Compass direction
+        compass = metadata.get("computed_compass_angle") or metadata.get("compass_angle")
+        if compass is not None:
+            exif_dict["GPS"][piexif.GPSIFD.GPSImgDirection] = (int(compass * 100), 100)
+            exif_dict["GPS"][piexif.GPSIFD.GPSImgDirectionRef] = b"T"  # True north
+
+        # GPS Version
+        exif_dict["GPS"][piexif.GPSIFD.GPSVersionID] = (2, 0, 0, 0)
+
+        # Dump and write EXIF data
+        exif_bytes = piexif.dump(exif_dict)
+        piexif.insert(exif_bytes, str(image_path))
+
+        return True
+
+    except Exception as e:
+        print(f"Warning: Failed to write EXIF data to {image_path}: {e}")
+        return False