mapillary-tools 0.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mapillary_tools-0.0.1/PKG-INFO +24 -0
- mapillary_tools-0.0.1/README.rst +77 -0
- mapillary_tools-0.0.1/bin/mapillary_auth.py +47 -0
- mapillary_tools-0.0.1/bin/mapillary_process.py +62 -0
- mapillary_tools-0.0.1/bin/mapillary_upload.py +178 -0
- mapillary_tools-0.0.1/mapillary_tools/__init__.py +329 -0
- mapillary_tools-0.0.1/mapillary_tools.egg-info/PKG-INFO +24 -0
- mapillary_tools-0.0.1/mapillary_tools.egg-info/SOURCES.txt +11 -0
- mapillary_tools-0.0.1/mapillary_tools.egg-info/dependency_links.txt +1 -0
- mapillary_tools-0.0.1/mapillary_tools.egg-info/requires.txt +4 -0
- mapillary_tools-0.0.1/mapillary_tools.egg-info/top_level.txt +1 -0
- mapillary_tools-0.0.1/setup.cfg +4 -0
- mapillary_tools-0.0.1/setup.py +41 -0
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
Metadata-Version: 1.2
|
|
2
|
+
Name: mapillary_tools
|
|
3
|
+
Version: 0.0.1
|
|
4
|
+
Summary: Mapillary Commandline Image Uploader
|
|
5
|
+
Home-page: https://github.com/MarcelloPerathoner/mapillary_tools
|
|
6
|
+
Author: Marcello Perathoner
|
|
7
|
+
Author-email: marcello@perathoner.de
|
|
8
|
+
License: GPL3
|
|
9
|
+
Description: UNKNOWN
|
|
10
|
+
Keywords: mapillary commandline console image upload
|
|
11
|
+
Platform: UNKNOWN
|
|
12
|
+
Classifier: Development Status :: 3 - Alpha
|
|
13
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
14
|
+
Classifier: Environment :: Console
|
|
15
|
+
Classifier: Intended Audience :: End Users/Desktop
|
|
16
|
+
Classifier: License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)
|
|
17
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.5
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.6
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.7
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
23
|
+
Classifier: Topic :: Utilities
|
|
24
|
+
Requires-Python: >=3.5.0
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
==============================
|
|
2
|
+
Mapillary Tools for Python 3
|
|
3
|
+
==============================
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
Tool to upload discrete images to Mapillary. Does not support videos.
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
Rewrite
|
|
10
|
+
=======
|
|
11
|
+
|
|
12
|
+
This is a complete rewrite of a portion of the official Mapillary tools found at:
|
|
13
|
+
https://github.com/mapillary/mapillary_tools
|
|
14
|
+
|
|
15
|
+
The official Mapillary tools are broken because Python 2 is dead. Major
|
|
16
|
+
distributions have already removed Python 2 from their standard installation. A
|
|
17
|
+
Python 3 port has been requested a long time ago but nothing happened.
|
|
18
|
+
|
|
19
|
+
A quick look at the official code shows that it is not worth porting anyway.
|
|
20
|
+
That code was obviously written by lots of different people with varying
|
|
21
|
+
understanding of Python and then duct-taped together to make it look like one
|
|
22
|
+
application. It also pulls in way too many dependencies, like a full image
|
|
23
|
+
manipulation library, 2 different Exif libraries (one of them custom patched)
|
|
24
|
+
and an advanced keystore.
|
|
25
|
+
|
|
26
|
+
The rewritten code uses Python 3 and only a few external libraries.
|
|
27
|
+
|
|
28
|
+
This code does not change your image files, neither does it copy them. All data
|
|
29
|
+
is put into small sidecar files. This keeps your disk lean and your backups
|
|
30
|
+
small.
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
Usage
|
|
34
|
+
=====
|
|
35
|
+
|
|
36
|
+
1. Authorize: This will prompt for your Mapillary password:
|
|
37
|
+
|
|
38
|
+
.. code-block:: shell
|
|
39
|
+
|
|
40
|
+
mapillary_auth.py --user_name <your_username> --user_email <your_email>
|
|
41
|
+
|
|
42
|
+
This step is needed only once. Your credentials are now stored in the file
|
|
43
|
+
:file:`~/.config/mapillary/configs/<CLIENT_ID>`. Keep this file secret.
|
|
44
|
+
Mapillary credentials do not expire.
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
2. Pre-process the images:
|
|
48
|
+
|
|
49
|
+
.. code-block:: shell
|
|
50
|
+
|
|
51
|
+
mapillary_process.py ~/Pictures/Mapillary/*.jpg
|
|
52
|
+
|
|
53
|
+
This step extracts GPS data from your images and stores it in sidecar files.
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
3. Upload the images:
|
|
57
|
+
|
|
58
|
+
.. code-block:: shell
|
|
59
|
+
|
|
60
|
+
mapillary_upload.py ~/Pictures/Mapillary/*.jpg
|
|
61
|
+
|
|
62
|
+
The script remembers which images were successfully uploaded. If you run the
|
|
63
|
+
upload script on the same images again, the ones already uploaded will not be
|
|
64
|
+
uploaded again.
|
|
65
|
+
|
|
66
|
+
Run the scripts with '-h' to see more options.
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
GPS Tracks
|
|
70
|
+
==========
|
|
71
|
+
|
|
72
|
+
This code has no support for GPS tracks.
|
|
73
|
+
|
|
74
|
+
To use GPS information stored in a GPS track use exiftool to insert GPS
|
|
75
|
+
information into your image files, then proces them.
|
|
76
|
+
|
|
77
|
+
https://exiftool.org/geotag.html
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
#!/usr/bin/python3
|
|
2
|
+
|
|
3
|
+
'''Get Mapillary authentication tokens and store them.
|
|
4
|
+
|
|
5
|
+
Asks for your Mapillary password and retrieves authentication tokens
|
|
6
|
+
from the Mapillary API and stores them in the configuration file.
|
|
7
|
+
|
|
8
|
+
Mapillary authentication tokens do not expire so you need to run this
|
|
9
|
+
only once.
|
|
10
|
+
|
|
11
|
+
'''
|
|
12
|
+
|
|
13
|
+
import argparse
|
|
14
|
+
import getpass
|
|
15
|
+
|
|
16
|
+
from mapillary_tools import get_auth_tokens, write_config_file
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def build_parser ():
|
|
20
|
+
''' Build the commandline parser. '''
|
|
21
|
+
|
|
22
|
+
parser = argparse.ArgumentParser (
|
|
23
|
+
description = __doc__,
|
|
24
|
+
formatter_class = argparse.RawDescriptionHelpFormatter # don't wrap my description
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
parser.add_argument (
|
|
28
|
+
'-v', '--verbose', dest='verbose', action='count',
|
|
29
|
+
help='increase output verbosity', default=0
|
|
30
|
+
)
|
|
31
|
+
parser.add_argument (
|
|
32
|
+
'-u', '--user_name', required = True,
|
|
33
|
+
help='your username on Mapillary'
|
|
34
|
+
)
|
|
35
|
+
parser.add_argument (
|
|
36
|
+
'-e', '--user_email', required = True,
|
|
37
|
+
help='your email on Mapillary'
|
|
38
|
+
)
|
|
39
|
+
return parser
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
if __name__ == '__main__':
|
|
43
|
+
args = build_parser ().parse_args ()
|
|
44
|
+
|
|
45
|
+
password = getpass.getpass ('Please enter your Mapillary user password : ')
|
|
46
|
+
|
|
47
|
+
write_config_file (get_auth_tokens (args.user_name, args.user_email, password))
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
#!/usr/bin/python3
|
|
2
|
+
|
|
3
|
+
'''Extract EXIF info from discrete image files.
|
|
4
|
+
|
|
5
|
+
Extract Latitude, Longitude, Altitude, DateTime and Heading from the image files
|
|
6
|
+
and put the information into sidecar files (same filename with an added extension
|
|
7
|
+
of .mapillary).
|
|
8
|
+
|
|
9
|
+
Process images from a directory:
|
|
10
|
+
|
|
11
|
+
mapillary_process.py ~/Pictures/Mapillary/*.jpg
|
|
12
|
+
|
|
13
|
+
Process images from a list of images (one complete filepath per line):
|
|
14
|
+
|
|
15
|
+
mapillary_process.py @picture_list.txt
|
|
16
|
+
find ~/Pictures/Mapillary -name "*.jpg" | mapillary_process.py @/dev/stdin
|
|
17
|
+
|
|
18
|
+
'''
|
|
19
|
+
|
|
20
|
+
import argparse
|
|
21
|
+
import sys
|
|
22
|
+
import uuid
|
|
23
|
+
|
|
24
|
+
from mapillary_tools import get_image_geotags, read_sidecar_file, write_sidecar_file
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def build_parser ():
|
|
28
|
+
''' Build the commandline parser. '''
|
|
29
|
+
|
|
30
|
+
parser = argparse.ArgumentParser (
|
|
31
|
+
description = __doc__,
|
|
32
|
+
formatter_class = argparse.RawDescriptionHelpFormatter, # don't wrap my description
|
|
33
|
+
fromfile_prefix_chars = '@'
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
parser.add_argument (
|
|
37
|
+
'images', metavar='FILENAME', type=str, nargs='+',
|
|
38
|
+
help='the images to process'
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
parser.add_argument (
|
|
42
|
+
'-v', '--verbose', dest='verbose', action='count',
|
|
43
|
+
help='increase output verbosity', default=0
|
|
44
|
+
)
|
|
45
|
+
return parser
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
if __name__ == '__main__':
|
|
49
|
+
args = build_parser ().parse_args ()
|
|
50
|
+
|
|
51
|
+
for image in args.images:
|
|
52
|
+
geotags = read_sidecar_file (image)
|
|
53
|
+
geotags.setdefault ('MAPPhotoUUID', str (uuid.uuid4 ()))
|
|
54
|
+
geotags.update (get_image_geotags (image))
|
|
55
|
+
|
|
56
|
+
if args.verbose:
|
|
57
|
+
print (image, geotags)
|
|
58
|
+
|
|
59
|
+
write_sidecar_file (image, geotags)
|
|
60
|
+
|
|
61
|
+
if 'MAPLatitude' not in geotags or 'MAPCaptureTime' not in geotags:
|
|
62
|
+
sys.stderr.write ('No GPS data found in {filename}\n'.format (filename = image))
|
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
#!/usr/bin/python3
|
|
2
|
+
|
|
3
|
+
''' Upload images to Mapillary.
|
|
4
|
+
|
|
5
|
+
Sorts images into sequences and uploads them to the Mapillary server.
|
|
6
|
+
|
|
7
|
+
Upload images from a directory:
|
|
8
|
+
|
|
9
|
+
mapillary_upload.py ~/Pictures/Mapillary/*.jpg
|
|
10
|
+
mapillary_upload.py -n -v ~/Pictures/Mapillary/*.jpg
|
|
11
|
+
mapillary_upload.py -d 50 -t 5 ~/Pictures/Mapillary/*.jpg
|
|
12
|
+
|
|
13
|
+
Upload images from a list of images (one complete filepath per line):
|
|
14
|
+
|
|
15
|
+
mapillary_upload.py @picture_list.txt
|
|
16
|
+
list_builder | mapillary_upload.py @/dev/stdin
|
|
17
|
+
|
|
18
|
+
'''
|
|
19
|
+
|
|
20
|
+
import argparse
|
|
21
|
+
import itertools
|
|
22
|
+
import json
|
|
23
|
+
from multiprocessing.dummy import Pool
|
|
24
|
+
import operator
|
|
25
|
+
import sys
|
|
26
|
+
|
|
27
|
+
import requests
|
|
28
|
+
from tqdm import tqdm
|
|
29
|
+
|
|
30
|
+
from mapillary_tools import read_config_file, read_sidecar_file, \
|
|
31
|
+
cut_sequences, managed_session, upload_image
|
|
32
|
+
|
|
33
|
+
SEQUENCE_THREADS = 2 # upload this many sequences in parallel
|
|
34
|
+
UPLOAD_THREADS = 2 # upload this many images in parallel per sequence
|
|
35
|
+
|
|
36
|
+
DEFAULT_MAX_TIME = 5 * 60.0 # in seconds. max time between images in same sequence
|
|
37
|
+
DEFAULT_MAX_DIST = 100.0 # in meters. max distance between images in same seq.
|
|
38
|
+
DEFAULT_MAX_DOP = 20.0 # discard images with GPS DOP greater than this
|
|
39
|
+
|
|
40
|
+
def build_parser ():
|
|
41
|
+
''' Build the commandline parser. '''
|
|
42
|
+
|
|
43
|
+
parser = argparse.ArgumentParser (
|
|
44
|
+
description = __doc__,
|
|
45
|
+
formatter_class = argparse.RawDescriptionHelpFormatter, # don't wrap my description
|
|
46
|
+
fromfile_prefix_chars = '@'
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
parser.add_argument (
|
|
50
|
+
'images', metavar='FILENAME', type=str, nargs='+',
|
|
51
|
+
help='the images to upload'
|
|
52
|
+
)
|
|
53
|
+
|
|
54
|
+
parser.add_argument (
|
|
55
|
+
'-v', '--verbose', dest='verbose', action='count',
|
|
56
|
+
help='increase output verbosity', default=0
|
|
57
|
+
)
|
|
58
|
+
parser.add_argument (
|
|
59
|
+
'-n', '--dry-run', dest='dry_run', action='store_true',
|
|
60
|
+
help='dry run: do not upload any images',
|
|
61
|
+
)
|
|
62
|
+
parser.add_argument (
|
|
63
|
+
'-t', '--t_max', type = float, metavar='SECONDS',
|
|
64
|
+
help='max time delta between images in sequence', default=DEFAULT_MAX_TIME
|
|
65
|
+
)
|
|
66
|
+
parser.add_argument (
|
|
67
|
+
'-d', '--d_max', type = float, metavar='METERS',
|
|
68
|
+
help='max distance between images in sequence', default=DEFAULT_MAX_DIST
|
|
69
|
+
)
|
|
70
|
+
parser.add_argument (
|
|
71
|
+
'--dop_max', type = float, metavar='DOP',
|
|
72
|
+
help='discard images with GPS DOP greater than this', default=DEFAULT_MAX_DOP
|
|
73
|
+
)
|
|
74
|
+
return parser
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def build_list (args):
|
|
78
|
+
''' Build a list of files to upload '''
|
|
79
|
+
|
|
80
|
+
geotags = []
|
|
81
|
+
discarded = 0
|
|
82
|
+
errors = 0
|
|
83
|
+
|
|
84
|
+
for filename in args.images:
|
|
85
|
+
try:
|
|
86
|
+
gt = read_sidecar_file (filename)
|
|
87
|
+
except (OSError, json.decoder.JSONDecodeError) as e:
|
|
88
|
+
errors += 1
|
|
89
|
+
sys.stderr.write (str (e))
|
|
90
|
+
continue
|
|
91
|
+
|
|
92
|
+
if gt.get ('status_code', 0) == 204:
|
|
93
|
+
discarded += 1
|
|
94
|
+
if args.verbose:
|
|
95
|
+
sys.stderr.write ("Image %s was already uploaded\n" % filename)
|
|
96
|
+
continue
|
|
97
|
+
|
|
98
|
+
if 'MAPLatitude' not in gt or 'MAPCaptureTime' not in gt:
|
|
99
|
+
discarded += 1
|
|
100
|
+
if args.verbose:
|
|
101
|
+
sys.stderr.write ("No GPS data for file %s\n" % filename)
|
|
102
|
+
continue
|
|
103
|
+
|
|
104
|
+
if gt.get ('MAPDOP', 1.0) > args.dop_max:
|
|
105
|
+
discarded += 1
|
|
106
|
+
if args.verbose:
|
|
107
|
+
sys.stderr.write ("Max GPS DOP exceeded in file %s\n" % filename)
|
|
108
|
+
continue
|
|
109
|
+
|
|
110
|
+
geotags.append (gt)
|
|
111
|
+
|
|
112
|
+
if args.verbose:
|
|
113
|
+
print ("%d images were discarded and %d had errors"
|
|
114
|
+
% (discarded, errors))
|
|
115
|
+
|
|
116
|
+
return geotags
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def upload_image_file (args, session, geotags):
|
|
120
|
+
''' Upload one image. '''
|
|
121
|
+
|
|
122
|
+
try:
|
|
123
|
+
upload_image (session, geotags, args.dry_run)
|
|
124
|
+
except (requests.exceptions.HTTPError, ValueError) as e:
|
|
125
|
+
sys.stderr.write ("Error uploading image: %s\n" % geotags['filename'])
|
|
126
|
+
sys.stderr.write (str (e) + '\n')
|
|
127
|
+
args.pbar.update ()
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def upload_sequence (args, sequence):
|
|
132
|
+
''' Upload a sequence of images. '''
|
|
133
|
+
|
|
134
|
+
upload_token = args.config[args.config.sections()[0]]['user_upload_token']
|
|
135
|
+
with managed_session (upload_token, args.dry_run) as session:
|
|
136
|
+
with Pool (UPLOAD_THREADS) as pool:
|
|
137
|
+
pool.starmap (upload_image_file, zip (
|
|
138
|
+
itertools.repeat (args),
|
|
139
|
+
itertools.repeat (session),
|
|
140
|
+
sequence,
|
|
141
|
+
))
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def main ():
|
|
145
|
+
''' Main function. '''
|
|
146
|
+
|
|
147
|
+
args = build_parser ().parse_args ()
|
|
148
|
+
|
|
149
|
+
args.config = read_config_file ()
|
|
150
|
+
|
|
151
|
+
geotags = build_list (args)
|
|
152
|
+
if len (geotags) == 0:
|
|
153
|
+
if args.verbose:
|
|
154
|
+
print ("No images to upload.")
|
|
155
|
+
return
|
|
156
|
+
|
|
157
|
+
geotags = cut_sequences (geotags, args.t_max, args.d_max)
|
|
158
|
+
sequences = [list (v) for k, v in itertools.groupby (
|
|
159
|
+
geotags, operator.itemgetter ('sequence_id'))]
|
|
160
|
+
|
|
161
|
+
if args.verbose:
|
|
162
|
+
print ("Sequenced %d images in %d sequences" % (len (geotags), len (sequences)))
|
|
163
|
+
|
|
164
|
+
with tqdm (total = len (geotags), desc = 'Dry run' if args.dry_run else 'Uploading',
|
|
165
|
+
unit = 'image', disable = args.verbose == 0) as args.pbar:
|
|
166
|
+
|
|
167
|
+
with Pool (SEQUENCE_THREADS) as seq_pool:
|
|
168
|
+
seq_pool.starmap (upload_sequence, zip (
|
|
169
|
+
itertools.repeat (args),
|
|
170
|
+
sequences,
|
|
171
|
+
))
|
|
172
|
+
|
|
173
|
+
if args.verbose:
|
|
174
|
+
print ("Done.")
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
if __name__ == '__main__':
|
|
178
|
+
main ()
|
|
@@ -0,0 +1,329 @@
|
|
|
1
|
+
''' Global Configuration '''
|
|
2
|
+
|
|
3
|
+
import configparser
|
|
4
|
+
from contextlib import contextmanager
|
|
5
|
+
import datetime
|
|
6
|
+
import getpass
|
|
7
|
+
import io
|
|
8
|
+
import json
|
|
9
|
+
import operator
|
|
10
|
+
import os
|
|
11
|
+
import re
|
|
12
|
+
import uuid
|
|
13
|
+
|
|
14
|
+
from geopy.distance import geodesic
|
|
15
|
+
import piexif
|
|
16
|
+
import requests
|
|
17
|
+
|
|
18
|
+
CLIENT_ID = os.getenv (
|
|
19
|
+
'MAPILLARY_CLIENT_ID',
|
|
20
|
+
'MkJKbDA0bnZuZlcxeTJHTmFqN3g1dzo1YTM0NjRkM2EyZGU5MzBh'
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
CONFIG_FILEPATH = os.getenv (
|
|
24
|
+
'MAPILLARY_CONFIG_FILEPATH',
|
|
25
|
+
os.path.join (os.path.expanduser ('~'), '.config', 'mapillary', 'configs', CLIENT_ID)
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
API_ENDPOINT = 'https://a.mapillary.com'
|
|
29
|
+
|
|
30
|
+
USERS_ENDPOINT = API_ENDPOINT + '/v3/users'
|
|
31
|
+
LOGIN_ENDPOINT = API_ENDPOINT + '/v2/ua/login'
|
|
32
|
+
UPLOAD_TOKENS_ENDPOINT = API_ENDPOINT + '/v3/users/{user_key}/upload_tokens'
|
|
33
|
+
UPLOAD_ENDPOINT = API_ENDPOINT + '/v3/me/uploads'
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def get_auth_tokens (user_name, user_email, user_password):
|
|
37
|
+
''' Get the authorization tokens from the Mapillary API. '''
|
|
38
|
+
|
|
39
|
+
try:
|
|
40
|
+
# get user key
|
|
41
|
+
message = 'Error getting user key: No user by name %s.' % user_name
|
|
42
|
+
params = { 'client_id' : CLIENT_ID, 'usernames' : user_name }
|
|
43
|
+
r = requests.get (USERS_ENDPOINT, params = params)
|
|
44
|
+
r.raise_for_status ()
|
|
45
|
+
user_key = r.json ()[0]['key']
|
|
46
|
+
|
|
47
|
+
# get upload token
|
|
48
|
+
message = 'Error getting upload token: Wrong email %s or wrong password.' % user_email
|
|
49
|
+
params = { 'client_id' : CLIENT_ID }
|
|
50
|
+
data = { 'email' : user_email, 'password' : user_password }
|
|
51
|
+
r = requests.post (LOGIN_ENDPOINT, params = params, data = data)
|
|
52
|
+
r.raise_for_status ()
|
|
53
|
+
upload_token = r.json ()['token']
|
|
54
|
+
|
|
55
|
+
# get user hashes
|
|
56
|
+
message = 'Error getting user hashes.'
|
|
57
|
+
params = { 'client_id' : CLIENT_ID }
|
|
58
|
+
headers = { 'Authorization' : 'Bearer %s' % upload_token }
|
|
59
|
+
r = requests.get (UPLOAD_TOKENS_ENDPOINT.format (user_key = user_key),
|
|
60
|
+
params = params, headers = headers)
|
|
61
|
+
r.raise_for_status ()
|
|
62
|
+
j = r.json ()
|
|
63
|
+
|
|
64
|
+
return {
|
|
65
|
+
'MAPSettingsUsername' : user_name,
|
|
66
|
+
'MAPSettingsUserKey' : user_key,
|
|
67
|
+
'user_upload_token' : upload_token,
|
|
68
|
+
'user_signature_hash' : j.get ('images_hash', ''),
|
|
69
|
+
'user_permission_hash' : j.get ('images_policy', ''),
|
|
70
|
+
'aws_access_key_id' : j.get ('aws_access_key_id', ''),
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
except (IndexError, KeyError, requests.exceptions.HTTPError):
|
|
74
|
+
print (message)
|
|
75
|
+
raise
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def gps_ascii (value):
|
|
79
|
+
''' Decode ascii bytes to string. '''
|
|
80
|
+
return value.decode ('ascii')
|
|
81
|
+
|
|
82
|
+
def gps_rational (values):
|
|
83
|
+
''' Decode exif rational value to float. '''
|
|
84
|
+
return float (values[0]) / float (values [1])
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def gps_coordinate (values, reference):
|
|
88
|
+
''' Decode exif GPS coordinate as float.
|
|
89
|
+
|
|
90
|
+
GPS Coordinates are stored as 3 rationals.
|
|
91
|
+
'''
|
|
92
|
+
|
|
93
|
+
sign = 1 if gps_ascii (reference) in 'NE' else -1
|
|
94
|
+
degrees, minutes, seconds = map (gps_rational, values)
|
|
95
|
+
return sign * (degrees + minutes / 60 + seconds / 3600)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def get_image_geotags (filename):
|
|
99
|
+
''' Extract relevant exif tags from image. '''
|
|
100
|
+
|
|
101
|
+
# Exif Version 2.3 http://www.cipa.jp/std/documents/e/DC-008-2012_E.pdf
|
|
102
|
+
|
|
103
|
+
geotags = {}
|
|
104
|
+
exif_data = piexif.load (filename)
|
|
105
|
+
t = dict (exif_data['0th'])
|
|
106
|
+
t.update (exif_data['Exif'])
|
|
107
|
+
t.update (exif_data['GPS'])
|
|
108
|
+
|
|
109
|
+
# lat, long
|
|
110
|
+
try:
|
|
111
|
+
geotags['MAPLatitude'] = gps_coordinate (t[piexif.GPSIFD.GPSLatitude],
|
|
112
|
+
t[piexif.GPSIFD.GPSLatitudeRef])
|
|
113
|
+
geotags['MAPLongitude'] = gps_coordinate (t[piexif.GPSIFD.GPSLongitude],
|
|
114
|
+
t[piexif.GPSIFD.GPSLongitudeRef])
|
|
115
|
+
except KeyError:
|
|
116
|
+
pass
|
|
117
|
+
|
|
118
|
+
# altitude
|
|
119
|
+
try:
|
|
120
|
+
sign = -1 if t.get (piexif.GPSIFD.GPSAltitudeRef, 0) else 1
|
|
121
|
+
geotags['MAPAltitude'] = sign * gps_rational (t[piexif.GPSIFD.GPSAltitude])
|
|
122
|
+
except KeyError:
|
|
123
|
+
pass
|
|
124
|
+
|
|
125
|
+
# DOP
|
|
126
|
+
try:
|
|
127
|
+
geotags['MAPDOP'] = gps_rational (t[piexif.GPSIFD.GPSDOP])
|
|
128
|
+
except KeyError:
|
|
129
|
+
pass
|
|
130
|
+
|
|
131
|
+
# datetime
|
|
132
|
+
subseconds = 0.0
|
|
133
|
+
for f in [piexif.ExifIFD.SubSecTime,
|
|
134
|
+
piexif.ExifIFD.SubSecTimeOriginal,
|
|
135
|
+
piexif.ExifIFD.SubSecTimeDigitized]:
|
|
136
|
+
if f in t:
|
|
137
|
+
subseconds = float (b"0." + t[f])
|
|
138
|
+
break
|
|
139
|
+
|
|
140
|
+
for f in [piexif.ImageIFD.DateTime,
|
|
141
|
+
piexif.ExifIFD.DateTimeOriginal,
|
|
142
|
+
piexif.ExifIFD.DateTimeDigitized]:
|
|
143
|
+
if f in t:
|
|
144
|
+
dt = " ".join (re.sub ('[^0-9]', ' ', str (t[f])).split ())
|
|
145
|
+
dt = datetime.datetime.strptime (dt, '%Y %m %d %H %M %S')
|
|
146
|
+
dt += datetime.timedelta (seconds = subseconds)
|
|
147
|
+
geotags['MAPCaptureTime'] = dt.strftime ("%Y_%m_%d_%H_%M_%S_%f")
|
|
148
|
+
break
|
|
149
|
+
|
|
150
|
+
# heading
|
|
151
|
+
try:
|
|
152
|
+
heading = gps_rational (t[piexif.GPSIFD.GPSImgDirection]) % 360.0
|
|
153
|
+
ref = 'True' if gps_ascii (t[piexif.GPSIFD.GPSImgDirectionRef]) == 'T' else 'Magnetic'
|
|
154
|
+
geotags["MAPCompassHeading"] = {
|
|
155
|
+
ref + 'Heading' : heading,
|
|
156
|
+
}
|
|
157
|
+
except KeyError:
|
|
158
|
+
pass
|
|
159
|
+
|
|
160
|
+
return geotags
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def cut_sequences (geotags, time_max, dist_max):
|
|
164
|
+
''' Build image sequences '''
|
|
165
|
+
|
|
166
|
+
geotags = sorted (geotags, key = operator.itemgetter ('MAPCaptureTime'))
|
|
167
|
+
sequence_id = 0
|
|
168
|
+
geotags[0]['sequence_id'] = sequence_id
|
|
169
|
+
|
|
170
|
+
for a, b in zip (geotags, geotags[1:]):
|
|
171
|
+
dta = datetime.datetime.strptime (a['MAPCaptureTime'], '%Y_%m_%d_%H_%M_%S_%f')
|
|
172
|
+
dtb = datetime.datetime.strptime (b['MAPCaptureTime'], '%Y_%m_%d_%H_%M_%S_%f')
|
|
173
|
+
timedelta = dtb - dta
|
|
174
|
+
dist = geodesic (
|
|
175
|
+
(a['MAPLatitude'], a['MAPLongitude']),
|
|
176
|
+
(b['MAPLatitude'], b['MAPLongitude'])
|
|
177
|
+
).m
|
|
178
|
+
if timedelta.total_seconds () > time_max or dist > dist_max:
|
|
179
|
+
# start a new sequence
|
|
180
|
+
sequence_id += 1
|
|
181
|
+
b['sequence_id'] = sequence_id
|
|
182
|
+
|
|
183
|
+
return geotags
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def get_upload_filename (filename, geotags):
|
|
187
|
+
''' Generate a filename for the uploaded image. '''
|
|
188
|
+
|
|
189
|
+
_, ext = os.path.splitext (filename) # pylint: disable=unused-variable
|
|
190
|
+
try:
|
|
191
|
+
ch = geotags['MAPCompassHeading']['TrueHeading']
|
|
192
|
+
except KeyError:
|
|
193
|
+
ch = 0.0
|
|
194
|
+
return '{MAPLatitude}_{MAPLongitude}_{ch}_{MAPCaptureTime}_{MAPPhotoUUID}{ext}'.format (
|
|
195
|
+
ch = ch, ext = ext, **geotags)
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def upload_image (session, geotags, dry_run):
|
|
199
|
+
''' Upload one image to the server. '''
|
|
200
|
+
|
|
201
|
+
filename = geotags['filename']
|
|
202
|
+
upload_filename = get_upload_filename (filename, geotags)
|
|
203
|
+
|
|
204
|
+
data = session['fields'].copy ()
|
|
205
|
+
data['key'] = session['key_prefix'] + upload_filename
|
|
206
|
+
|
|
207
|
+
with open (filename, 'rb') as fpin:
|
|
208
|
+
|
|
209
|
+
# put a json dump of the geotags into the image description field
|
|
210
|
+
exif_dict = piexif.load (fpin.read ())
|
|
211
|
+
filtered_geotags = { k: v for k, v in geotags.items () if k.startswith ('MAP') }
|
|
212
|
+
exif_dict['0th'][piexif.ImageIFD.ImageDescription] = json.dumps (filtered_geotags)
|
|
213
|
+
|
|
214
|
+
# insert exif data into memory buffer
|
|
215
|
+
fpout = io.BytesIO ()
|
|
216
|
+
fpin.seek (0)
|
|
217
|
+
try:
|
|
218
|
+
piexif.insert (piexif.dump (exif_dict), fpin.read (), fpout)
|
|
219
|
+
except ValueError as e:
|
|
220
|
+
raise Exception ("Error: invalid Exif data in image: %s\n" % filename) from e
|
|
221
|
+
|
|
222
|
+
# and upload the memory buffer
|
|
223
|
+
if dry_run:
|
|
224
|
+
geotags['status_code'] = 204
|
|
225
|
+
else:
|
|
226
|
+
r = requests.post (session['url'], data = data,
|
|
227
|
+
files = { 'file' : (upload_filename, fpout) })
|
|
228
|
+
geotags['status_code'] = r.status_code
|
|
229
|
+
geotags['uploaded_timestamp'] = datetime.datetime.utcnow ().isoformat ()
|
|
230
|
+
geotags['uploaded_filename'] = upload_filename
|
|
231
|
+
geotags['uploaded_sequence_key'] = session['key']
|
|
232
|
+
del geotags['sequence_id']
|
|
233
|
+
write_sidecar_file (filename, geotags)
|
|
234
|
+
r.raise_for_status ()
|
|
235
|
+
|
|
236
|
+
return True
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def create_session (upload_token):
|
|
240
|
+
''' Create a new upload session. '''
|
|
241
|
+
|
|
242
|
+
params = { "client_id" : CLIENT_ID }
|
|
243
|
+
headers = { "Authorization" : "Bearer " + upload_token }
|
|
244
|
+
payload = { "type" : "images/sequence" }
|
|
245
|
+
|
|
246
|
+
r = requests.post (UPLOAD_ENDPOINT, params = params, headers = headers, json = payload)
|
|
247
|
+
r.raise_for_status()
|
|
248
|
+
return r.json ()
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
def close_session (session, upload_token):
|
|
252
|
+
''' Close an upload session. '''
|
|
253
|
+
|
|
254
|
+
url = '{endpoint}/{key}/closed'.format (endpoint = UPLOAD_ENDPOINT, key = session['key'])
|
|
255
|
+
params = { "client_id" : CLIENT_ID }
|
|
256
|
+
headers = { "Authorization" : "Bearer " + upload_token }
|
|
257
|
+
|
|
258
|
+
r = requests.put (url, params = params, headers = headers)
|
|
259
|
+
r.raise_for_status()
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
@contextmanager
|
|
263
|
+
def managed_session (upload_token, dry_run):
|
|
264
|
+
''' A context manager for sessions. '''
|
|
265
|
+
|
|
266
|
+
if dry_run:
|
|
267
|
+
session = {
|
|
268
|
+
'fields' : {},
|
|
269
|
+
'key_prefix' : '',
|
|
270
|
+
}
|
|
271
|
+
else:
|
|
272
|
+
session = create_session (upload_token)
|
|
273
|
+
|
|
274
|
+
try:
|
|
275
|
+
yield session
|
|
276
|
+
|
|
277
|
+
finally:
|
|
278
|
+
if not dry_run:
|
|
279
|
+
close_session (session, upload_token)
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
def read_config_file (filename = CONFIG_FILEPATH):
|
|
283
|
+
''' Read the configuration file. '''
|
|
284
|
+
|
|
285
|
+
config = configparser.ConfigParser ()
|
|
286
|
+
config.optionxform = str # make option names case sensitive
|
|
287
|
+
config.read (filename)
|
|
288
|
+
return config
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
def write_config_file (tokens, filename = CONFIG_FILEPATH):
|
|
292
|
+
''' Write the configuration file. '''
|
|
293
|
+
|
|
294
|
+
config = read_config_file (filename)
|
|
295
|
+
|
|
296
|
+
section = tokens['MAPSettingsUsername']
|
|
297
|
+
config[section] = tokens
|
|
298
|
+
|
|
299
|
+
# file contains sensitive information so make it chmod 600
|
|
300
|
+
with open (os.open (filename, os.O_CREAT | os.O_WRONLY | os.O_TRUNC, 0o600), 'w') as fp:
|
|
301
|
+
config.write (fp)
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
def get_sidecar_filename (image_filename):
|
|
305
|
+
''' return the sidecar filename for the image. '''
|
|
306
|
+
|
|
307
|
+
return image_filename + '.mapillary'
|
|
308
|
+
|
|
309
|
+
|
|
310
|
+
def read_sidecar_file (image_filename):
|
|
311
|
+
''' Read the sidecar file for image. '''
|
|
312
|
+
|
|
313
|
+
try:
|
|
314
|
+
sidecar = get_sidecar_filename (image_filename)
|
|
315
|
+
with open (sidecar, 'r') as fp:
|
|
316
|
+
data = json.loads (fp.read ())
|
|
317
|
+
data['filename'] = image_filename
|
|
318
|
+
return data
|
|
319
|
+
except OSError as e:
|
|
320
|
+
return { 'filename' : image_filename }
|
|
321
|
+
except json.decoder.JSONDecodeError:
|
|
322
|
+
raise Exception ("JSON syntax error in sidecar file %s\n" % sidecar) from e
|
|
323
|
+
|
|
324
|
+
|
|
325
|
+
def write_sidecar_file (image_filename, geotags):
|
|
326
|
+
''' Write the sidecar file for image. '''
|
|
327
|
+
|
|
328
|
+
with open (get_sidecar_filename (image_filename), 'w') as fp:
|
|
329
|
+
fp.write (json.dumps (geotags, sort_keys = True, indent = 4))
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
Metadata-Version: 1.2
|
|
2
|
+
Name: mapillary-tools
|
|
3
|
+
Version: 0.0.1
|
|
4
|
+
Summary: Mapillary Commandline Image Uploader
|
|
5
|
+
Home-page: https://github.com/MarcelloPerathoner/mapillary_tools
|
|
6
|
+
Author: Marcello Perathoner
|
|
7
|
+
Author-email: marcello@perathoner.de
|
|
8
|
+
License: GPL3
|
|
9
|
+
Description: UNKNOWN
|
|
10
|
+
Keywords: mapillary commandline console image upload
|
|
11
|
+
Platform: UNKNOWN
|
|
12
|
+
Classifier: Development Status :: 3 - Alpha
|
|
13
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
14
|
+
Classifier: Environment :: Console
|
|
15
|
+
Classifier: Intended Audience :: End Users/Desktop
|
|
16
|
+
Classifier: License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)
|
|
17
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.5
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.6
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.7
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
23
|
+
Classifier: Topic :: Utilities
|
|
24
|
+
Requires-Python: >=3.5.0
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
README.rst
|
|
2
|
+
setup.py
|
|
3
|
+
bin/mapillary_auth.py
|
|
4
|
+
bin/mapillary_process.py
|
|
5
|
+
bin/mapillary_upload.py
|
|
6
|
+
mapillary_tools/__init__.py
|
|
7
|
+
mapillary_tools.egg-info/PKG-INFO
|
|
8
|
+
mapillary_tools.egg-info/SOURCES.txt
|
|
9
|
+
mapillary_tools.egg-info/dependency_links.txt
|
|
10
|
+
mapillary_tools.egg-info/requires.txt
|
|
11
|
+
mapillary_tools.egg-info/top_level.txt
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
mapillary_tools
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
|
|
3
|
+
from setuptools import setup
|
|
4
|
+
|
|
5
|
+
setup (
|
|
6
|
+
name='mapillary_tools',
|
|
7
|
+
version='0.0.1',
|
|
8
|
+
description='Mapillary Commandline Image Uploader',
|
|
9
|
+
keywords='mapillary commandline console image upload',
|
|
10
|
+
url='https://github.com/MarcelloPerathoner/mapillary_tools',
|
|
11
|
+
author='Marcello Perathoner',
|
|
12
|
+
author_email='marcello@perathoner.de',
|
|
13
|
+
license='GPL3',
|
|
14
|
+
python_requires='>=3.5.0',
|
|
15
|
+
packages=['mapillary_tools'],
|
|
16
|
+
classifiers=[
|
|
17
|
+
'Development Status :: 3 - Alpha',
|
|
18
|
+
'License :: OSI Approved :: MIT License',
|
|
19
|
+
'Environment :: Console',
|
|
20
|
+
'Intended Audience :: End Users/Desktop',
|
|
21
|
+
'License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)',
|
|
22
|
+
'Operating System :: POSIX :: Linux',
|
|
23
|
+
'Programming Language :: Python :: 3.5',
|
|
24
|
+
'Programming Language :: Python :: 3.6',
|
|
25
|
+
'Programming Language :: Python :: 3.7',
|
|
26
|
+
'Programming Language :: Python :: 3.8',
|
|
27
|
+
'Programming Language :: Python :: 3.9',
|
|
28
|
+
'Topic :: Utilities',
|
|
29
|
+
],
|
|
30
|
+
scripts=[
|
|
31
|
+
'bin/mapillary_auth.py',
|
|
32
|
+
'bin/mapillary_process.py',
|
|
33
|
+
'bin/mapillary_upload.py',
|
|
34
|
+
],
|
|
35
|
+
install_requires=[
|
|
36
|
+
'geopy',
|
|
37
|
+
'piexif',
|
|
38
|
+
'requests',
|
|
39
|
+
'tqdm',
|
|
40
|
+
],
|
|
41
|
+
)
|