skilleter-thingy 0.1.22__py3-none-any.whl → 0.1.23__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of skilleter-thingy might be problematic. Click here for more details.

@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: skilleter_thingy
3
- Version: 0.1.22
3
+ Version: 0.1.23
4
4
  Summary: A collection of useful utilities, mainly aimed at making Git more friendly
5
5
  Author-email: John Skilleter <john@skilleter.org.uk>
6
6
  Project-URL: Home, https://skilleter.org.uk
@@ -15,14 +15,13 @@ skilleter_thingy/git_common.py,sha256=KCM5HcPEvOonClKfkY9vRcBFbHicRh7L17ysrGrQ5C
15
15
  skilleter_thingy/git_hold.py,sha256=EkIHzubG7BsR-Vv-UCj8rl-9Hx7ft31t7IEf1nGW6WE,4616
16
16
  skilleter_thingy/git_mr.py,sha256=g33FaRtJTbIQI0tfXv_a042YpGwtbg5fKw072aqjAdw,3086
17
17
  skilleter_thingy/git_parent.py,sha256=s0HYTuVlEHLQCI9ZQ6EcR_9rSU9Ps3U2oSUABLX-YQI,2800
18
- skilleter_thingy/git_retag.py,sha256=70rbybb1M2sCUblmD2-iLVG0xdFq-Bl5HG0U1LRnAvs,1442
18
+ skilleter_thingy/git_retag.py,sha256=_e-oODX4mMV5o3OXSBlSYeqCCx-S4EWHEmu6Az8ABgY,1450
19
19
  skilleter_thingy/git_review.py,sha256=Z_e0wyQJ2AzOSy5cPI3jmAWystpJbHH4ygENjFagepo,52576
20
20
  skilleter_thingy/git_update.py,sha256=Z3p3E33ZJ2DVa107UzaFR9V8LXADRuHjJL8TX3IhEa4,14348
21
21
  skilleter_thingy/git_wt.py,sha256=93mbh8AKej8_y3aL6mVxKjKLEqMczaDtHoYcWf70eU0,2231
22
22
  skilleter_thingy/gitcmp_helper.py,sha256=NgQ0BZfa4TVA-XV6YKvrm5147boWUpGw-jDPUsktiMg,11346
23
23
  skilleter_thingy/gitprompt.py,sha256=SzSMd0EGI7ftPko80Q2PipwbVA-qjU1jsmdpmTCM5GI,8912
24
24
  skilleter_thingy/gl.py,sha256=9zbGpKxw6lX9RghLkdy-Q5sZlqtbB3uGFO04qTu1dH8,5954
25
- skilleter_thingy/gphotosync.py,sha256=UZ60I3aF4QE_uwVsZSB-GDXoq-2rGuD1R30LbrJg0M8,22575
26
25
  skilleter_thingy/linecount.py,sha256=ehTN6VD76i4U5k6dXuYoiqSRHI67_BP-bziklNAJSKY,4309
27
26
  skilleter_thingy/localphotosync.py,sha256=WF0TcCvLfl7cVOLzYYQK_t2WebLfQ-5FM6UB3r7Fpvw,5952
28
27
  skilleter_thingy/moviemover.py,sha256=QzUAWQzQ1AWWREIhl-VMaLo2h8MMhOekBnao5jGWV1s,4470
@@ -53,7 +52,7 @@ skilleter_thingy/thingy/dircolors.py,sha256=aBcq9ci855GSOIjrZWm8kG0ksCodvUmc4FlI
53
52
  skilleter_thingy/thingy/docker.py,sha256=9EFatudoVPfB1UbDEtzdJDB3o6ToHiNHv8-oLsUeqiQ,2449
54
53
  skilleter_thingy/thingy/files.py,sha256=oW6E6WWwVFSUPdrZnKMx7P_w_hh3etjoN7RrqvYHCHc,4705
55
54
  skilleter_thingy/thingy/git.py,sha256=d7pVv9M6k_xay3DU8-F1UxMWrN2xs1PWc7UOVvCM7Ys,39146
56
- skilleter_thingy/thingy/git2.py,sha256=1q6IMfnS7HaVRqzRVH8JTGskuMRQX_n5gkV8Foct3lA,37894
55
+ skilleter_thingy/thingy/git2.py,sha256=jIQqr0Ds2dz0zLCxBeasv_A6Ry1mGJYyEezJjAHWHXg,39028
57
56
  skilleter_thingy/thingy/gitlab.py,sha256=uXAF918xnPk6qQyiwPQDbMZfqtJzhiRqDS7yEtJEIAg,6079
58
57
  skilleter_thingy/thingy/path.py,sha256=8uM2Q9zFRWv_SaVOX49PeecQXttl7J6lsmBuRXWsXKY,4732
59
58
  skilleter_thingy/thingy/popup.py,sha256=hNfA9yh4jCv2su8XK33udcTWwgf98noBdYRRkFX1mxc,2517
@@ -62,10 +61,9 @@ skilleter_thingy/thingy/run.py,sha256=6SNKWF01fSxzB10GMU9ajraXYZqAL1w0PXkqjJdr1U
62
61
  skilleter_thingy/thingy/tfm_pane.py,sha256=XTTpSm71CyQyGmlVLuCthioOwech0jhUiFUXb-chS_Q,19792
63
62
  skilleter_thingy/thingy/tidy.py,sha256=AQ2RawsZJg6WHrgayi_ZptFL9occ7suSdCHbU3P-cys,5971
64
63
  skilleter_thingy/thingy/venv_template.py,sha256=SsVNvSwojd8NnFeQaZPCRQYTNdwJRplpZpygbUEXRnY,1015
65
- skilleter_thingy-0.1.22.dist-info/licenses/LICENSE,sha256=ljOS4DjXvqEo5VzGfdaRwgRZPbNScGBmfwyC8PChvmQ,32422
66
- skilleter_thingy-0.1.22.dist-info/METADATA,sha256=1-WECA256HZjDQ8CfNqffjh86fM2r-W5mQDEJ3U_LpI,29914
67
- skilleter_thingy-0.1.22.dist-info/PKG-INFO 2,sha256=BryF-1KGwXobPiiN60EmDKuqAtIBSPadlgmJ72xZhMk,5313
68
- skilleter_thingy-0.1.22.dist-info/WHEEL,sha256=Nw36Djuh_5VDukK0H78QzOX-_FQEo6V37m3nkm96gtU,91
69
- skilleter_thingy-0.1.22.dist-info/entry_points.txt,sha256=mklrWFvNKw9Hyem9RG3x0PoVYjlx2fDnJ3xWMTMOmfs,2258
70
- skilleter_thingy-0.1.22.dist-info/top_level.txt,sha256=8-JhgToBBiWURunmvfpSxEvNkDHQQ7r25-aBXtZv61g,17
71
- skilleter_thingy-0.1.22.dist-info/RECORD,,
64
+ skilleter_thingy-0.1.23.dist-info/licenses/LICENSE,sha256=ljOS4DjXvqEo5VzGfdaRwgRZPbNScGBmfwyC8PChvmQ,32422
65
+ skilleter_thingy-0.1.23.dist-info/METADATA,sha256=nOW1SaUso5DcdLq5F8ZEz7-RESutqKh8FHOjCVWEm9E,29914
66
+ skilleter_thingy-0.1.23.dist-info/WHEEL,sha256=Nw36Djuh_5VDukK0H78QzOX-_FQEo6V37m3nkm96gtU,91
67
+ skilleter_thingy-0.1.23.dist-info/entry_points.txt,sha256=mklrWFvNKw9Hyem9RG3x0PoVYjlx2fDnJ3xWMTMOmfs,2258
68
+ skilleter_thingy-0.1.23.dist-info/top_level.txt,sha256=8-JhgToBBiWURunmvfpSxEvNkDHQQ7r25-aBXtZv61g,17
69
+ skilleter_thingy-0.1.23.dist-info/RECORD,,
@@ -1,622 +0,0 @@
1
- #!/usr/bin/env python3
2
-
3
- """
4
- Sync Google photos with a local directory
5
-
6
- TODO: Sync local -> remote (leave this to Google Drive app?)
7
- TODO: Tidy cache (either automatic or command line option) - just remove anything with a date N months before start month
8
- TODO: When checking photos are present both locally and remotely don't just check filename (what do we check and would it help?)
9
- TODO: Investigate access to remote photos by day - is it really too slow for practical use as the rclone web site says? - ANS: Looks feasible, but have to sync each day separately, doable though would be problems with local directories with suffixes after date - probably not worth it as month works OK.
10
- """
11
-
12
- import os
13
- import sys
14
- import datetime
15
- import logging
16
- import argparse
17
- import subprocess
18
- import glob
19
- import re
20
- import shutil
21
- import PIL
22
- import imagehash
23
-
24
- from collections import defaultdict
25
-
26
- from dateutil.relativedelta import relativedelta
27
- from PIL import Image, ExifTags
28
-
29
- import thingy.colour as colour
30
-
31
- ################################################################################
32
-
33
- # Default locations for local storage of photos and videos
34
-
35
- DEFAULT_PHOTO_DIR = os.path.expanduser('~/Pictures')
36
- DEFAULT_VIDEO_DIR = os.path.expanduser('~/Videos')
37
-
38
- # Default remote name to use with rclone
39
-
40
- DEFAULT_RCLONE_REMOTE = 'GooglePhotos'
41
-
42
- # File extensions
43
-
44
- IMAGE_EXTENSIONS = ('.jpg', '.jpeg', '.rw2', '.png', )
45
- VIDEO_EXTENSIONS = ('.mp4', '.mov', )
46
- IGNORE_EXTENSIONS = ('.ini', )
47
-
48
- # Default number of months to sync
49
-
50
- DEFAULT_MONTHS = 2
51
-
52
- # Default number of months to keep in cache prior to current start date
53
-
54
- DEFAULT_KEEP = 1
55
-
56
- # Default cache location
57
-
58
- DEFAULT_CACHE_DIR = os.path.expanduser('~/.cache/gphotosync')
59
-
60
- # Enum of filetypes
61
-
62
- FILETYPE_IMAGE = 0
63
- FILETYPE_VIDEO = 1
64
- FILETYPE_UNKNOWN = 2
65
- FILETYPE_IGNORE = 3
66
-
67
- # Regexes for matching date strings
68
-
69
- YYYY_MM_DD_re = re.compile(r'^(\d{4}):(\d{2}):(\d{2})')
70
- IMG_DATE_re = re.compile(r'(?:IMG|VID)[-_](\d{4})(\d{2})(\d{2})[-_.].*')
71
-
72
- GENERAL_DATE_re = re.compile(r'(\d{4})[-_ ](\d{2})[-_ ](\d{2})')
73
-
74
- YEAR_MONTH_PATH_re = re.compile(r'/(\d{4})/(\d{2})/')
75
-
76
- YYYY_MM_re = re.compile(r'(\d{4})-(\d{2})')
77
-
78
- DUP_RE = re.compile(r'(.*) \{aalq_f.*\}(.*)')
79
-
80
- # Date format for YYYY-MM
81
-
82
- DATE_FORMAT = '%Y-%m'
83
-
84
- # If two pictures with the same name prefix have a hash differing by less than
85
- # this then we don't hash the duplicates
86
-
87
- MIN_HASH_DIFF = 15
88
-
89
- ################################################################################
90
-
91
- def parse_yyyymm(datestr):
92
- """Convert a date string in the form YYYY-MM to a datetime.date"""
93
-
94
- date_match = YYYY_MM_re.fullmatch(datestr)
95
-
96
- if not date_match:
97
- colour.error(f'Invalid date: {datestr}', prefix=True)
98
-
99
- return datetime.date(int(date_match.group(1)), int(date_match.group(2)), day=1)
100
-
101
- ################################################################################
102
-
103
- def parse_command_line():
104
- """Parse and validate the command line options"""
105
-
106
- parser = argparse.ArgumentParser(description='Sync photos from Google Photos')
107
-
108
- today = datetime.date.today()
109
-
110
- default_end_date = datetime.date(today.year, today.month, 1)
111
-
112
- parser.add_argument('--verbose', '-v', action='store_true', help='Output verbose status information')
113
- parser.add_argument('--dryrun', '-D', action='store_true', help='Just list files to be copied, without actually copying them')
114
- parser.add_argument('--picturedir', '-P', action='store', default=DEFAULT_PHOTO_DIR,
115
- help=f'Location of local picture storage directory (defaults to {DEFAULT_PHOTO_DIR})')
116
- parser.add_argument('--videodir', '-V', action='store', default=DEFAULT_VIDEO_DIR,
117
- help=f'Location of local video storage directory (defaults to {DEFAULT_VIDEO_DIR})')
118
- parser.add_argument('--start', '-s', action='store', default=None, help='Start date (in the form YYYY-MM, defaults to current month)')
119
- parser.add_argument('--end', '-e', action='store', default=None, help=f'End date (in the form YYYY-MM, defaults to {DEFAULT_MONTHS} before the start date)')
120
- parser.add_argument('--months', '-m', action='store', type=int, default=None, help='Synchronise this number of months of data (current month included)')
121
- parser.add_argument('--cache', '-c', action='store', default=DEFAULT_CACHE_DIR, help=f'Cache directory for Google photos (defaults to {DEFAULT_CACHE_DIR})')
122
- parser.add_argument('--rclone', '-r', action='store', default=DEFAULT_RCLONE_REMOTE,
123
- help=f'rclone remote name for Google photos (defaults to {DEFAULT_RCLONE_REMOTE})')
124
- parser.add_argument('--no-update', '-N', action='store_true', help='Do not update local cache')
125
- parser.add_argument('--keep', '-k', action='store', type=int, default=DEFAULT_KEEP,
126
- help=f'Keep this number of months before the start date in the cache (defaults to {DEFAULT_KEEP})')
127
- parser.add_argument('--skip-no-day', '-z', action='store_true', help='Don\'t sync files where the day of the month could not be determined')
128
- parser.add_argument('action', nargs='*', help='Actions to perform (report or sync)')
129
-
130
- args = parser.parse_args()
131
-
132
- # Set the start and end date based on parameters and defaults.
133
- # Use can specify between zero and up to 2 of start date, end date, and number of months
134
-
135
- if args.months and args.start and args.end:
136
- colour.error('You cannot specify a number of months and a start date AND an end date')
137
-
138
- # If nothing specified, then we sync the default number of months ending at this month
139
-
140
- if not args.start and not args.end and not args.months:
141
- args.months = DEFAULT_MONTHS
142
-
143
- if args.start:
144
- # If the start date has been specified then use the specified end data, or end date + months
145
-
146
- args.start = parse_yyyymm(args.start)
147
-
148
- if args.end:
149
- args.end = parse_yyyymm(args.end)
150
- else:
151
- args.end = args.start + relativedelta(months=args.months-1)
152
-
153
- else:
154
- # Otherwise, use the end date if specified and calculate the start date as being end_date - months
155
-
156
- if args.end:
157
- args.end = parse_yyyymm(args.end)
158
- else:
159
- args.end = default_end_date
160
-
161
- args.start = args.end - relativedelta(months=args.months-1)
162
-
163
- # Sanity check
164
-
165
- if args.end > default_end_date:
166
- colour.error(f'End date for synchronisation is in the future ({args.end})')
167
-
168
- # Configure debugging
169
-
170
- logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO)
171
-
172
- # Report parameters if verbose
173
-
174
- logging.debug('Start: %s', args.start)
175
- logging.debug('End: %s', args.end)
176
- logging.debug('Months: %d', args.months)
177
- logging.debug('Pictures: %s', args.picturedir)
178
- logging.debug('Videos: %s', args.videodir)
179
- logging.debug('Cache: %s', args.cache)
180
- logging.debug('Keep: %d', args.keep)
181
- logging.debug('rclone: %s', args.rclone)
182
- logging.debug('No update: %d', args.no_update)
183
- logging.debug('Dry run: %d', args.dryrun)
184
-
185
- args.local_dir = {'photo': args.picturedir, 'video': args.videodir}
186
-
187
- return args
188
-
189
- ################################################################################
190
-
191
- def get_exif_data(image):
192
- """Return EXIF data for the image as a dictionary"""
193
-
194
- try:
195
- img = Image.open(image)
196
-
197
- img_exif = img.getexif()
198
- except OSError as exc:
199
- logging.info('Error reading EXIF data for %s - %s', image, exc)
200
- img_exif = None
201
-
202
- result = {}
203
-
204
- if img_exif is None:
205
- return result
206
-
207
- for key, val in img_exif.items():
208
- if key in ExifTags.TAGS:
209
- result[ExifTags.TAGS[key]] = val
210
- else:
211
- result[key] = val
212
-
213
- return result
214
-
215
- ################################################################################
216
-
217
- def get_filetype(filename):
218
- """Return the type of a file"""
219
-
220
- _, ext = os.path.splitext(filename)
221
-
222
- ext = ext.lower()
223
-
224
- if ext in IMAGE_EXTENSIONS:
225
- return FILETYPE_IMAGE
226
-
227
- if ext in VIDEO_EXTENSIONS:
228
- return FILETYPE_VIDEO
229
-
230
- if ext in IGNORE_EXTENSIONS:
231
- return FILETYPE_IGNORE
232
-
233
- return FILETYPE_UNKNOWN
234
-
235
- ################################################################################
236
-
237
- def find_files(directory_wildcards):
238
- """Return a list of all the files in the specified directory tree, which can contain wildcards,
239
- as 3 lists; pictures, videos and unknown."""
240
-
241
- image_list = {}
242
- video_list = {}
243
- unknown_list = []
244
-
245
- logging.info('Reading files in the directory tree(s) at %s', ', '.join(directory_wildcards))
246
-
247
- for directory_wildcard in directory_wildcards:
248
- directories = glob.glob(directory_wildcard)
249
-
250
- for directory in directories:
251
- for root, _, files in os.walk(directory):
252
- logging.debug('Reading %s', root)
253
-
254
- for file in files:
255
- filepath = os.path.join(root, file)
256
-
257
- file_type = get_filetype(filepath)
258
-
259
- if file_type == FILETYPE_IMAGE:
260
- try:
261
- exif = get_exif_data(filepath)
262
-
263
- image_list[filepath] = exif
264
- except PIL.UnidentifiedImageError:
265
- colour.write(f'[BOLD:WARNING:] Unable to get EXIF data from [BLUE:{filepath}]')
266
- image_list[filepath] = {}
267
-
268
- elif file_type == FILETYPE_VIDEO:
269
- # TODO: Is there a way of getting EXIF-type data from video files? (https://thepythoncode.com/article/extract-media-metadata-in-python but does it include date info?)
270
- video_list[filepath] = {}
271
-
272
- elif file_type == FILETYPE_UNKNOWN:
273
- unknown_list.append(filepath)
274
-
275
- logging.info('Read %s image files', len(image_list))
276
- logging.info('Read %s video files', len(video_list))
277
- logging.info('Read %s unknown files', len(unknown_list))
278
-
279
- return image_list, video_list, unknown_list
280
-
281
- ################################################################################
282
-
283
- def get_media_date(name, info):
284
- """Try and determine the date for a given picture. Returns y, m, d or
285
- None, None, None"""
286
-
287
- # If the EXIF data has the date & time, just return that
288
-
289
- if 'DateTimeOriginal' in info:
290
- original_date_time = info['DateTimeOriginal']
291
-
292
- date_match = YYYY_MM_DD_re.match(original_date_time)
293
- if date_match:
294
- year = date_match.group(1)
295
- month = date_match.group(2)
296
- day = date_match.group(3)
297
-
298
- return year, month, day
299
-
300
- # No EXIF date and time, try and parse it out of the filename
301
-
302
- picture_name = os.path.basename(name)
303
-
304
- date_match = IMG_DATE_re.match(picture_name) or GENERAL_DATE_re.search(picture_name)
305
-
306
- if date_match:
307
- year = date_match.group(1)
308
- month = date_match.group(2)
309
- day = date_match.group(3)
310
-
311
- return year, month, day
312
-
313
- date_match = YEAR_MONTH_PATH_re.search(name)
314
- if date_match:
315
- year = date_match.group(1)
316
- month = date_match.group(2)
317
- day = '00'
318
-
319
- return year, month, day
320
-
321
- # A miserable failure
322
-
323
- return None, None, None
324
-
325
- ################################################################################
326
-
327
- def sync_media_local(dryrun, skip_no_day, media_files, destination_dir):
328
- """Sync files from the cache to local storage"""
329
-
330
- # Iterate through the list of remote media_files to try work out the date and
331
- # time so that we can copy it the correct local location
332
-
333
- for media_file in media_files:
334
- year, month, day = get_media_date(media_file, media_files[media_file])
335
-
336
- # If specified, skip files where the day of the month could not be determined
337
-
338
- if skip_no_day and day == '00':
339
- day = None
340
-
341
- if year and month and day:
342
- destination_media_file_path = os.path.join(destination_dir, year, f'{year}-{month}-{day}', os.path.basename(media_file))
343
-
344
- if os.path.exists(destination_media_file_path):
345
- colour.write(f'[RED:WARNING]: Destination [BLUE:{destination_media_file_path}] already exists - file will not be overwritten!')
346
- else:
347
- destination_dir_name = os.path.dirname(destination_media_file_path)
348
-
349
- colour.write(f'Copying [BLUE:{media_file}] to [BLUE:{destination_dir_name}]')
350
-
351
- if not dryrun:
352
- os.makedirs(destination_dir_name, exist_ok=True)
353
-
354
- shutil.copyfile(media_file, destination_media_file_path)
355
- else:
356
- colour.write(f'[RED:ERROR]: Unable to determine where to copy [BLUE:{media_file}]')
357
-
358
- ################################################################################
359
-
360
- def cache_directory(args, year, month):
361
- """Return the location of the cache directory for the specified year/month"""
362
-
363
- return os.path.join(args.cache, str(year), f'{month:02}')
364
-
365
- ################################################################################
366
-
367
- def local_directory(args, mediatype, year, month):
368
- """Return the location of the local picture directory for the specified year/month"""
369
-
370
- return os.path.join(args.local_dir[mediatype], str(year), f'{year}-{month:02}')
371
-
372
- ###############################################################################
373
-
374
- def update_cache(args, year, month):
375
- """Update the local cache for the specified year and month"""
376
-
377
- cache_dir = cache_directory(args, year, month)
378
-
379
- os.makedirs(cache_dir, exist_ok=True)
380
-
381
- # Sync Google photos for the specified year and month into it
382
-
383
- if not args.no_update:
384
- cmd = ['rclone', 'sync', '--progress', f'{args.rclone}:media/by-month/{year}/{year}-{month:02}/', cache_dir]
385
-
386
- colour.write('[GREEN:%s]' % '-'*80)
387
- colour.write(f'[BOLD:Caching photos for] [BLUE:{month:02}/{year}]')
388
-
389
- try:
390
- logging.info('Running %s', ' '.join(cmd))
391
-
392
- subprocess.run(cmd, check=True)
393
- except subprocess.CalledProcessError:
394
- colour.error(f'Failed to sync Google photos for month [BLUE:{month}] of year [BLUE:{year}]', prefix=True)
395
- except FileNotFoundError as exc:
396
- colour.error(exc, prefix=True)
397
-
398
- ################################################################################
399
-
400
- def media_sync(dryrun, skip_no_day, media, media_files, local_dir):
401
- """Given a media type and list of local and remote files of the type, check
402
- for out-of-sync files and sync any missing remote files to local storage"""
403
-
404
- # Get the list of local and remote names of the specified media type
405
- # TODO: Could be a problem if we have multiple files with the same name (e.g. in different months)
406
-
407
- names = {'local': {}, 'remote': {}}
408
-
409
- for name in media_files['local']:
410
- names['local'][os.path.basename(name)] = name
411
-
412
- for name in media_files['remote']:
413
- names['remote'][os.path.basename(name)] = name
414
-
415
- # Find matches and remove them
416
-
417
- matching = 0
418
- for name in names['local']:
419
- if name in names['remote']:
420
- matching += 1
421
-
422
- del media_files['remote'][names['remote'][name]]
423
- del media_files['local'][names['local'][name]]
424
-
425
- if matching:
426
- colour.write(f' [BOLD:{matching} {media} files are in sync]')
427
- else:
428
- colour.write(f' [BOLD:No {media} files are in sync]')
429
-
430
- if media_files['local']:
431
- colour.write(f' [BOLD:{len(media_files["local"])} local {media} files are out of sync]')
432
- else:
433
- colour.write(f' [BOLD:No local {media} files are out of sync]')
434
-
435
- if media_files['remote']:
436
- colour.write(f' [BOLD:{len(media_files["remote"])} remote {media} files are out of sync]')
437
- sync_media_local(dryrun, skip_no_day, media_files['remote'], local_dir)
438
- else:
439
- colour.write(f' [BOLD:No remote {media} files are out of sync]')
440
-
441
- colour.write('')
442
-
443
- ################################################################################
444
-
445
- # TODO: Tidy this up!
446
- def remove_duplicates(media_files):
447
- """Look for remote files which have an original and multiple
448
- copies and remove the copies from the list of files to consider using the
449
- imagehash library to detect duplicate or near-duplicate files.
450
- """
451
-
452
- print('Checking for duplicate files')
453
-
454
- # Originals can have upper or lower case extensions, copies only tend to have lower
455
- # case, so build a lower case to original lookup table
456
-
457
- names = {name.lower(): name for name in media_files}
458
-
459
- duplicates = defaultdict(list)
460
-
461
- # Build a list of duplicates for each filename in the list - i.e. files with the same
462
- # prefix and a suffix matching DUP_RE, indexed by the base filename (without the suffix)
463
-
464
- for entry in names:
465
- orig_match = DUP_RE.fullmatch(entry)
466
- if orig_match:
467
- original = orig_match.group(1) + orig_match.group(2)
468
-
469
- duplicates[original].append(entry)
470
-
471
- # Now use the imagehash library to check each list of maybe-duplicate files
472
- # to build a list of actual duplicates (or at least nearly-indistinguishable images)
473
- # TODO: Better to build list of all hashes, then find near-duplicates
474
-
475
- actual_duplicates = set()
476
- for entry, dupes in duplicates.items():
477
- # If the base file (no suffix) exists use that as the base, otherwise
478
- # use the first duplicate (we can have a situation where we have duplicates
479
- # and no original).
480
-
481
- hash_list = defaultdict(list)
482
-
483
- # Start with the base file, it it exists
484
-
485
- if entry in names:
486
- try:
487
- base_hash = str(imagehash.average_hash(Image.open(names[entry])))
488
-
489
- hash_list[base_hash].append(names[entry])
490
- except OSError:
491
- pass
492
-
493
- # Calculate the hash of each of the potential duplicates and if they
494
- # are close enough to the base hash, then add them to the real duplicate list
495
-
496
- for entry in dupes:
497
- filename = names[entry]
498
- try:
499
- dupe_hash = str(imagehash.average_hash(Image.open(filename)))
500
-
501
- hash_list[dupe_hash].append(filename)
502
- except OSError:
503
- colour.write(f'[BOLD:WARNING]: Unable to read {filename}')
504
-
505
- # Remove entries with identical hash values
506
-
507
- for dupes in hash_list:
508
- for dupe in hash_list[dupes][1:]:
509
- actual_duplicates.add(dupe)
510
- hash_list[dupes] = hash_list[dupes][0]
511
-
512
- # Look for adjaced entries in the sorted list of hash values that differ by less then the minimum
513
- # and remove the duplicates
514
-
515
- hash_values = sorted(hash_list.keys())
516
- logging.debug(f'Hash values for duplicates: {hash_values}')
517
-
518
- for i in range(len(hash_values)-1):
519
- if int(hash_values[i+1], 16) - int(hash_values[i], 16) < MIN_HASH_DIFF:
520
- actual_duplicates.add(hash_list[hash_values[i+1]])
521
-
522
- # Remove all the entries in the real duplicates list
523
-
524
- for entry in actual_duplicates:
525
- logging.info(f'Removing {os.path.basename(entry)} as a (near-)duplicate')
526
- del media_files[entry]
527
-
528
- ################################################################################
529
-
530
- def gphoto_sync(args, year, month):
531
- """Synchronise a month's worth of photos"""
532
-
533
- colour.write('[GREEN:%s]' % '-'*80)
534
- colour.write(f'[BOLD:Reading files for {month:02}/{year}]')
535
-
536
- # List of directories to search for media associated with this month/year
537
-
538
- cache_dirs = [cache_directory(args, year, month)]
539
- local_dirs = [local_directory(args, 'photo', year, month)+'*',
540
- local_directory(args, 'video', year, month)+'*']
541
-
542
- # Read the pictures and their EXIF data to get the dates
543
-
544
- media_files = {'photo': {}, 'video': {}}
545
- unknown_files = {}
546
-
547
- media_files['photo']['remote'], media_files['video']['remote'], unknown_files['remote'] = find_files(cache_dirs)
548
- media_files['photo']['local'], media_files['video']['local'], unknown_files['local'] = find_files(local_dirs)
549
-
550
- for media in ('photo', 'video'):
551
- remove_duplicates(media_files[media]['remote'])
552
-
553
- colour.write('[GREEN:%s]' % '-'*80)
554
- colour.write(f'[BOLD:Syncing files for {month:02}/{year}]')
555
-
556
- for media in ('photo', 'video'):
557
- media_sync(args.dryrun, args.skip_no_day, media, media_files[media], args.local_dir[media])
558
-
559
- ################################################################################
560
-
561
- def clear_cache(cache_dir, keep, start):
562
- """Clear cache entries more than keep months before the start date"""
563
-
564
- start_name = (start - relativedelta(months=keep)).strftime(DATE_FORMAT)
565
-
566
- for year_dir in glob.glob(os.path.join(cache_dir, '*')):
567
- if os.path.isdir(year_dir):
568
- for month_dir in glob.glob(os.path.join(year_dir, '*')):
569
- if os.path.isdir(month_dir):
570
- entry_date = os.path.basename(year_dir) + '-' + os.path.basename(month_dir)
571
-
572
- if entry_date < start_name:
573
- logging.info('Removing obsolete cache entry %s', month_dir)
574
- shutil.rmtree(month_dir)
575
-
576
- ################################################################################
577
-
578
- def main():
579
- """Entry point"""
580
-
581
- # Handle the command line
582
-
583
- args = parse_command_line()
584
-
585
- # Clear old entries from the cache
586
-
587
- if not args.no_update and not args.dryrun:
588
- clear_cache(args.cache, args.keep, args.start)
589
-
590
- # Update the cache
591
-
592
- for year in range(args.start.year, args.end.year+1):
593
- start_month = args.start.month if year == args.start.year else 1
594
- end_month = args.end.month if year == args.end.year else 12
595
-
596
- for month in range(start_month, end_month+1):
597
- update_cache(args, year, month)
598
-
599
- # Perform the sync
600
-
601
- for year in range(args.start.year, args.end.year+1):
602
- start_month = args.start.month if year == args.start.year else 1
603
- end_month = args.end.month if year == args.end.year else 12
604
-
605
- for month in range(start_month, end_month+1):
606
- gphoto_sync(args, year, month)
607
-
608
- ################################################################################
609
-
610
- def gphotosync():
611
- """Entry point"""
612
- try:
613
- main()
614
- except KeyboardInterrupt:
615
- sys.exit(1)
616
- except BrokenPipeError:
617
- sys.exit(2)
618
-
619
- ################################################################################
620
-
621
- if __name__ == '__main__':
622
- gphotosync()