galup 2026.10.25__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
galup/GalUp.py ADDED
@@ -0,0 +1,558 @@
1
+ import os
2
+ import re
3
+ import glob
4
+ import time
5
+ import json
6
+ import logging
7
+ import collections
8
+ import internetarchive
9
+
10
+ from internetarchive.config import parse_config_file
11
+ from datetime import datetime
12
+ from gallery_dl import config as gdl_config
13
+ from gallery_dl import exception as gdl_exception
14
+ from gallery_dl.job import DownloadJob
15
+ from .utils import get_itemname
16
+ from logging import getLogger
17
+ from urllib.parse import urlparse
18
+
19
+ from galup import __version__
20
+
21
+
22
+ DOWNLOAD_DIR_NAME = 'downloads'
23
+
24
+ VIDEO_EXTENSIONS = {'mp4', 'webm', 'mkv', 'mov', 'avi', 'flv', 'm4v', 'wmv',
25
+ 'ts', 'mpg', 'mpeg', '3gp'}
26
+ AUDIO_EXTENSIONS = {'mp3', 'm4a', 'ogg', 'opus', 'flac', 'wav', 'aac', 'wma'}
27
+
28
+
29
+ class GalleryJob(DownloadJob):
30
+ """
31
+ gallery-dl DownloadJob that reports every file it downloads (or finds
32
+ already on disk) and can skip files whose archive.org item already exists.
33
+
34
+ Child jobs (e.g. the posts of a user profile) are created by gallery-dl
35
+ with `self.__class__(extractor, parent)`, so the callbacks are inherited
36
+ from the parent job.
37
+ """
38
+
39
+ def __init__(self, url, parent=None, on_file=None, should_skip=None):
40
+ DownloadJob.__init__(self, url, parent)
41
+ if parent is not None:
42
+ on_file = parent.on_file
43
+ should_skip = parent.should_skip
44
+ self.on_file = on_file
45
+ self.should_skip = should_skip
46
+
47
+ def initialize(self, kwdict=None):
48
+ DownloadJob.initialize(self, kwdict)
49
+ if self.on_file is None:
50
+ return
51
+ if not self.hooks:
52
+ self.hooks = collections.defaultdict(list)
53
+ self.register_hooks({'after': self._report_file,
54
+ 'skip': self._report_file})
55
+
56
+ def handle_url(self, url, kwdict):
57
+ if self.should_skip is not None:
58
+ if self.pathfmt is None:
59
+ self.initialize(kwdict)
60
+ self.pathfmt.set_filename(kwdict)
61
+ basename = self.pathfmt.build_filename(kwdict)
62
+ if self.pathfmt.extension:
63
+ basename = basename[:-len(self.pathfmt.extension)]
64
+ basename = basename.rstrip('.')
65
+ if self.should_skip(kwdict, basename):
66
+ if self.archive is not None:
67
+ self.archive.add(kwdict)
68
+ return
69
+ DownloadJob.handle_url(self, url, kwdict)
70
+
71
+ def _report_file(self, pathfmt):
72
+ filepath = pathfmt.realpath
73
+ if filepath and os.path.isfile(filepath):
74
+ self.on_file(filepath, pathfmt.kwdict, self.extractor.url)
75
+
76
+
77
+ class GalUp(object):
78
+ class DirError(Exception):
79
+ pass
80
+
81
+ def __init__(self,
82
+ verbose=False,
83
+ dir_path='~/.galup',
84
+ ia_config_path=None,
85
+ output_template=None):
86
+ """
87
+ `galup` (a gallery-dl fork of TubeUp) is a tool to archive images, galleries and
88
+ other media by downloading them with gallery-dl and uploading them to
89
+ archive.org.
90
+
91
+ :param verbose: A boolean, True means all loggings will be
92
+ printed out to stdout.
93
+ :param dir_path: A path to directory that will be used for
94
+ saving the downloaded resources. Default to
95
+ '~/.galup'.
96
+ :param ia_config_path: Path to an internetarchive config file, will
97
+ be used in uploading the file.
98
+ :param output_template: A gallery-dl filename format string that will
99
+ be used to generate the output filenames.
100
+ Defaults to each extractor's own format.
101
+ """
102
+ self.dir_path = dir_path
103
+ self.verbose = verbose
104
+ self.ia_config_path = ia_config_path
105
+ self.logger = getLogger(__name__)
106
+ self.output_template = output_template
107
+
108
+ # Just print errors in quiet mode
109
+ if not self.verbose:
110
+ self.logger.setLevel(logging.ERROR)
111
+
112
+ @property
113
+ def dir_path(self):
114
+ return self._dir_path
115
+
116
+ @dir_path.setter
117
+ def dir_path(self, dir_path):
118
+ """
119
+ Set a directory to be the saving directory for resources that have
120
+ been downloaded.
121
+
122
+ :param dir_path: Path to a directory that will be used to save the
123
+ videos, if it not created yet, the directory
124
+ will be created.
125
+ """
126
+ try:
127
+ dir_path = os.fspath(dir_path)
128
+ except TypeError as exc:
129
+ raise GalUp.DirError(
130
+ 'Download directory must be a string or path-like object.'
131
+ ) from exc
132
+
133
+ if not dir_path or not dir_path.strip():
134
+ raise GalUp.DirError('Download directory must not be empty.')
135
+
136
+ extended_usr_dir_path = os.path.abspath(os.path.expanduser(dir_path))
137
+ downloads_dir_path = os.path.join(extended_usr_dir_path,
138
+ DOWNLOAD_DIR_NAME)
139
+
140
+ if (os.path.exists(extended_usr_dir_path) and
141
+ not os.path.isdir(extended_usr_dir_path)):
142
+ raise GalUp.DirError(
143
+ 'Download root path "%s" already exists as a file. '
144
+ 'Choose a different --dir to avoid data loss.'
145
+ % extended_usr_dir_path)
146
+
147
+ if os.path.exists(downloads_dir_path) and not os.path.isdir(downloads_dir_path):
148
+ raise GalUp.DirError(
149
+ 'Download path "%s" already exists as a file. '
150
+ 'Choose a different --dir to avoid data loss.'
151
+ % downloads_dir_path)
152
+
153
+ try:
154
+ os.makedirs(extended_usr_dir_path, exist_ok=True)
155
+ os.makedirs(downloads_dir_path, exist_ok=True)
156
+ except OSError as exc:
157
+ raise GalUp.DirError(
158
+ 'Cannot create download directory "%s": %s'
159
+ % (downloads_dir_path, exc)
160
+ ) from exc
161
+
162
+ self._dir_path = {
163
+ 'root': extended_usr_dir_path,
164
+ 'downloads': downloads_dir_path
165
+ }
166
+
167
+ def get_resource_basenames(self, urls,
168
+ cookie_file=None, proxy_url=None,
169
+ gdl_username=None, gdl_password=None,
170
+ use_download_archive=False,
171
+ ignore_existing_item=False):
172
+ """
173
+ Download urls with gallery-dl and get the basenames of the files.
174
+
175
+ :param urls: A list of urls that will be downloaded with
176
+ gallery-dl.
177
+ :param cookie_file: A cookie file for gallery-dl.
178
+ :param proxy_url: A proxy url for gallery-dl.
179
+ :param gdl_username: Username that will be used to download the
180
+ resources with gallery-dl.
181
+ :param gdl_password: Password of the related username, will be used
182
+ to download the resources with gallery-dl.
183
+ :param use_download_archive: Record downloaded files in the download archive.
184
+ This will download only files not listed in
185
+ the archive file.
186
+ :param ignore_existing_item: Ignores the check for existing items on archive.org.
187
+ :return: Set of file basenames that have been downloaded.
188
+ """
189
+ downloaded_files_basename = set()
190
+
191
+ def on_file(filepath, kwdict, page_url):
192
+ basename = os.path.splitext(filepath)[0]
193
+ self.write_info_json(basename, kwdict, page_url)
194
+ downloaded_files_basename.add(basename)
195
+
196
+ def check_if_ia_item_exists(kwdict, basename):
197
+ itemname = get_itemname(kwdict.get('category'), basename)
198
+ item = internetarchive.get_item(itemname)
199
+ if item.exists:
200
+ if self.verbose:
201
+ print("\n:: Item already exists. Not downloading.")
202
+ print('Item: %s\n' % itemname)
203
+ return True
204
+ return False
205
+
206
+ should_skip = None if ignore_existing_item else check_if_ia_item_exists
207
+
208
+ gdl_opts = self.generate_gdl_options(cookie_file, proxy_url,
209
+ gdl_username, gdl_password,
210
+ use_download_archive)
211
+ self.apply_gdl_options(gdl_opts)
212
+
213
+ for url in urls:
214
+ try:
215
+ job = GalleryJob(url, on_file=on_file,
216
+ should_skip=should_skip)
217
+ except gdl_exception.NoExtractorError:
218
+ self.logger.warning('URL "%s" is not supported by gallery-dl. '
219
+ 'Skipping.' % url)
220
+ continue
221
+ job.run()
222
+
223
+ self.logger.debug(
224
+ 'Basenames obtained from urls (%s): %s'
225
+ % (urls, downloaded_files_basename))
226
+
227
+ return downloaded_files_basename
228
+
229
+ @staticmethod
230
+ def write_info_json(basename, kwdict, page_url):
231
+ """
232
+ Write the gallery-dl metadata of a file to `<basename>.info.json`.
233
+
234
+ :param basename: Path of the downloaded file without its extension.
235
+ :param kwdict: gallery-dl metadata dictionary of the file.
236
+ :param page_url: URL of the page the file was extracted from.
237
+ :return: Path of the written json file.
238
+ """
239
+ meta = {key: value for key, value in kwdict.items()
240
+ if not key.startswith('_')}
241
+ meta['webpage_url'] = page_url
242
+ json_path = basename + '.info.json'
243
+ with open(json_path, 'w', encoding='utf-8') as f:
244
+ json.dump(meta, f, ensure_ascii=False, indent=4, default=str)
245
+ return json_path
246
+
247
+ def generate_gdl_options(self,
248
+ cookie_file=None,
249
+ proxy_url=None,
250
+ gdl_username=None,
251
+ gdl_password=None,
252
+ use_download_archive=False):
253
+ """
254
+ Generate a dictionary that contains options that will be used
255
+ by gallery-dl, keyed by their gallery-dl config section.
256
+
257
+ :param cookie_file: A cookie file for gallery-dl.
258
+ :param proxy_url: A proxy url for gallery-dl.
259
+ :param gdl_username: Username that will be used to download the
260
+ resources with gallery-dl.
261
+ :param gdl_password: Password of the related username, will be
262
+ used to download the resources with
263
+ gallery-dl.
264
+ :param use_download_archive: Record downloaded files in the download archive.
265
+ This will download only files not listed in
266
+ the archive file.
267
+ :return: A dictionary that contains options that will
268
+ be used by gallery-dl.
269
+ """
270
+ extractor_opts = {
271
+ 'base-directory': self.dir_path['downloads'],
272
+ 'skip': True,
273
+ 'url-metadata': 'file_url',
274
+ }
275
+
276
+ if self.output_template is not None:
277
+ extractor_opts['filename'] = self.output_template
278
+
279
+ if cookie_file is not None:
280
+ extractor_opts['cookies'] = cookie_file
281
+
282
+ if proxy_url is not None:
283
+ extractor_opts['proxy'] = proxy_url
284
+
285
+ if gdl_username is not None:
286
+ extractor_opts['username'] = gdl_username
287
+
288
+ if gdl_password is not None:
289
+ extractor_opts['password'] = gdl_password
290
+
291
+ if use_download_archive:
292
+ extractor_opts['archive'] = os.path.join(self.dir_path['root'],
293
+ '.gallerydlarchive')
294
+
295
+ return {
296
+ 'extractor': extractor_opts,
297
+ 'downloader': {
298
+ 'part': True,
299
+ 'retries': 9001,
300
+ },
301
+ 'output': {
302
+ 'mode': 'auto' if self.verbose else 'null',
303
+ 'skip': self.verbose,
304
+ },
305
+ }
306
+
307
+ @staticmethod
308
+ def apply_gdl_options(gdl_opts):
309
+ """
310
+ Load the user's gallery-dl configuration files, then apply the options
311
+ generated by `generate_gdl_options` on top of them.
312
+
313
+ :param gdl_opts: A dictionary returned by `generate_gdl_options`.
314
+ """
315
+ gdl_config.clear()
316
+ gdl_config.load()
317
+ for section, options in gdl_opts.items():
318
+ for key, value in options.items():
319
+ gdl_config.set((section,), key, value)
320
+
321
+ def upload_ia(self, basename, custom_meta=None):
322
+ """
323
+ Upload a downloaded file to archive.org.
324
+
325
+ :param basename: Path of the downloaded file without its extension.
326
+ :param custom_meta: A custom meta, will be used by internetarchive
327
+ library when uploading to archive.org.
328
+ :return: A tuple containing item name and metadata used
329
+ when uploading to archive.org.
330
+ """
331
+ json_metadata_filepath = basename + '.info.json'
332
+ with open(json_metadata_filepath, 'r', encoding='utf-8') as f:
333
+ file_meta = json.load(f)
334
+
335
+ # Exit if download did not complete, don't upload .part files to IA
336
+ pattern = glob.escape(basename)
337
+ if glob.glob(pattern + '.*.part') or glob.glob(pattern + '.part'):
338
+ msg = ('Download incomplete, please re-run or delete the .part '
339
+ 'files in downloads folder, exiting...')
340
+ raise Exception(msg)
341
+
342
+ itemname = get_itemname(file_meta.get('category'), basename)
343
+ metadata = self.create_archive_org_metadata_from_gallerydl_meta(
344
+ file_meta, os.path.basename(basename))
345
+
346
+ # Upload all files with this basename: e.g. image.jpg and
347
+ # image.info.json
348
+ files_to_upload = glob.glob(pattern + '.*')
349
+
350
+ # Upload the item to the Internet Archive
351
+ item = internetarchive.get_item(itemname)
352
+
353
+ if custom_meta:
354
+ metadata.update(custom_meta)
355
+
356
+ # Parse internetarchive configuration file.
357
+ parsed_ia_s3_config = parse_config_file(self.ia_config_path)[2]['s3']
358
+ s3_access_key = parsed_ia_s3_config['access']
359
+ s3_secret_key = parsed_ia_s3_config['secret']
360
+
361
+ if None in {s3_access_key, s3_secret_key}:
362
+ msg = ('`internetarchive` configuration file is not configured'
363
+ ' properly.')
364
+
365
+ self.logger.error(msg)
366
+ if self.verbose:
367
+ print(msg)
368
+ raise Exception(msg)
369
+
370
+ item.upload(files_to_upload, metadata=metadata, retries=9001,
371
+ request_kwargs=dict(timeout=(9001, 9001)), delete=True,
372
+ verbose=self.verbose, access_key=s3_access_key,
373
+ secret_key=s3_secret_key)
374
+
375
+ return itemname, metadata
376
+
377
+ def archive_urls(self, urls, custom_meta=None,
378
+ cookie_file=None, proxy=None,
379
+ gdl_username=None, gdl_password=None,
380
+ use_download_archive=False,
381
+ ignore_existing_item=False):
382
+ """
383
+ Download and upload files from gallery-dl supported sites to
384
+ archive.org
385
+
386
+ :param urls: List of url that will be downloaded and uploaded
387
+ to archive.org
388
+ :param custom_meta: A custom metadata that will be used when
389
+ uploading the file with archive.org.
390
+ :param cookie_file: A cookie file for gallery-dl.
391
+ :param proxy: A proxy url for gallery-dl.
392
+ :param gdl_username: Username that will be used to download the
393
+ resources with gallery-dl.
394
+ :param gdl_password: Password of the related username, will be used
395
+ to download the resources with gallery-dl.
396
+ :param use_download_archive: Record downloaded files in the download archive.
397
+ This will download only files not listed in
398
+ the archive file.
399
+ :param ignore_existing_item: Ignores the check for existing items on archive.org.
400
+ :return: Tuple containing identifier and metadata of the
401
+ file that has been uploaded to archive.org.
402
+ """
403
+ downloaded_file_basenames = self.get_resource_basenames(
404
+ urls, cookie_file, proxy, gdl_username, gdl_password,
405
+ use_download_archive, ignore_existing_item)
406
+ for basename in sorted(downloaded_file_basenames):
407
+ identifier, meta = self.upload_ia(basename, custom_meta)
408
+ yield identifier, meta
409
+
410
+ @staticmethod
411
+ def determine_media_type(file_meta, url):
412
+ """
413
+ Determine the archive.org mediatype and collection of a file.
414
+
415
+ :param file_meta: gallery-dl metadata of the file.
416
+ :param url: URL the file was downloaded from.
417
+ :return: Tuple of (mediatype, collection).
418
+ """
419
+ extension = str(file_meta.get('extension') or '').lower()
420
+ if (extension in AUDIO_EXTENSIONS or
421
+ urlparse(url).netloc == 'soundcloud.com'):
422
+ return 'audio', 'opensource_audio'
423
+ if extension in VIDEO_EXTENSIONS:
424
+ return 'movies', 'opensource_movies'
425
+ return 'image', 'opensource_image'
426
+
427
+ @staticmethod
428
+ def determine_licenseurl(file_meta):
429
+ """
430
+ Determine licenseurl for a file
431
+
432
+ :param file_meta: gallery-dl metadata of the file.
433
+ :return: License url, or an empty string if unknown.
434
+ """
435
+ licenseurl = ''
436
+ licenses = {
437
+ "Creative Commons Attribution license (reuse allowed)": "https://creativecommons.org/licenses/by/3.0/",
438
+ "Attribution-NonCommercial-ShareAlike": "https://creativecommons.org/licenses/by-nc-sa/2.0/",
439
+ "Attribution-NonCommercial": "https://creativecommons.org/licenses/by-nc/2.0/",
440
+ "Attribution-NonCommercial-NoDerivs": "https://creativecommons.org/licenses/by-nc-nd/2.0/",
441
+ "Attribution": "https://creativecommons.org/licenses/by/2.0/",
442
+ "Attribution-ShareAlike": "https://creativecommons.org/licenses/by-sa/2.0/",
443
+ "Attribution-NoDerivs": "https://creativecommons.org/licenses/by-nd/2.0/"
444
+ }
445
+
446
+ license_name = file_meta.get('license')
447
+ if isinstance(license_name, str) and license_name:
448
+ licenseurl = licenses.get(license_name, '')
449
+
450
+ return licenseurl
451
+
452
+ @staticmethod
453
+ def _first_text(file_meta, keys):
454
+ """
455
+ Return the first non-empty string value found under one of `keys`.
456
+ Dict values (e.g. gallery-dl's `user` or `author`) are searched for a
457
+ name-like field.
458
+ """
459
+ for key in keys:
460
+ value = file_meta.get(key)
461
+ if isinstance(value, dict):
462
+ value = (value.get('name') or value.get('username') or
463
+ value.get('nick'))
464
+ if isinstance(value, str) and value.strip():
465
+ return value.strip()
466
+ return None
467
+
468
+ @staticmethod
469
+ def _tags_list(value):
470
+ if isinstance(value, str):
471
+ return value.split()
472
+ if isinstance(value, list):
473
+ tags = []
474
+ for tag in value:
475
+ if isinstance(tag, dict):
476
+ tag = tag.get('name')
477
+ if isinstance(tag, str) and tag:
478
+ tags.append(tag)
479
+ return tags
480
+ return []
481
+
482
+ @staticmethod
483
+ def create_archive_org_metadata_from_gallerydl_meta(file_meta, basename):
484
+ """
485
+ Create an archive.org metadata from gallery-dl-generated metadata.
486
+
487
+ :param file_meta: A dict containing gallery-dl-generated metadata.
488
+ :param basename: Filename of the downloaded file without its
489
+ extension, used as a fallback title.
490
+ :return: A dict containing metadata to be used by
491
+ internetarchive library.
492
+ """
493
+ category = file_meta.get('category') or 'gallery-dl'
494
+ pageurl = file_meta.get('webpage_url') or file_meta.get('file_url') or ''
495
+
496
+ mediatype, collection = GalUp.determine_media_type(file_meta, pageurl)
497
+
498
+ title = GalUp._first_text(file_meta, ('title', 'post_title')) or \
499
+ '%s %s' % (category, basename)
500
+
501
+ # Some services don't tell you the uploader,
502
+ # use our program's name in that case.
503
+ uploader = GalUp._first_text(
504
+ file_meta, ('uploader', 'artist', 'author', 'user', 'username',
505
+ 'creator', 'owner')) or 'galup'
506
+
507
+ try: # some files don't give an upload date
508
+ d = datetime.fromisoformat(str(file_meta['date']))
509
+ upload_date = d.date().isoformat()
510
+ upload_year = upload_date[:4]
511
+ except (KeyError, TypeError, ValueError):
512
+ # Use current date and time as default values
513
+ upload_date = time.strftime("%Y-%m-%d")
514
+ upload_year = time.strftime("%Y")
515
+
516
+ # load up tags into an IA compatible semicolon-separated string
517
+ # example: twitter;image;
518
+ tags_string = '%s;%s;' % (category, mediatype)
519
+
520
+ for key in ('tags', 'hashtags', 'categories'):
521
+ for tag in GalUp._tags_list(file_meta.get(key)):
522
+ tags_string += '%s;' % tag
523
+
524
+ # IA's subject field has a 255 bytes length limit, so we need to truncate tags_string
525
+ while len(tags_string.encode('utf-8')) > 255:
526
+ tags_list = tags_string.split(';')
527
+ tags_list.pop()
528
+ tags_string = ';'.join(tags_list)
529
+
530
+ # license
531
+ licenseurl = GalUp.determine_licenseurl(file_meta)
532
+
533
+ description_text = GalUp._first_text(
534
+ file_meta, ('description', 'content', 'caption', 'text')) or ''
535
+ # archive.org does not display raw newlines
536
+ description = re.sub('\r?\n', '<br>', description_text)
537
+
538
+ metadata = dict(
539
+ mediatype=mediatype,
540
+ creator=uploader,
541
+ collection=collection,
542
+ title=title,
543
+ description=description,
544
+ date=upload_date,
545
+ year=upload_year,
546
+ subject=tags_string,
547
+ originalurl=pageurl,
548
+ licenseurl=licenseurl,
549
+
550
+ # Set 'scanner' metadata pair to allow tracking of galup
551
+ # powered uploads, per request from archive.org. This fork
552
+ # identifies itself separately from upstream TubeUp.
553
+ scanner='GalUp Image Video Mirroring Application {}'.format(__version__))
554
+
555
+ if file_meta.get('file_url'):
556
+ metadata['source'] = file_meta['file_url']
557
+
558
+ return metadata
galup/__init__.py ADDED
@@ -0,0 +1 @@
1
+ __version__ = '2026.10.25'
galup/__main__.py ADDED
@@ -0,0 +1,134 @@
1
+ #!/usr/bin/env python3
2
+ # -*- coding: utf-8 -*-
3
+ # galup.py - Download media using gallery-dl and upload to the Internet Archive with metadata
4
+ #
5
+ # This is a fork of TubeUp (https://github.com/bibanon/tubeup) that uses
6
+ # gallery-dl to download.
7
+
8
+ # Copyright (C) 2026 Bibliotheca Anonoma
9
+ # This program is free software: you can redistribute it and/or modify
10
+ # it under the terms of the GNU General Public License as published by
11
+ # the Free Software Foundation, either version 3 of the License, or
12
+ # (at your option) any later version.
13
+ #
14
+ # This program is distributed in the hope that it will be useful,
15
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
16
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17
+ # GNU General Public License for more details.
18
+ #
19
+ # You should have received a copy of the GNU General Public License
20
+ # along with this program. If not, see <http://www.gnu.org/licenses/>.
21
+ """galup - Download media with gallery-dl, then upload to Internet Archive, passing all metadata.
22
+
23
+ This is a fork of TubeUp (https://github.com/bibanon/tubeup) that uses gallery-dl
24
+ to download. Report issues to https://github.com/goodvids56/galup/issues
25
+
26
+ Usage:
27
+ galup <url>... [--username <user>] [--password <pass>]
28
+ [--metadata=<key:value>...]
29
+ [--cookies=<filename>]
30
+ [--proxy <prox>]
31
+ [--quiet] [--debug]
32
+ [--use-download-archive]
33
+ [--output <output>]
34
+ [--dir <dir>]
35
+ [--ignore-existing-item]
36
+ galup -h | --help
37
+ galup --version
38
+
39
+ Arguments:
40
+ <url> gallery-dl compatible URL to download.
41
+ Check gallery-dl documentation for a list
42
+ of compatible websites.
43
+ --metadata=<key:value> Custom metadata to add to the archive.org
44
+ item.
45
+ --dir <dir> Provide a directory for downloads and metadata.
46
+
47
+ Options:
48
+ -h --help Show this screen.
49
+ -x --proxy <prox> Use a proxy while downloading.
50
+ -u --username <user> Provide a username, for sites that need a login.
51
+ -p --password <pass> Provide a password, for sites that need a login.
52
+ -a --use-download-archive Record downloaded files in the download archive.
53
+ This will download only files not listed in
54
+ the archive file.
55
+ -q --quiet Just print errors.
56
+ -d --debug Print all logs to stdout.
57
+ -o --output <output> gallery-dl filename format string.
58
+ -i --ignore-existing-item Don't check if an item already exists on archive.org
59
+ """
60
+
61
+ import sys
62
+ import docopt
63
+ import logging
64
+ import traceback
65
+
66
+ from galup.utils import key_value_to_dict
67
+ from galup.GalUp import GalUp
68
+ from galup import __version__
69
+
70
+
71
+ def main():
72
+ # Parse arguments from file docstring
73
+ args = docopt.docopt(__doc__, version=__version__)
74
+
75
+ URLs = args['<url>']
76
+ cookie_file = args['--cookies']
77
+ proxy_url = args['--proxy']
78
+ username = args['--username']
79
+ password = args['--password']
80
+ quiet_mode = args['--quiet']
81
+ debug_mode = args['--debug']
82
+ use_download_archive = args['--use-download-archive']
83
+ ignore_existing_item = args['--ignore-existing-item']
84
+ dir_path = args['--dir'] or '~/.galup'
85
+
86
+ if debug_mode:
87
+ # Display log messages.
88
+ root = logging.getLogger()
89
+ root.setLevel(logging.DEBUG)
90
+
91
+ ch = logging.StreamHandler(sys.stdout)
92
+ ch.setLevel(logging.DEBUG)
93
+ formatter = logging.Formatter(
94
+ '\033[92m[DEBUG]\033[0m %(asctime)s - %(name)s - %(levelname)s - '
95
+ '%(message)s')
96
+ ch.setFormatter(formatter)
97
+ root.addHandler(ch)
98
+
99
+ metadata = key_value_to_dict(args['--metadata'])
100
+
101
+ try:
102
+ tu = GalUp(verbose=not quiet_mode,
103
+ dir_path=dir_path,
104
+ output_template=args['--output'])
105
+ except GalUp.DirError as exc:
106
+ print('\n\033[91m'
107
+ 'Cannot use download directory: %s\n'
108
+ '%s\n'
109
+ 'Please provide a valid writable path with --dir.\033[0m'
110
+ % (dir_path, exc))
111
+ sys.exit(1)
112
+
113
+ try:
114
+ for identifier, meta in tu.archive_urls(URLs, metadata,
115
+ cookie_file, proxy_url,
116
+ username, password,
117
+ use_download_archive,
118
+ ignore_existing_item):
119
+ print('\n:: Upload Finished. Item information:')
120
+ print('Title: %s' % meta['title'])
121
+ print('Item URL: https://archive.org/details/%s\n' % identifier)
122
+ except Exception:
123
+ print('\n\033[91m' # Start red color text
124
+ 'An exception just occured, if you found this '
125
+ "exception isn't related with any of your connection problem, "
126
+ 'please report this issue to '
127
+ 'https://github.com/goodvids56/galup/issues')
128
+ traceback.print_exc()
129
+ print('\033[0m') # End the red color text
130
+ sys.exit(1)
131
+
132
+
133
+ if __name__ == '__main__':
134
+ main()