galup 2026.10.25__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- galup/GalUp.py +558 -0
- galup/__init__.py +1 -0
- galup/__main__.py +134 -0
- galup/utils.py +53 -0
- galup-2026.10.25.dist-info/METADATA +260 -0
- galup-2026.10.25.dist-info/RECORD +10 -0
- galup-2026.10.25.dist-info/WHEEL +5 -0
- galup-2026.10.25.dist-info/entry_points.txt +2 -0
- galup-2026.10.25.dist-info/licenses/LICENSE +675 -0
- galup-2026.10.25.dist-info/top_level.txt +1 -0
galup/GalUp.py
ADDED
|
@@ -0,0 +1,558 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import re
|
|
3
|
+
import glob
|
|
4
|
+
import time
|
|
5
|
+
import json
|
|
6
|
+
import logging
|
|
7
|
+
import collections
|
|
8
|
+
import internetarchive
|
|
9
|
+
|
|
10
|
+
from internetarchive.config import parse_config_file
|
|
11
|
+
from datetime import datetime
|
|
12
|
+
from gallery_dl import config as gdl_config
|
|
13
|
+
from gallery_dl import exception as gdl_exception
|
|
14
|
+
from gallery_dl.job import DownloadJob
|
|
15
|
+
from .utils import get_itemname
|
|
16
|
+
from logging import getLogger
|
|
17
|
+
from urllib.parse import urlparse
|
|
18
|
+
|
|
19
|
+
from galup import __version__
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
DOWNLOAD_DIR_NAME = 'downloads'
|
|
23
|
+
|
|
24
|
+
VIDEO_EXTENSIONS = {'mp4', 'webm', 'mkv', 'mov', 'avi', 'flv', 'm4v', 'wmv',
|
|
25
|
+
'ts', 'mpg', 'mpeg', '3gp'}
|
|
26
|
+
AUDIO_EXTENSIONS = {'mp3', 'm4a', 'ogg', 'opus', 'flac', 'wav', 'aac', 'wma'}
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class GalleryJob(DownloadJob):
|
|
30
|
+
"""
|
|
31
|
+
gallery-dl DownloadJob that reports every file it downloads (or finds
|
|
32
|
+
already on disk) and can skip files whose archive.org item already exists.
|
|
33
|
+
|
|
34
|
+
Child jobs (e.g. the posts of a user profile) are created by gallery-dl
|
|
35
|
+
with `self.__class__(extractor, parent)`, so the callbacks are inherited
|
|
36
|
+
from the parent job.
|
|
37
|
+
"""
|
|
38
|
+
|
|
39
|
+
def __init__(self, url, parent=None, on_file=None, should_skip=None):
|
|
40
|
+
DownloadJob.__init__(self, url, parent)
|
|
41
|
+
if parent is not None:
|
|
42
|
+
on_file = parent.on_file
|
|
43
|
+
should_skip = parent.should_skip
|
|
44
|
+
self.on_file = on_file
|
|
45
|
+
self.should_skip = should_skip
|
|
46
|
+
|
|
47
|
+
def initialize(self, kwdict=None):
|
|
48
|
+
DownloadJob.initialize(self, kwdict)
|
|
49
|
+
if self.on_file is None:
|
|
50
|
+
return
|
|
51
|
+
if not self.hooks:
|
|
52
|
+
self.hooks = collections.defaultdict(list)
|
|
53
|
+
self.register_hooks({'after': self._report_file,
|
|
54
|
+
'skip': self._report_file})
|
|
55
|
+
|
|
56
|
+
def handle_url(self, url, kwdict):
|
|
57
|
+
if self.should_skip is not None:
|
|
58
|
+
if self.pathfmt is None:
|
|
59
|
+
self.initialize(kwdict)
|
|
60
|
+
self.pathfmt.set_filename(kwdict)
|
|
61
|
+
basename = self.pathfmt.build_filename(kwdict)
|
|
62
|
+
if self.pathfmt.extension:
|
|
63
|
+
basename = basename[:-len(self.pathfmt.extension)]
|
|
64
|
+
basename = basename.rstrip('.')
|
|
65
|
+
if self.should_skip(kwdict, basename):
|
|
66
|
+
if self.archive is not None:
|
|
67
|
+
self.archive.add(kwdict)
|
|
68
|
+
return
|
|
69
|
+
DownloadJob.handle_url(self, url, kwdict)
|
|
70
|
+
|
|
71
|
+
def _report_file(self, pathfmt):
|
|
72
|
+
filepath = pathfmt.realpath
|
|
73
|
+
if filepath and os.path.isfile(filepath):
|
|
74
|
+
self.on_file(filepath, pathfmt.kwdict, self.extractor.url)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
class GalUp(object):
|
|
78
|
+
class DirError(Exception):
|
|
79
|
+
pass
|
|
80
|
+
|
|
81
|
+
def __init__(self,
|
|
82
|
+
verbose=False,
|
|
83
|
+
dir_path='~/.galup',
|
|
84
|
+
ia_config_path=None,
|
|
85
|
+
output_template=None):
|
|
86
|
+
"""
|
|
87
|
+
`galup` (a gallery-dl fork of TubeUp) is a tool to archive images, galleries and
|
|
88
|
+
other media by downloading them with gallery-dl and uploading them to
|
|
89
|
+
archive.org.
|
|
90
|
+
|
|
91
|
+
:param verbose: A boolean, True means all loggings will be
|
|
92
|
+
printed out to stdout.
|
|
93
|
+
:param dir_path: A path to directory that will be used for
|
|
94
|
+
saving the downloaded resources. Default to
|
|
95
|
+
'~/.galup'.
|
|
96
|
+
:param ia_config_path: Path to an internetarchive config file, will
|
|
97
|
+
be used in uploading the file.
|
|
98
|
+
:param output_template: A gallery-dl filename format string that will
|
|
99
|
+
be used to generate the output filenames.
|
|
100
|
+
Defaults to each extractor's own format.
|
|
101
|
+
"""
|
|
102
|
+
self.dir_path = dir_path
|
|
103
|
+
self.verbose = verbose
|
|
104
|
+
self.ia_config_path = ia_config_path
|
|
105
|
+
self.logger = getLogger(__name__)
|
|
106
|
+
self.output_template = output_template
|
|
107
|
+
|
|
108
|
+
# Just print errors in quiet mode
|
|
109
|
+
if not self.verbose:
|
|
110
|
+
self.logger.setLevel(logging.ERROR)
|
|
111
|
+
|
|
112
|
+
@property
|
|
113
|
+
def dir_path(self):
|
|
114
|
+
return self._dir_path
|
|
115
|
+
|
|
116
|
+
@dir_path.setter
|
|
117
|
+
def dir_path(self, dir_path):
|
|
118
|
+
"""
|
|
119
|
+
Set a directory to be the saving directory for resources that have
|
|
120
|
+
been downloaded.
|
|
121
|
+
|
|
122
|
+
:param dir_path: Path to a directory that will be used to save the
|
|
123
|
+
videos, if it not created yet, the directory
|
|
124
|
+
will be created.
|
|
125
|
+
"""
|
|
126
|
+
try:
|
|
127
|
+
dir_path = os.fspath(dir_path)
|
|
128
|
+
except TypeError as exc:
|
|
129
|
+
raise GalUp.DirError(
|
|
130
|
+
'Download directory must be a string or path-like object.'
|
|
131
|
+
) from exc
|
|
132
|
+
|
|
133
|
+
if not dir_path or not dir_path.strip():
|
|
134
|
+
raise GalUp.DirError('Download directory must not be empty.')
|
|
135
|
+
|
|
136
|
+
extended_usr_dir_path = os.path.abspath(os.path.expanduser(dir_path))
|
|
137
|
+
downloads_dir_path = os.path.join(extended_usr_dir_path,
|
|
138
|
+
DOWNLOAD_DIR_NAME)
|
|
139
|
+
|
|
140
|
+
if (os.path.exists(extended_usr_dir_path) and
|
|
141
|
+
not os.path.isdir(extended_usr_dir_path)):
|
|
142
|
+
raise GalUp.DirError(
|
|
143
|
+
'Download root path "%s" already exists as a file. '
|
|
144
|
+
'Choose a different --dir to avoid data loss.'
|
|
145
|
+
% extended_usr_dir_path)
|
|
146
|
+
|
|
147
|
+
if os.path.exists(downloads_dir_path) and not os.path.isdir(downloads_dir_path):
|
|
148
|
+
raise GalUp.DirError(
|
|
149
|
+
'Download path "%s" already exists as a file. '
|
|
150
|
+
'Choose a different --dir to avoid data loss.'
|
|
151
|
+
% downloads_dir_path)
|
|
152
|
+
|
|
153
|
+
try:
|
|
154
|
+
os.makedirs(extended_usr_dir_path, exist_ok=True)
|
|
155
|
+
os.makedirs(downloads_dir_path, exist_ok=True)
|
|
156
|
+
except OSError as exc:
|
|
157
|
+
raise GalUp.DirError(
|
|
158
|
+
'Cannot create download directory "%s": %s'
|
|
159
|
+
% (downloads_dir_path, exc)
|
|
160
|
+
) from exc
|
|
161
|
+
|
|
162
|
+
self._dir_path = {
|
|
163
|
+
'root': extended_usr_dir_path,
|
|
164
|
+
'downloads': downloads_dir_path
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
def get_resource_basenames(self, urls,
|
|
168
|
+
cookie_file=None, proxy_url=None,
|
|
169
|
+
gdl_username=None, gdl_password=None,
|
|
170
|
+
use_download_archive=False,
|
|
171
|
+
ignore_existing_item=False):
|
|
172
|
+
"""
|
|
173
|
+
Download urls with gallery-dl and get the basenames of the files.
|
|
174
|
+
|
|
175
|
+
:param urls: A list of urls that will be downloaded with
|
|
176
|
+
gallery-dl.
|
|
177
|
+
:param cookie_file: A cookie file for gallery-dl.
|
|
178
|
+
:param proxy_url: A proxy url for gallery-dl.
|
|
179
|
+
:param gdl_username: Username that will be used to download the
|
|
180
|
+
resources with gallery-dl.
|
|
181
|
+
:param gdl_password: Password of the related username, will be used
|
|
182
|
+
to download the resources with gallery-dl.
|
|
183
|
+
:param use_download_archive: Record downloaded files in the download archive.
|
|
184
|
+
This will download only files not listed in
|
|
185
|
+
the archive file.
|
|
186
|
+
:param ignore_existing_item: Ignores the check for existing items on archive.org.
|
|
187
|
+
:return: Set of file basenames that have been downloaded.
|
|
188
|
+
"""
|
|
189
|
+
downloaded_files_basename = set()
|
|
190
|
+
|
|
191
|
+
def on_file(filepath, kwdict, page_url):
|
|
192
|
+
basename = os.path.splitext(filepath)[0]
|
|
193
|
+
self.write_info_json(basename, kwdict, page_url)
|
|
194
|
+
downloaded_files_basename.add(basename)
|
|
195
|
+
|
|
196
|
+
def check_if_ia_item_exists(kwdict, basename):
|
|
197
|
+
itemname = get_itemname(kwdict.get('category'), basename)
|
|
198
|
+
item = internetarchive.get_item(itemname)
|
|
199
|
+
if item.exists:
|
|
200
|
+
if self.verbose:
|
|
201
|
+
print("\n:: Item already exists. Not downloading.")
|
|
202
|
+
print('Item: %s\n' % itemname)
|
|
203
|
+
return True
|
|
204
|
+
return False
|
|
205
|
+
|
|
206
|
+
should_skip = None if ignore_existing_item else check_if_ia_item_exists
|
|
207
|
+
|
|
208
|
+
gdl_opts = self.generate_gdl_options(cookie_file, proxy_url,
|
|
209
|
+
gdl_username, gdl_password,
|
|
210
|
+
use_download_archive)
|
|
211
|
+
self.apply_gdl_options(gdl_opts)
|
|
212
|
+
|
|
213
|
+
for url in urls:
|
|
214
|
+
try:
|
|
215
|
+
job = GalleryJob(url, on_file=on_file,
|
|
216
|
+
should_skip=should_skip)
|
|
217
|
+
except gdl_exception.NoExtractorError:
|
|
218
|
+
self.logger.warning('URL "%s" is not supported by gallery-dl. '
|
|
219
|
+
'Skipping.' % url)
|
|
220
|
+
continue
|
|
221
|
+
job.run()
|
|
222
|
+
|
|
223
|
+
self.logger.debug(
|
|
224
|
+
'Basenames obtained from urls (%s): %s'
|
|
225
|
+
% (urls, downloaded_files_basename))
|
|
226
|
+
|
|
227
|
+
return downloaded_files_basename
|
|
228
|
+
|
|
229
|
+
@staticmethod
|
|
230
|
+
def write_info_json(basename, kwdict, page_url):
|
|
231
|
+
"""
|
|
232
|
+
Write the gallery-dl metadata of a file to `<basename>.info.json`.
|
|
233
|
+
|
|
234
|
+
:param basename: Path of the downloaded file without its extension.
|
|
235
|
+
:param kwdict: gallery-dl metadata dictionary of the file.
|
|
236
|
+
:param page_url: URL of the page the file was extracted from.
|
|
237
|
+
:return: Path of the written json file.
|
|
238
|
+
"""
|
|
239
|
+
meta = {key: value for key, value in kwdict.items()
|
|
240
|
+
if not key.startswith('_')}
|
|
241
|
+
meta['webpage_url'] = page_url
|
|
242
|
+
json_path = basename + '.info.json'
|
|
243
|
+
with open(json_path, 'w', encoding='utf-8') as f:
|
|
244
|
+
json.dump(meta, f, ensure_ascii=False, indent=4, default=str)
|
|
245
|
+
return json_path
|
|
246
|
+
|
|
247
|
+
def generate_gdl_options(self,
|
|
248
|
+
cookie_file=None,
|
|
249
|
+
proxy_url=None,
|
|
250
|
+
gdl_username=None,
|
|
251
|
+
gdl_password=None,
|
|
252
|
+
use_download_archive=False):
|
|
253
|
+
"""
|
|
254
|
+
Generate a dictionary that contains options that will be used
|
|
255
|
+
by gallery-dl, keyed by their gallery-dl config section.
|
|
256
|
+
|
|
257
|
+
:param cookie_file: A cookie file for gallery-dl.
|
|
258
|
+
:param proxy_url: A proxy url for gallery-dl.
|
|
259
|
+
:param gdl_username: Username that will be used to download the
|
|
260
|
+
resources with gallery-dl.
|
|
261
|
+
:param gdl_password: Password of the related username, will be
|
|
262
|
+
used to download the resources with
|
|
263
|
+
gallery-dl.
|
|
264
|
+
:param use_download_archive: Record downloaded files in the download archive.
|
|
265
|
+
This will download only files not listed in
|
|
266
|
+
the archive file.
|
|
267
|
+
:return: A dictionary that contains options that will
|
|
268
|
+
be used by gallery-dl.
|
|
269
|
+
"""
|
|
270
|
+
extractor_opts = {
|
|
271
|
+
'base-directory': self.dir_path['downloads'],
|
|
272
|
+
'skip': True,
|
|
273
|
+
'url-metadata': 'file_url',
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
if self.output_template is not None:
|
|
277
|
+
extractor_opts['filename'] = self.output_template
|
|
278
|
+
|
|
279
|
+
if cookie_file is not None:
|
|
280
|
+
extractor_opts['cookies'] = cookie_file
|
|
281
|
+
|
|
282
|
+
if proxy_url is not None:
|
|
283
|
+
extractor_opts['proxy'] = proxy_url
|
|
284
|
+
|
|
285
|
+
if gdl_username is not None:
|
|
286
|
+
extractor_opts['username'] = gdl_username
|
|
287
|
+
|
|
288
|
+
if gdl_password is not None:
|
|
289
|
+
extractor_opts['password'] = gdl_password
|
|
290
|
+
|
|
291
|
+
if use_download_archive:
|
|
292
|
+
extractor_opts['archive'] = os.path.join(self.dir_path['root'],
|
|
293
|
+
'.gallerydlarchive')
|
|
294
|
+
|
|
295
|
+
return {
|
|
296
|
+
'extractor': extractor_opts,
|
|
297
|
+
'downloader': {
|
|
298
|
+
'part': True,
|
|
299
|
+
'retries': 9001,
|
|
300
|
+
},
|
|
301
|
+
'output': {
|
|
302
|
+
'mode': 'auto' if self.verbose else 'null',
|
|
303
|
+
'skip': self.verbose,
|
|
304
|
+
},
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
@staticmethod
|
|
308
|
+
def apply_gdl_options(gdl_opts):
|
|
309
|
+
"""
|
|
310
|
+
Load the user's gallery-dl configuration files, then apply the options
|
|
311
|
+
generated by `generate_gdl_options` on top of them.
|
|
312
|
+
|
|
313
|
+
:param gdl_opts: A dictionary returned by `generate_gdl_options`.
|
|
314
|
+
"""
|
|
315
|
+
gdl_config.clear()
|
|
316
|
+
gdl_config.load()
|
|
317
|
+
for section, options in gdl_opts.items():
|
|
318
|
+
for key, value in options.items():
|
|
319
|
+
gdl_config.set((section,), key, value)
|
|
320
|
+
|
|
321
|
+
def upload_ia(self, basename, custom_meta=None):
|
|
322
|
+
"""
|
|
323
|
+
Upload a downloaded file to archive.org.
|
|
324
|
+
|
|
325
|
+
:param basename: Path of the downloaded file without its extension.
|
|
326
|
+
:param custom_meta: A custom meta, will be used by internetarchive
|
|
327
|
+
library when uploading to archive.org.
|
|
328
|
+
:return: A tuple containing item name and metadata used
|
|
329
|
+
when uploading to archive.org.
|
|
330
|
+
"""
|
|
331
|
+
json_metadata_filepath = basename + '.info.json'
|
|
332
|
+
with open(json_metadata_filepath, 'r', encoding='utf-8') as f:
|
|
333
|
+
file_meta = json.load(f)
|
|
334
|
+
|
|
335
|
+
# Exit if download did not complete, don't upload .part files to IA
|
|
336
|
+
pattern = glob.escape(basename)
|
|
337
|
+
if glob.glob(pattern + '.*.part') or glob.glob(pattern + '.part'):
|
|
338
|
+
msg = ('Download incomplete, please re-run or delete the .part '
|
|
339
|
+
'files in downloads folder, exiting...')
|
|
340
|
+
raise Exception(msg)
|
|
341
|
+
|
|
342
|
+
itemname = get_itemname(file_meta.get('category'), basename)
|
|
343
|
+
metadata = self.create_archive_org_metadata_from_gallerydl_meta(
|
|
344
|
+
file_meta, os.path.basename(basename))
|
|
345
|
+
|
|
346
|
+
# Upload all files with this basename: e.g. image.jpg and
|
|
347
|
+
# image.info.json
|
|
348
|
+
files_to_upload = glob.glob(pattern + '.*')
|
|
349
|
+
|
|
350
|
+
# Upload the item to the Internet Archive
|
|
351
|
+
item = internetarchive.get_item(itemname)
|
|
352
|
+
|
|
353
|
+
if custom_meta:
|
|
354
|
+
metadata.update(custom_meta)
|
|
355
|
+
|
|
356
|
+
# Parse internetarchive configuration file.
|
|
357
|
+
parsed_ia_s3_config = parse_config_file(self.ia_config_path)[2]['s3']
|
|
358
|
+
s3_access_key = parsed_ia_s3_config['access']
|
|
359
|
+
s3_secret_key = parsed_ia_s3_config['secret']
|
|
360
|
+
|
|
361
|
+
if None in {s3_access_key, s3_secret_key}:
|
|
362
|
+
msg = ('`internetarchive` configuration file is not configured'
|
|
363
|
+
' properly.')
|
|
364
|
+
|
|
365
|
+
self.logger.error(msg)
|
|
366
|
+
if self.verbose:
|
|
367
|
+
print(msg)
|
|
368
|
+
raise Exception(msg)
|
|
369
|
+
|
|
370
|
+
item.upload(files_to_upload, metadata=metadata, retries=9001,
|
|
371
|
+
request_kwargs=dict(timeout=(9001, 9001)), delete=True,
|
|
372
|
+
verbose=self.verbose, access_key=s3_access_key,
|
|
373
|
+
secret_key=s3_secret_key)
|
|
374
|
+
|
|
375
|
+
return itemname, metadata
|
|
376
|
+
|
|
377
|
+
def archive_urls(self, urls, custom_meta=None,
|
|
378
|
+
cookie_file=None, proxy=None,
|
|
379
|
+
gdl_username=None, gdl_password=None,
|
|
380
|
+
use_download_archive=False,
|
|
381
|
+
ignore_existing_item=False):
|
|
382
|
+
"""
|
|
383
|
+
Download and upload files from gallery-dl supported sites to
|
|
384
|
+
archive.org
|
|
385
|
+
|
|
386
|
+
:param urls: List of url that will be downloaded and uploaded
|
|
387
|
+
to archive.org
|
|
388
|
+
:param custom_meta: A custom metadata that will be used when
|
|
389
|
+
uploading the file with archive.org.
|
|
390
|
+
:param cookie_file: A cookie file for gallery-dl.
|
|
391
|
+
:param proxy: A proxy url for gallery-dl.
|
|
392
|
+
:param gdl_username: Username that will be used to download the
|
|
393
|
+
resources with gallery-dl.
|
|
394
|
+
:param gdl_password: Password of the related username, will be used
|
|
395
|
+
to download the resources with gallery-dl.
|
|
396
|
+
:param use_download_archive: Record downloaded files in the download archive.
|
|
397
|
+
This will download only files not listed in
|
|
398
|
+
the archive file.
|
|
399
|
+
:param ignore_existing_item: Ignores the check for existing items on archive.org.
|
|
400
|
+
:return: Tuple containing identifier and metadata of the
|
|
401
|
+
file that has been uploaded to archive.org.
|
|
402
|
+
"""
|
|
403
|
+
downloaded_file_basenames = self.get_resource_basenames(
|
|
404
|
+
urls, cookie_file, proxy, gdl_username, gdl_password,
|
|
405
|
+
use_download_archive, ignore_existing_item)
|
|
406
|
+
for basename in sorted(downloaded_file_basenames):
|
|
407
|
+
identifier, meta = self.upload_ia(basename, custom_meta)
|
|
408
|
+
yield identifier, meta
|
|
409
|
+
|
|
410
|
+
@staticmethod
|
|
411
|
+
def determine_media_type(file_meta, url):
|
|
412
|
+
"""
|
|
413
|
+
Determine the archive.org mediatype and collection of a file.
|
|
414
|
+
|
|
415
|
+
:param file_meta: gallery-dl metadata of the file.
|
|
416
|
+
:param url: URL the file was downloaded from.
|
|
417
|
+
:return: Tuple of (mediatype, collection).
|
|
418
|
+
"""
|
|
419
|
+
extension = str(file_meta.get('extension') or '').lower()
|
|
420
|
+
if (extension in AUDIO_EXTENSIONS or
|
|
421
|
+
urlparse(url).netloc == 'soundcloud.com'):
|
|
422
|
+
return 'audio', 'opensource_audio'
|
|
423
|
+
if extension in VIDEO_EXTENSIONS:
|
|
424
|
+
return 'movies', 'opensource_movies'
|
|
425
|
+
return 'image', 'opensource_image'
|
|
426
|
+
|
|
427
|
+
@staticmethod
|
|
428
|
+
def determine_licenseurl(file_meta):
|
|
429
|
+
"""
|
|
430
|
+
Determine licenseurl for a file
|
|
431
|
+
|
|
432
|
+
:param file_meta: gallery-dl metadata of the file.
|
|
433
|
+
:return: License url, or an empty string if unknown.
|
|
434
|
+
"""
|
|
435
|
+
licenseurl = ''
|
|
436
|
+
licenses = {
|
|
437
|
+
"Creative Commons Attribution license (reuse allowed)": "https://creativecommons.org/licenses/by/3.0/",
|
|
438
|
+
"Attribution-NonCommercial-ShareAlike": "https://creativecommons.org/licenses/by-nc-sa/2.0/",
|
|
439
|
+
"Attribution-NonCommercial": "https://creativecommons.org/licenses/by-nc/2.0/",
|
|
440
|
+
"Attribution-NonCommercial-NoDerivs": "https://creativecommons.org/licenses/by-nc-nd/2.0/",
|
|
441
|
+
"Attribution": "https://creativecommons.org/licenses/by/2.0/",
|
|
442
|
+
"Attribution-ShareAlike": "https://creativecommons.org/licenses/by-sa/2.0/",
|
|
443
|
+
"Attribution-NoDerivs": "https://creativecommons.org/licenses/by-nd/2.0/"
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
license_name = file_meta.get('license')
|
|
447
|
+
if isinstance(license_name, str) and license_name:
|
|
448
|
+
licenseurl = licenses.get(license_name, '')
|
|
449
|
+
|
|
450
|
+
return licenseurl
|
|
451
|
+
|
|
452
|
+
@staticmethod
|
|
453
|
+
def _first_text(file_meta, keys):
|
|
454
|
+
"""
|
|
455
|
+
Return the first non-empty string value found under one of `keys`.
|
|
456
|
+
Dict values (e.g. gallery-dl's `user` or `author`) are searched for a
|
|
457
|
+
name-like field.
|
|
458
|
+
"""
|
|
459
|
+
for key in keys:
|
|
460
|
+
value = file_meta.get(key)
|
|
461
|
+
if isinstance(value, dict):
|
|
462
|
+
value = (value.get('name') or value.get('username') or
|
|
463
|
+
value.get('nick'))
|
|
464
|
+
if isinstance(value, str) and value.strip():
|
|
465
|
+
return value.strip()
|
|
466
|
+
return None
|
|
467
|
+
|
|
468
|
+
@staticmethod
|
|
469
|
+
def _tags_list(value):
|
|
470
|
+
if isinstance(value, str):
|
|
471
|
+
return value.split()
|
|
472
|
+
if isinstance(value, list):
|
|
473
|
+
tags = []
|
|
474
|
+
for tag in value:
|
|
475
|
+
if isinstance(tag, dict):
|
|
476
|
+
tag = tag.get('name')
|
|
477
|
+
if isinstance(tag, str) and tag:
|
|
478
|
+
tags.append(tag)
|
|
479
|
+
return tags
|
|
480
|
+
return []
|
|
481
|
+
|
|
482
|
+
@staticmethod
|
|
483
|
+
def create_archive_org_metadata_from_gallerydl_meta(file_meta, basename):
|
|
484
|
+
"""
|
|
485
|
+
Create an archive.org metadata from gallery-dl-generated metadata.
|
|
486
|
+
|
|
487
|
+
:param file_meta: A dict containing gallery-dl-generated metadata.
|
|
488
|
+
:param basename: Filename of the downloaded file without its
|
|
489
|
+
extension, used as a fallback title.
|
|
490
|
+
:return: A dict containing metadata to be used by
|
|
491
|
+
internetarchive library.
|
|
492
|
+
"""
|
|
493
|
+
category = file_meta.get('category') or 'gallery-dl'
|
|
494
|
+
pageurl = file_meta.get('webpage_url') or file_meta.get('file_url') or ''
|
|
495
|
+
|
|
496
|
+
mediatype, collection = GalUp.determine_media_type(file_meta, pageurl)
|
|
497
|
+
|
|
498
|
+
title = GalUp._first_text(file_meta, ('title', 'post_title')) or \
|
|
499
|
+
'%s %s' % (category, basename)
|
|
500
|
+
|
|
501
|
+
# Some services don't tell you the uploader,
|
|
502
|
+
# use our program's name in that case.
|
|
503
|
+
uploader = GalUp._first_text(
|
|
504
|
+
file_meta, ('uploader', 'artist', 'author', 'user', 'username',
|
|
505
|
+
'creator', 'owner')) or 'galup'
|
|
506
|
+
|
|
507
|
+
try: # some files don't give an upload date
|
|
508
|
+
d = datetime.fromisoformat(str(file_meta['date']))
|
|
509
|
+
upload_date = d.date().isoformat()
|
|
510
|
+
upload_year = upload_date[:4]
|
|
511
|
+
except (KeyError, TypeError, ValueError):
|
|
512
|
+
# Use current date and time as default values
|
|
513
|
+
upload_date = time.strftime("%Y-%m-%d")
|
|
514
|
+
upload_year = time.strftime("%Y")
|
|
515
|
+
|
|
516
|
+
# load up tags into an IA compatible semicolon-separated string
|
|
517
|
+
# example: twitter;image;
|
|
518
|
+
tags_string = '%s;%s;' % (category, mediatype)
|
|
519
|
+
|
|
520
|
+
for key in ('tags', 'hashtags', 'categories'):
|
|
521
|
+
for tag in GalUp._tags_list(file_meta.get(key)):
|
|
522
|
+
tags_string += '%s;' % tag
|
|
523
|
+
|
|
524
|
+
# IA's subject field has a 255 bytes length limit, so we need to truncate tags_string
|
|
525
|
+
while len(tags_string.encode('utf-8')) > 255:
|
|
526
|
+
tags_list = tags_string.split(';')
|
|
527
|
+
tags_list.pop()
|
|
528
|
+
tags_string = ';'.join(tags_list)
|
|
529
|
+
|
|
530
|
+
# license
|
|
531
|
+
licenseurl = GalUp.determine_licenseurl(file_meta)
|
|
532
|
+
|
|
533
|
+
description_text = GalUp._first_text(
|
|
534
|
+
file_meta, ('description', 'content', 'caption', 'text')) or ''
|
|
535
|
+
# archive.org does not display raw newlines
|
|
536
|
+
description = re.sub('\r?\n', '<br>', description_text)
|
|
537
|
+
|
|
538
|
+
metadata = dict(
|
|
539
|
+
mediatype=mediatype,
|
|
540
|
+
creator=uploader,
|
|
541
|
+
collection=collection,
|
|
542
|
+
title=title,
|
|
543
|
+
description=description,
|
|
544
|
+
date=upload_date,
|
|
545
|
+
year=upload_year,
|
|
546
|
+
subject=tags_string,
|
|
547
|
+
originalurl=pageurl,
|
|
548
|
+
licenseurl=licenseurl,
|
|
549
|
+
|
|
550
|
+
# Set 'scanner' metadata pair to allow tracking of galup
|
|
551
|
+
# powered uploads, per request from archive.org. This fork
|
|
552
|
+
# identifies itself separately from upstream TubeUp.
|
|
553
|
+
scanner='GalUp Image Video Mirroring Application {}'.format(__version__))
|
|
554
|
+
|
|
555
|
+
if file_meta.get('file_url'):
|
|
556
|
+
metadata['source'] = file_meta['file_url']
|
|
557
|
+
|
|
558
|
+
return metadata
|
galup/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = '2026.10.25'
|
galup/__main__.py
ADDED
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
# -*- coding: utf-8 -*-
|
|
3
|
+
# galup.py - Download media using gallery-dl and upload to the Internet Archive with metadata
|
|
4
|
+
#
|
|
5
|
+
# This is a fork of TubeUp (https://github.com/bibanon/tubeup) that uses
|
|
6
|
+
# gallery-dl to download.
|
|
7
|
+
|
|
8
|
+
# Copyright (C) 2026 Bibliotheca Anonoma
|
|
9
|
+
# This program is free software: you can redistribute it and/or modify
|
|
10
|
+
# it under the terms of the GNU General Public License as published by
|
|
11
|
+
# the Free Software Foundation, either version 3 of the License, or
|
|
12
|
+
# (at your option) any later version.
|
|
13
|
+
#
|
|
14
|
+
# This program is distributed in the hope that it will be useful,
|
|
15
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
16
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
17
|
+
# GNU General Public License for more details.
|
|
18
|
+
#
|
|
19
|
+
# You should have received a copy of the GNU General Public License
|
|
20
|
+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
21
|
+
"""galup - Download media with gallery-dl, then upload to Internet Archive, passing all metadata.
|
|
22
|
+
|
|
23
|
+
This is a fork of TubeUp (https://github.com/bibanon/tubeup) that uses gallery-dl
|
|
24
|
+
to download. Report issues to https://github.com/goodvids56/galup/issues
|
|
25
|
+
|
|
26
|
+
Usage:
|
|
27
|
+
galup <url>... [--username <user>] [--password <pass>]
|
|
28
|
+
[--metadata=<key:value>...]
|
|
29
|
+
[--cookies=<filename>]
|
|
30
|
+
[--proxy <prox>]
|
|
31
|
+
[--quiet] [--debug]
|
|
32
|
+
[--use-download-archive]
|
|
33
|
+
[--output <output>]
|
|
34
|
+
[--dir <dir>]
|
|
35
|
+
[--ignore-existing-item]
|
|
36
|
+
galup -h | --help
|
|
37
|
+
galup --version
|
|
38
|
+
|
|
39
|
+
Arguments:
|
|
40
|
+
<url> gallery-dl compatible URL to download.
|
|
41
|
+
Check gallery-dl documentation for a list
|
|
42
|
+
of compatible websites.
|
|
43
|
+
--metadata=<key:value> Custom metadata to add to the archive.org
|
|
44
|
+
item.
|
|
45
|
+
--dir <dir> Provide a directory for downloads and metadata.
|
|
46
|
+
|
|
47
|
+
Options:
|
|
48
|
+
-h --help Show this screen.
|
|
49
|
+
-x --proxy <prox> Use a proxy while downloading.
|
|
50
|
+
-u --username <user> Provide a username, for sites that need a login.
|
|
51
|
+
-p --password <pass> Provide a password, for sites that need a login.
|
|
52
|
+
-a --use-download-archive Record downloaded files in the download archive.
|
|
53
|
+
This will download only files not listed in
|
|
54
|
+
the archive file.
|
|
55
|
+
-q --quiet Just print errors.
|
|
56
|
+
-d --debug Print all logs to stdout.
|
|
57
|
+
-o --output <output> gallery-dl filename format string.
|
|
58
|
+
-i --ignore-existing-item Don't check if an item already exists on archive.org
|
|
59
|
+
"""
|
|
60
|
+
|
|
61
|
+
import sys
|
|
62
|
+
import docopt
|
|
63
|
+
import logging
|
|
64
|
+
import traceback
|
|
65
|
+
|
|
66
|
+
from galup.utils import key_value_to_dict
|
|
67
|
+
from galup.GalUp import GalUp
|
|
68
|
+
from galup import __version__
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def main():
|
|
72
|
+
# Parse arguments from file docstring
|
|
73
|
+
args = docopt.docopt(__doc__, version=__version__)
|
|
74
|
+
|
|
75
|
+
URLs = args['<url>']
|
|
76
|
+
cookie_file = args['--cookies']
|
|
77
|
+
proxy_url = args['--proxy']
|
|
78
|
+
username = args['--username']
|
|
79
|
+
password = args['--password']
|
|
80
|
+
quiet_mode = args['--quiet']
|
|
81
|
+
debug_mode = args['--debug']
|
|
82
|
+
use_download_archive = args['--use-download-archive']
|
|
83
|
+
ignore_existing_item = args['--ignore-existing-item']
|
|
84
|
+
dir_path = args['--dir'] or '~/.galup'
|
|
85
|
+
|
|
86
|
+
if debug_mode:
|
|
87
|
+
# Display log messages.
|
|
88
|
+
root = logging.getLogger()
|
|
89
|
+
root.setLevel(logging.DEBUG)
|
|
90
|
+
|
|
91
|
+
ch = logging.StreamHandler(sys.stdout)
|
|
92
|
+
ch.setLevel(logging.DEBUG)
|
|
93
|
+
formatter = logging.Formatter(
|
|
94
|
+
'\033[92m[DEBUG]\033[0m %(asctime)s - %(name)s - %(levelname)s - '
|
|
95
|
+
'%(message)s')
|
|
96
|
+
ch.setFormatter(formatter)
|
|
97
|
+
root.addHandler(ch)
|
|
98
|
+
|
|
99
|
+
metadata = key_value_to_dict(args['--metadata'])
|
|
100
|
+
|
|
101
|
+
try:
|
|
102
|
+
tu = GalUp(verbose=not quiet_mode,
|
|
103
|
+
dir_path=dir_path,
|
|
104
|
+
output_template=args['--output'])
|
|
105
|
+
except GalUp.DirError as exc:
|
|
106
|
+
print('\n\033[91m'
|
|
107
|
+
'Cannot use download directory: %s\n'
|
|
108
|
+
'%s\n'
|
|
109
|
+
'Please provide a valid writable path with --dir.\033[0m'
|
|
110
|
+
% (dir_path, exc))
|
|
111
|
+
sys.exit(1)
|
|
112
|
+
|
|
113
|
+
try:
|
|
114
|
+
for identifier, meta in tu.archive_urls(URLs, metadata,
|
|
115
|
+
cookie_file, proxy_url,
|
|
116
|
+
username, password,
|
|
117
|
+
use_download_archive,
|
|
118
|
+
ignore_existing_item):
|
|
119
|
+
print('\n:: Upload Finished. Item information:')
|
|
120
|
+
print('Title: %s' % meta['title'])
|
|
121
|
+
print('Item URL: https://archive.org/details/%s\n' % identifier)
|
|
122
|
+
except Exception:
|
|
123
|
+
print('\n\033[91m' # Start red color text
|
|
124
|
+
'An exception just occured, if you found this '
|
|
125
|
+
"exception isn't related with any of your connection problem, "
|
|
126
|
+
'please report this issue to '
|
|
127
|
+
'https://github.com/goodvids56/galup/issues')
|
|
128
|
+
traceback.print_exc()
|
|
129
|
+
print('\033[0m') # End the red color text
|
|
130
|
+
sys.exit(1)
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
if __name__ == '__main__':
|
|
134
|
+
main()
|