bandcampsync 0.2.0__tar.gz → 0.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {bandcampsync-0.2.0/bandcampsync.egg-info → bandcampsync-0.2.2}/PKG-INFO +1 -1
- {bandcampsync-0.2.0 → bandcampsync-0.2.2}/bandcampsync/__init__.py +24 -26
- {bandcampsync-0.2.0 → bandcampsync-0.2.2}/bandcampsync/bandcamp.py +68 -33
- {bandcampsync-0.2.0 → bandcampsync-0.2.2}/bandcampsync/config.py +1 -1
- {bandcampsync-0.2.0 → bandcampsync-0.2.2}/bandcampsync/download.py +15 -3
- {bandcampsync-0.2.0 → bandcampsync-0.2.2}/bandcampsync/media.py +6 -8
- {bandcampsync-0.2.0 → bandcampsync-0.2.2/bandcampsync.egg-info}/PKG-INFO +1 -1
- {bandcampsync-0.2.0 → bandcampsync-0.2.2}/bin/bandcampsync-service +9 -0
- {bandcampsync-0.2.0 → bandcampsync-0.2.2}/setup.py +1 -1
- {bandcampsync-0.2.0 → bandcampsync-0.2.2}/LICENSE +0 -0
- {bandcampsync-0.2.0 → bandcampsync-0.2.2}/MANIFEST.in +0 -0
- {bandcampsync-0.2.0 → bandcampsync-0.2.2}/README.md +0 -0
- {bandcampsync-0.2.0 → bandcampsync-0.2.2}/bandcampsync/logger.py +0 -0
- {bandcampsync-0.2.0 → bandcampsync-0.2.2}/bandcampsync.egg-info/SOURCES.txt +0 -0
- {bandcampsync-0.2.0 → bandcampsync-0.2.2}/bandcampsync.egg-info/dependency_links.txt +0 -0
- {bandcampsync-0.2.0 → bandcampsync-0.2.2}/bandcampsync.egg-info/requires.txt +0 -0
- {bandcampsync-0.2.0 → bandcampsync-0.2.2}/bandcampsync.egg-info/top_level.txt +0 -0
- {bandcampsync-0.2.0 → bandcampsync-0.2.2}/bin/bandcampsync +0 -0
- {bandcampsync-0.2.0 → bandcampsync-0.2.2}/cookies.txt +0 -0
- {bandcampsync-0.2.0 → bandcampsync-0.2.2}/requirements.txt +0 -0
- {bandcampsync-0.2.0 → bandcampsync-0.2.2}/setup.cfg +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: bandcampsync
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.2
|
|
4
4
|
Summary: A Python module and script to synchronise media purchased on bandcamp.com with a local directory.
|
|
5
5
|
Home-page: https://github.com/meeb/bandcampsync
|
|
6
6
|
Author: https://github.com/meeb
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import json
|
|
1
2
|
from pathlib import Path
|
|
2
3
|
from tempfile import NamedTemporaryFile, TemporaryDirectory
|
|
3
4
|
from .config import VERSION as version
|
|
@@ -5,7 +6,8 @@ from .logger import get_logger
|
|
|
5
6
|
from .bandcamp import Bandcamp
|
|
6
7
|
from .media import LocalMedia
|
|
7
8
|
from .download import (download_file, unzip_file, move_file, copy_file,
|
|
8
|
-
mask_sig, is_zip_file,
|
|
9
|
+
mask_sig, is_zip_file, DownloadInvalidContentType,
|
|
10
|
+
DownloadBadStatusCode)
|
|
9
11
|
|
|
10
12
|
|
|
11
13
|
log = logger.get_logger('sync')
|
|
@@ -16,31 +18,26 @@ def do_sync(cookies_path, cookies, dir_path, media_format):
|
|
|
16
18
|
bandcamp = Bandcamp(cookies=cookies)
|
|
17
19
|
bandcamp.verify_authentication()
|
|
18
20
|
bandcamp.load_purchases()
|
|
19
|
-
for
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
slug = purchase['url_hints']['slug']
|
|
24
|
-
item_type = purchase['item_type']
|
|
25
|
-
if local_media.is_locally_downloaded(item_id):
|
|
26
|
-
log.info(f'Already locally downloaded, skipping: {band_name} / {title} (id:{item_id})')
|
|
21
|
+
for item in bandcamp.purchases:
|
|
22
|
+
if local_media.is_locally_downloaded(item.item_id):
|
|
23
|
+
log.info(f'Already locally downloaded, skipping: {item.band_name} / {item.item_title} '
|
|
24
|
+
f'(id:{item.item_id})')
|
|
27
25
|
continue
|
|
28
26
|
else:
|
|
29
|
-
log.info(f'New media item, downloading: {band_name} / {
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
local_path.mkdir(parents=True)
|
|
37
|
-
with NamedTemporaryFile(mode='w+b', delete=True, suffix='.zip') as temp_file:
|
|
38
|
-
log.info(f'Downloading item {band_name} / {title} (id:{item_id}) '
|
|
39
|
-
f'from {mask_sig(download_url)} to {temp_file.name}')
|
|
27
|
+
log.info(f'New media item, downloading: {item.band_name} / {item.item_title} '
|
|
28
|
+
f'(id:{item.item_id}) in "{media_format}"')
|
|
29
|
+
local_path = local_media.get_path_for_purchase(item)
|
|
30
|
+
download_file_url = bandcamp.get_download_file_url(item, encoding=media_format)
|
|
31
|
+
with NamedTemporaryFile(mode='w+b', delete=True) as temp_file:
|
|
32
|
+
log.info(f'Downloading item {item.band_name} / {item.item_title} (id:{item.item_id}) '
|
|
33
|
+
f'from {mask_sig(download_file_url)} to {temp_file.name}')
|
|
40
34
|
try:
|
|
41
|
-
download_file(
|
|
42
|
-
except
|
|
43
|
-
log.error(f'Download
|
|
35
|
+
download_file(download_file_url, temp_file)
|
|
36
|
+
except DownloadBadStatusCode as e:
|
|
37
|
+
log.error(f'Download attempt returned an unexpected status code ({e}), skipping')
|
|
38
|
+
continue
|
|
39
|
+
except DownloadInvalidContentType as e:
|
|
40
|
+
log.error(f'Download attempt returned an unexpected Content-Type ({e}), skipping')
|
|
44
41
|
continue
|
|
45
42
|
temp_file.seek(0)
|
|
46
43
|
temp_file_path = Path(temp_file.name)
|
|
@@ -54,13 +51,14 @@ def do_sync(cookies_path, cookies, dir_path, media_format):
|
|
|
54
51
|
log.info(f'Moving extracted file: "{file_path}" to "{file_dest}"')
|
|
55
52
|
move_file(file_path, file_dest)
|
|
56
53
|
local_media.write_bandcamp_id(item_id, local_path)
|
|
57
|
-
elif item_type == 'track':
|
|
54
|
+
elif item.item_type == 'track':
|
|
58
55
|
log.info(f'Downloaded file is not a single track: {temp_file_path}')
|
|
56
|
+
slug = item.url_hints.get('slug', item.item_title)
|
|
59
57
|
file_dest = local_path / f'{slug}.{media_format}'
|
|
60
58
|
log.info(f'Copying single file: "{temp_file_path}" to "{file_dest}"')
|
|
61
59
|
copy_file(temp_file_path, file_dest)
|
|
62
|
-
local_media.write_bandcamp_id(
|
|
60
|
+
local_media.write_bandcamp_id(item, local_path)
|
|
63
61
|
else:
|
|
64
|
-
log.error(f'Downloaded file for {band_name} / {
|
|
62
|
+
log.error(f'Downloaded file for {item.band_name} / {item.item_title} (id:{item.item_id}) '
|
|
65
63
|
f'is not a zip archive or a single track, skipping')
|
|
66
64
|
return True
|
|
@@ -95,7 +95,32 @@ class Bandcamp:
|
|
|
95
95
|
else:
|
|
96
96
|
return BeautifulSoup(response.text, 'html.parser')
|
|
97
97
|
|
|
98
|
-
def
|
|
98
|
+
def _get_redirect_url(self, url):
|
|
99
|
+
"""
|
|
100
|
+
Requests (HTTP GET) a URL assuming it will return a redirect (301 or
|
|
101
|
+
303) and return the URL you are being redirected to.
|
|
102
|
+
"""
|
|
103
|
+
headers = {'User-Agent': USER_AGENT}
|
|
104
|
+
try:
|
|
105
|
+
log.debug(f'Making GET (redirect check) request to {url}')
|
|
106
|
+
response = requests.get(
|
|
107
|
+
url,
|
|
108
|
+
headers=headers,
|
|
109
|
+
cookies=self._plain_cookies(),
|
|
110
|
+
allow_redirects=False
|
|
111
|
+
)
|
|
112
|
+
except Exception as e:
|
|
113
|
+
raise BandcampError(f'Failed to make HTTP request to {url}: {e}') from e
|
|
114
|
+
if response.status_code in (301, 303):
|
|
115
|
+
try:
|
|
116
|
+
return response.headers['Location']
|
|
117
|
+
except KeyError as e:
|
|
118
|
+
raise BandcampError(f'Redirect URL {url} returned a 301 or 303 but no Location header')
|
|
119
|
+
else:
|
|
120
|
+
# Not a redirect
|
|
121
|
+
return False
|
|
122
|
+
|
|
123
|
+
def _extract_pagedata_from_soup(self, soup):
|
|
99
124
|
pagedata_tag = soup.find('div', id='pagedata')
|
|
100
125
|
if not pagedata_tag:
|
|
101
126
|
raise BandcampError(f'Failed to locate <div id="pagedata"> in index HTML, this may '
|
|
@@ -111,6 +136,13 @@ class Bandcamp:
|
|
|
111
136
|
except Exception as e:
|
|
112
137
|
raise BandcampError(f'Failed to parse pagedata as JSON: {e}') from e
|
|
113
138
|
|
|
139
|
+
def _extract_pagedata_from_html(self, html):
|
|
140
|
+
"""
|
|
141
|
+
Wrapper for _extract_pagedata_from_soup() that can accept HTML rather than a bs4 soup.
|
|
142
|
+
"""
|
|
143
|
+
soup = BeautifulSoup(html, 'html.parser')
|
|
144
|
+
return self._extract_pagedata_from_soup(soup)
|
|
145
|
+
|
|
114
146
|
def verify_authentication(self):
|
|
115
147
|
"""
|
|
116
148
|
Loads the initial account and session data from a request to the index page
|
|
@@ -119,7 +151,7 @@ class Bandcamp:
|
|
|
119
151
|
"""
|
|
120
152
|
url = self._construct_url('index')
|
|
121
153
|
soup = self._request('get', url)
|
|
122
|
-
pagedata = self.
|
|
154
|
+
pagedata = self._extract_pagedata_from_soup(soup)
|
|
123
155
|
try:
|
|
124
156
|
identities = pagedata['identities']
|
|
125
157
|
except KeyError as e:
|
|
@@ -177,43 +209,30 @@ class Bandcamp:
|
|
|
177
209
|
redownload_urls = data['redownload_urls']
|
|
178
210
|
except KeyError as e:
|
|
179
211
|
raise BandcampError(f'Failed to extract redownload_urls from collection results page')
|
|
180
|
-
for
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
token = item['token']
|
|
185
|
-
log.info(f'Found item: {band_name} / {title} (id:{item_id})')
|
|
186
|
-
download_url_type = item['sale_item_type']
|
|
187
|
-
download_url_id = item['sale_item_id']
|
|
188
|
-
download_url_key = f'{download_url_type}{download_url_id}'
|
|
212
|
+
for item_data in items:
|
|
213
|
+
sale_item_type = item_data['sale_item_type']
|
|
214
|
+
sale_item_id = item_data['sale_item_id']
|
|
215
|
+
download_url_key = f'{sale_item_type}{sale_item_id}'
|
|
189
216
|
try:
|
|
190
217
|
download_url = redownload_urls[download_url_key]
|
|
191
218
|
except KeyError:
|
|
192
|
-
log.error(f'Failed to locate download URL for {band_name} / {title} '
|
|
219
|
+
log.error(f'Failed to locate download URL for {item.band_name} / {item.title} '
|
|
193
220
|
f'(key:{download_url_key}), skipping item...')
|
|
194
221
|
continue
|
|
195
|
-
|
|
222
|
+
item_data['download_url'] = download_url
|
|
223
|
+
item = BandcampItem(item_data)
|
|
224
|
+
token = item.token
|
|
225
|
+
log.info(f'Found item: {item.band_name} / {item.item_title} (id:{item.item_id})')
|
|
196
226
|
self.purchases.append(item)
|
|
197
227
|
log.info(f'Loaded {len(self.purchases)} purchases')
|
|
198
|
-
#print(json.dumps(self.purchases[0], indent=4, sort_keys=True))
|
|
199
228
|
return True
|
|
200
|
-
|
|
201
|
-
def
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
item_id = purchase['item_id']
|
|
208
|
-
item_type = purchase['item_type']
|
|
209
|
-
if item_type not in ('album', 'track'):
|
|
210
|
-
band_name = item['band_name']
|
|
211
|
-
title = item['item_title']
|
|
212
|
-
log.error(f'Item {item_id} ({band_name} / {title}) has an unknown item type: {item_type}, skipping')
|
|
213
|
-
return False
|
|
214
|
-
# get the pagedata
|
|
215
|
-
soup = self._request('get', download_url)
|
|
216
|
-
pagedata = self._extract_pagedata(soup)
|
|
229
|
+
|
|
230
|
+
def get_download_file_url(self, item, encoding='flac'):
|
|
231
|
+
soup = self._request('get', item.download_url)
|
|
232
|
+
pagedata = self._extract_pagedata_from_soup(soup)
|
|
233
|
+
download_url = None
|
|
234
|
+
if not pagedata:
|
|
235
|
+
raise ValueError(f'Either "url" or "pagedata" must be supplied')
|
|
217
236
|
try:
|
|
218
237
|
digital_items = pagedata['digital_items']
|
|
219
238
|
except KeyError as e:
|
|
@@ -225,7 +244,7 @@ class Bandcamp:
|
|
|
225
244
|
except KeyError as e:
|
|
226
245
|
raise BandcampError(f'Failed to parse pagedata JSON, does not contain an '
|
|
227
246
|
f'"digital_items[].art_id" key') from e
|
|
228
|
-
if digital_item_id == item_id:
|
|
247
|
+
if digital_item_id == item.item_id:
|
|
229
248
|
try:
|
|
230
249
|
downloads = digital_item['downloads']
|
|
231
250
|
except KeyError as e:
|
|
@@ -238,8 +257,24 @@ class Bandcamp:
|
|
|
238
257
|
raise BandcampError(f'Download formats does not contain requested encoding: {encoding} '
|
|
239
258
|
f'(available encodings: {encodings})') from e
|
|
240
259
|
try:
|
|
241
|
-
|
|
260
|
+
download_url = download_format['url']
|
|
242
261
|
except KeyError as e:
|
|
243
262
|
raise BandcampError(f'Failed to parse pagedata JSON, does not contain an '
|
|
244
263
|
f'"digital_items.downloads.[encoding].url" key') from e
|
|
264
|
+
return download_url
|
|
245
265
|
return False
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
class BandcampItem:
|
|
269
|
+
|
|
270
|
+
def __init__(self, data):
|
|
271
|
+
self._data = data
|
|
272
|
+
|
|
273
|
+
def __repr__(self):
|
|
274
|
+
return json.dumps(self._data, indent=4, sort_keys=True)
|
|
275
|
+
|
|
276
|
+
def __getattr__(self, key):
|
|
277
|
+
try:
|
|
278
|
+
return self._data[key]
|
|
279
|
+
except KeyError as e:
|
|
280
|
+
raise KeyError(f'BandcampItem value "{key}" does not exist') from e
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
VERSION = '0.2.
|
|
1
|
+
VERSION = '0.2.2'
|
|
2
2
|
USER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/116.0'
|
|
@@ -19,24 +19,36 @@ def mask_sig(url):
|
|
|
19
19
|
return '&'.join(url_parts)
|
|
20
20
|
|
|
21
21
|
|
|
22
|
-
class
|
|
22
|
+
class DownloadBadStatusCode(ValueError):
|
|
23
|
+
pass
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class DownloadInvalidContentType(ValueError):
|
|
23
27
|
pass
|
|
24
28
|
|
|
25
29
|
|
|
26
30
|
def download_file(url, target, mode='wb', chunk_size=8192, logevery=10, disallow_content_type='text/html'):
|
|
31
|
+
"""
|
|
32
|
+
Attempts to stream a download to an open target file handle in chunks. If the
|
|
33
|
+
request returns a disallowed content type then return a failed state with the
|
|
34
|
+
response content.
|
|
35
|
+
"""
|
|
27
36
|
text = True if 't' in mode else False
|
|
28
37
|
data_streamed = 0
|
|
29
38
|
last_log = 0
|
|
30
39
|
headers = {'User-Agent': USER_AGENT}
|
|
31
40
|
with requests.get(url, stream=True, headers=headers) as r:
|
|
32
41
|
r.raise_for_status()
|
|
42
|
+
if r.status_code != 200:
|
|
43
|
+
raise DownloadBadStatusCode(f'Got non-200 status code: {r.status_code}')
|
|
33
44
|
try:
|
|
34
45
|
content_type = r.headers.get('Content-Type', '')
|
|
35
46
|
except (ValueError, KeyError):
|
|
36
47
|
content_type = ''
|
|
37
48
|
content_type_parts = content_type.split(';')
|
|
38
|
-
|
|
39
|
-
|
|
49
|
+
major_content_type = content_type_parts[0].strip()
|
|
50
|
+
if major_content_type == disallow_content_type:
|
|
51
|
+
raise DownloadInvalidContentType(f'Invalid content type: {major_content_type}')
|
|
40
52
|
try:
|
|
41
53
|
content_length = int(r.headers.get('Content-Length', '0'))
|
|
42
54
|
except (ValueError, KeyError):
|
|
@@ -44,7 +44,7 @@ class LocalMedia:
|
|
|
44
44
|
try:
|
|
45
45
|
return int(item_id)
|
|
46
46
|
except Exception as e:
|
|
47
|
-
raise ValueError(f'Failed to cast item ID "{item_id}" as an int: {e}') from e
|
|
47
|
+
raise ValueError(f'Failed to cast item ID from {filepath} "{item_id}" as an int: {e}') from e
|
|
48
48
|
|
|
49
49
|
def is_locally_downloaded(self, item_id):
|
|
50
50
|
return item_id in self.media
|
|
@@ -52,14 +52,12 @@ class LocalMedia:
|
|
|
52
52
|
def is_dir(self, path):
|
|
53
53
|
return path in self.dirs
|
|
54
54
|
|
|
55
|
-
def get_path_for_purchase(self,
|
|
56
|
-
band_name
|
|
57
|
-
title = purchase['item_title']
|
|
58
|
-
return self.media_dir / band_name / title
|
|
55
|
+
def get_path_for_purchase(self, item):
|
|
56
|
+
return self.media_dir / item.band_name / item.item_title
|
|
59
57
|
|
|
60
|
-
def write_bandcamp_id(self,
|
|
58
|
+
def write_bandcamp_id(self, item, dirpath):
|
|
61
59
|
outfile = dirpath / self.ITEM_INDEX_FILENAME
|
|
62
|
-
log.info(f'Writing bandcamp item id:{item_id} to: {outfile}')
|
|
60
|
+
log.info(f'Writing bandcamp item id:{item.item_id} to: {outfile}')
|
|
63
61
|
with open(outfile, 'wt') as f:
|
|
64
|
-
f.write(f'{item_id}\n')
|
|
62
|
+
f.write(f'{item.item_id}\n')
|
|
65
63
|
return True
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: bandcampsync
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.2
|
|
4
4
|
Summary: A Python module and script to synchronise media purchased on bandcamp.com with a local directory.
|
|
5
5
|
Home-page: https://github.com/meeb/bandcampsync
|
|
6
6
|
Author: https://github.com/meeb
|
|
@@ -31,6 +31,7 @@ if __name__ == '__main__':
|
|
|
31
31
|
dir_path_env = os.getenv('DIRECTORY', '/downloads')
|
|
32
32
|
media_format_env = os.getenv('FORMAT', 'flac')
|
|
33
33
|
run_daily_at_env = os.getenv('RUN_DAILY_AT', '3')
|
|
34
|
+
exit_after_run_env = os.getenv('EXIT_AFTER_RUN', '0')
|
|
34
35
|
cookies_path = Path(cookies_path_env).resolve()
|
|
35
36
|
dir_path = Path(dir_path_env).resolve()
|
|
36
37
|
if not cookies_path.is_file():
|
|
@@ -51,6 +52,11 @@ if __name__ == '__main__':
|
|
|
51
52
|
raise ValueError(f'Invalid RUN_DAILY_AT, got: {run_daily_at}') from e
|
|
52
53
|
if not 0 <= run_daily_at <= 23:
|
|
53
54
|
raise ValueError(f'Invalid RUN_DAILY_AT, must be between 0 and 23, got: {run_daily_at}')
|
|
55
|
+
try:
|
|
56
|
+
exit_after_run = int(exit_after_run_env)
|
|
57
|
+
except (ValueError, TypeError):
|
|
58
|
+
exit_after_run = 0
|
|
59
|
+
exit_after_run = True if exit_after_run else False
|
|
54
60
|
time_now = datetime.now(tz).replace(microsecond=0)
|
|
55
61
|
log.info(f'Time now in {tz}: {time_now}')
|
|
56
62
|
log.info(f'Running an initial one-off synchronisation immediately')
|
|
@@ -64,6 +70,9 @@ if __name__ == '__main__':
|
|
|
64
70
|
time_tomorrow = time_now + timedelta(days=1)
|
|
65
71
|
time_tomorrow = time_tomorrow.replace(hour=run_daily_at, minute=0, second=0, microsecond=0)
|
|
66
72
|
next_sleep = int((time_tomorrow - time_now).total_seconds() + random_delay)
|
|
73
|
+
if exit_after_run:
|
|
74
|
+
log.info(f'Exiting after run')
|
|
75
|
+
break
|
|
67
76
|
log.info(f'Scheduling next run for {time_tomorrow} + {random_delay} second random offset')
|
|
68
77
|
log.info(f'Sleeping for {next_sleep} seconds')
|
|
69
78
|
sleep(next_sleep)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|