bandcampsync 0.6.0__py3-none-any.whl

bandcampsync/__init__.py

@@ -0,0 +1,191 @@
+from pathlib import Path
+from tempfile import NamedTemporaryFile, TemporaryDirectory
+from .config import VERSION as version
+from .logger import get_logger
+from .bandcamp import Bandcamp, BandcampError
+from .ignores import Ignores
+from .media import LocalMedia
+from .notify import NotifyURL
+from .download import (
+    download_file,
+    unzip_file,
+    move_file,
+    copy_file,
+    mask_sig,
+    is_zip_file,
+    DownloadInvalidContentType,
+    DownloadBadStatusCode,
+)
+
+
+__all__ = ["version", "do_sync"]
+
+
+log = get_logger("sync")
+
+
+def do_sync(
+    cookies_path,
+    cookies,
+    dir_path,
+    media_format,
+    temp_dir_root,
+    ign_file_path,
+    ign_patterns,
+    notify_url,
+):
+    ignores = Ignores(ign_file_path=ign_file_path, ign_patterns=ign_patterns)
+    local_media = LocalMedia(media_dir=dir_path)
+    bandcamp = Bandcamp(cookies=cookies)
+    bandcamp.verify_authentication()
+    bandcamp.load_purchases()
+    new_items_downloaded = False
+    show_id_file_warning = False
+    if notify_url:
+        notifier = NotifyURL(notify_url)
+    else:
+        notifier = None
+
+    for item in bandcamp.purchases:
+        local_path = local_media.get_path_for_purchase(item)
+
+        # Check if any ignore pattern matches the band name
+        if ignores.is_ignored(item):
+            if not show_id_file_warning and local_media.is_locally_downloaded(
+                item, local_path
+            ):
+                show_id_file_warning = True
+            continue
+
+        if item.is_preorder:
+            log.info(
+                f'Item is a preorder, skipping: "{item.band_name} / {item.item_title}" '
+                f"(id:{item.item_id})"
+            )
+            continue
+        elif local_media.is_locally_downloaded(item, local_path):
+            log.info(
+                f'Already locally downloaded, skipping: "{item.band_name} / {item.item_title}" '
+                f"(id:{item.item_id})"
+            )
+            continue
+        else:
+            log.info(
+                f'New media item, will download: "{item.band_name} / {item.item_title}" '
+                f'(id:{item.item_id}) in "{media_format}"'
+            )
+            try:
+                local_path.mkdir(parents=True, exist_ok=True)
+            except OSError as e:
+                log.error(
+                    f"Failed to create directory: {local_path} ({e}), skipping purchase..."
+                )
+                continue
+            try:
+                initial_download_url = bandcamp.get_download_file_url(
+                    item, encoding=media_format
+                )
+            except BandcampError as e:
+                log.error(
+                    f'Failed to locate download URL for media item "{item.band_name} / {item.item_title}" '
+                    f"(id:{item.item_id}), unable to download release ({e}), skipping"
+                )
+                continue
+            download_file_url = bandcamp.check_download_stat(item, initial_download_url)
+            with NamedTemporaryFile(
+                mode="w+b", delete=True, dir=temp_dir_root
+            ) as temp_file:
+                log.info(
+                    f'Downloading item "{item.band_name} / {item.item_title}" (id:{item.item_id}) '
+                    f"from {mask_sig(download_file_url)} to {temp_file.name}"
+                )
+                try:
+                    download_file(download_file_url, temp_file)
+                except DownloadBadStatusCode as e:
+                    log.error(
+                        f"Download attempt returned an unexpected status code ({e}), skipping"
+                    )
+                    continue
+                except DownloadInvalidContentType as e:
+                    log.error(
+                        f"Download attempt returned an unexpected content type ({e}), skipping"
+                    )
+                    continue
+                temp_file.seek(0)
+                temp_file_path = Path(temp_file.name)
+                if is_zip_file(temp_file_path):
+                    with TemporaryDirectory(dir=temp_dir_root) as temp_dir:
+                        log.info(
+                            f'Decompressing downloaded zip "{temp_file.name}" to "{temp_dir}"'
+                        )
+                        unzip_file(temp_file.name, temp_dir)
+                        temp_path = Path(temp_dir)
+                        for file_path in temp_path.iterdir():
+                            file_dest = local_media.get_path_for_file(
+                                local_path, file_path.name
+                            )
+                            log.info(
+                                f'Moving extracted file: "{file_path}" to "{file_dest}"'
+                            )
+                            try:
+                                move_file(file_path, file_dest)
+                            except Exception as e:
+                                log.error(
+                                    f"Failed to move {file_path} to {file_dest}: {e}"
+                                )
+                elif item.item_type == "track":
+                    slug = item.item_title
+                    if item.url_hints and isinstance(item.url_hints, dict):
+                        slug = item.url_hints.get("slug", item.item_title)
+                    format_extension = local_media.clean_format(media_format)
+                    file_dest = local_media.get_path_for_file(
+                        local_path, f"{slug}.{format_extension}"
+                    )
+                    log.info(
+                        f'Copying single track: "{temp_file_path}" to "{file_dest}"'
+                    )
+                    try:
+                        copy_file(temp_file_path, file_dest)
+                    except Exception as e:
+                        log.error(f"Failed to copy {file_path} to {file_dest}: {e}")
+                else:
+                    log.error(
+                        f'Downloaded file for "{item.band_name} / {item.item_title}" (id:{item.item_id}) '
+                        f'at "{temp_file_path}" is not a zip archive or a single track, skipping'
+                    )
+                    continue
+
+                if ign_file_path:
+                    # We assume that if you use an ignore file once, you'll
+                    # keep using it forever (e.g. Docker).
+                    # In case you don't, you'll get warnings for missing id file
+                    # on the items downloaded in the current session.
+                    ignores.add(item)
+                else:
+                    local_media.write_bandcamp_id(item, local_path)
+                new_items_downloaded = True
+
+    if new_items_downloaded:
+        log.info("New media items downloaded")
+        if notifier:
+            notifier.notify()
+
+    if show_id_file_warning:
+        log.warning(
+            f"The {ign_file_path} file is tracking already downloaded items, "
+            f"but some directories are using bandcamp_item_id.txt files. "
+            f"If you want to get migrate from ID files to using the {ign_file_path} file, "
+            f"run the following script inside the downloads directory, then append the "
+            f"content of the new ignores.txt file to the ignores file in your "
+            f"config directory:\n"
+            '  find . -name "bandcamp_item_id.txt" \\\n'
+            "    | while read -r id_file\n"
+            "      do \\\n"
+            '        comment="$(echo "$id_file" \\\n'
+            '            | sed -r -e "s%./([^/]+)/([^/]+)/bandcamp_item_id.txt%\\1 / \\2%")"\n'
+            '        echo "$(cat "$id_file")  # $comment"\n'
+            '        rm "$id_file"\n'
+            "      done >> ignores.txt\n"
+        )
+
+    return True

bandcampsync/bandcamp.py

@@ -0,0 +1,364 @@
+import json
+import re
+from time import time
+from http.cookies import SimpleCookie
+from html import unescape as html_unescape
+from urllib.parse import urlsplit, urlunsplit
+from bs4 import BeautifulSoup
+from curl_cffi import requests
+from .download import mask_sig
+from .logger import get_logger
+
+
+log = get_logger("bandcamp")
+
+
+class BandcampError(ValueError):
+    pass
+
+
+class Bandcamp:
+    BASE_PROTO = "https"
+    BASE_DOMAIN = "bandcamp.com"
+    URLS = {
+        "index": "/",
+        "collection_items": "/api/fancollection/1/collection_items",
+    }
+
+    def __init__(self, cookies=""):
+        self.is_authenticated = False
+        self.user_id = 0
+        self.user_verified = False
+        self.cookies = None
+        self.purchases = []
+        self.load_cookies(cookies)
+        identity = self.cookies.get("identity")
+        if not identity:
+            raise BandcampError(
+                "Cookie data does not contain an identity value, make sure your "
+                "cookies.txt file is valid and you copied it from an "
+                "authenticated browser"
+            )
+        identity_snip = identity.value[:20]
+        log.info(f"Located Bandcamp identity in cookies: {identity_snip}...")
+        # Create a requests session and map our SimpleCookie to it
+        self.session = requests.Session(impersonate="chrome")
+        for cookie_name, morsel in self.cookies.items():
+            self.session.cookies.set(cookie_name, morsel.value)
+
+    def load_cookies(self, cookies_str):
+        self.cookies = SimpleCookie()
+        try:
+            self.cookies.load(cookies_str)
+        except Exception as e:
+            raise BandcampError(f"Failed to parse cookies string: {e}") from e
+        if len(self.cookies) == 0:
+            # Failed to load any cookies, attempt to parse the cookies string as a Netscape cookies export
+            lines = cookies_str.strip().split("\n")
+            for line in lines:
+                if line.startswith("#"):
+                    continue
+                line = line.strip()
+                if not line:
+                    continue
+                parts = line.split("\t")
+                if len(parts) == 7:
+                    domain, tailmatch, path, secure, expires, name, value = parts
+                    cookie_string = f"{name.strip()}={value.strip()}; Domain={domain.strip()}; Path={path.strip()}"
+                    if secure == "TRUE":
+                        cookie_string += "; Secure"
+                    self.cookies.load(cookie_string)
+        return True
+
+    @property
+    def cookies_str(self):
+        return self.cookies.output(header="").strip().replace("\r\n", ";")
+
+    def refresh_cookes(self, file_path):
+        log.info(f"Refreshing cookies: {file_path}")
+        with open(file_path, "wt") as f:
+            f.write(self.cookies_str)
+
+    def _construct_url(self, url_name):
+        if url_name not in self.URLS:
+            raise BandcampError(f"URL name is unknown: {url_name}")
+        return urlunsplit(
+            (self.BASE_PROTO, self.BASE_DOMAIN, self.URLS[url_name], "", "")
+        )
+
+    def _plain_cookies(self):
+        cookies = {}
+        for cookie_name, cookie_value in self.cookies.items():
+            cookies[cookie_value.key] = cookie_value.value
+        return cookies
+
+    def _request(
+        self, method, url, data=None, json_data=None, is_json=False, as_raw=False
+    ):
+        try:
+            # The debug logs do not mask the URL which may be a security issue if you run
+            # with level=logging.DEBUG
+            log.debug(f"Making {method} request to {url}")
+            response = self.session.request(
+                method,
+                url,
+                cookies=self._plain_cookies(),
+                data=data,
+                json=json_data,
+            )
+        except Exception as e:
+            raise BandcampError(
+                f"Failed to make HTTP request to {mask_sig(url)}: {e}"
+            ) from e
+        if response.status_code != 200:
+            raise BandcampError(
+                f"Failed to make HTTP request to {mask_sig(url)}: "
+                f"unknown status code response: {response.status_code}"
+            )
+        if as_raw:
+            return response.text
+        elif is_json:
+            return json.loads(response.text)
+        else:
+            return BeautifulSoup(response.text, "html.parser")
+
+    def _extract_pagedata_from_soup(self, soup, id_name="pagedata"):
+        pagedata_tag = soup.find("div", id=id_name)
+        if not pagedata_tag:
+            raise BandcampError(
+                'Failed to locate <div id="HomepageApp"> in index HTML, this may '
+                "be an authentication issue or it may be that bandcamp.com has "
+                "updated their website and this tool needs to be updated."
+            )
+        encoded_pagedata = pagedata_tag.attrs.get("data-blob")
+        if not encoded_pagedata:
+            raise BandcampError(
+                "Failed to extract page data, check your cookies are from an ",
+                "authenticated session",
+            )
+        pagedata_str = html_unescape(encoded_pagedata)
+        try:
+            return json.loads(pagedata_str)
+        except Exception as e:
+            raise BandcampError(f"Failed to parse pagedata as JSON: {e}") from e
+
+    def _extract_pagedata_from_html(self, html):
+        """
+        Wrapper for _extract_pagedata_from_soup() that can accept HTML rather than a bs4 soup.
+        """
+        soup = BeautifulSoup(html, "html.parser")
+        return self._extract_pagedata_from_soup(soup)
+
+    def _get_js_stat_url(self, body, download_url):
+        """
+        Checks the "stat" download URL body, which is in JavaScript, for
+        either the OK response or a new updated download URL.
+        """
+        body = body.strip()
+        if body == "var _statDL_result = { result: 'ok'};":
+            # Download is OK, original download URL will work
+            return download_url
+        # Attempt to find the updated download_url in the JavaScript with a hacky regex
+        pattern = re.compile('"([^"]+)":"([^"]+)"')
+        for k, v in pattern.findall(body):
+            if k == "download_url":
+                return v
+        # Fallback to the original download URL
+        return download_url
+
+    def verify_authentication(self):
+        """
+        Loads the initial account and session data from a request to the index page
+        of bandcamp.com. When properly authenticated an HTML data attribute is present
+        that contains account information in an encoded form.
+        """
+        url = self._construct_url("index")
+        soup = self._request("get", url)
+        pagedata = self._extract_pagedata_from_soup(soup, id_name="HomepageApp")
+        try:
+            pagecontext = pagedata["pageContext"]
+        except KeyError as e:
+            raise BandcampError(
+                'Failed to parse pagedata JSON, does not contain an "pageContext" key'
+            ) from e
+        try:
+            identity = pagecontext["identity"]
+        except KeyError as e:
+            raise BandcampError(
+                'Failed to parse pagecontext JSON, does not contain an "identity" key'
+            ) from e
+        if not isinstance(identity, dict):
+            raise BandcampError(
+                'Failed to parse pagedata JSON, "identity" is not '
+                "a dictionary. Check your cookies.txt file is valid "
+                "and up to date"
+            )
+        try:
+            self.user_id = identity["fanId"]
+            self.user_verified = identity["isFanVerified"]
+        except (KeyError, TypeError) as e:
+            raise BandcampError(
+                f'Failed to parse pagedata JSON, "identity.fan" seems invalid: {identity}'
+            ) from e
+        self.is_authenticated = self.user_id > 0
+        log.info(
+            f"Loaded page data, session is authenticated for user id: {self.user_id})"
+        )
+        return True
+
+    def load_purchases(self):
+        """
+        Loads all purchases on the authenticated account and returns a list of
+        purchase data. Each purchase is a dict of data.
+        """
+        if not self.is_authenticated:
+            raise BandcampError(
+                "Authentication not verified, call load_pagedata() first"
+            )
+        log.info(f"Loading purchases for user id: {self.user_id}")
+        self.purchases = []
+        now = int(time())
+        page_ts = 0
+        token = f"{now}:{page_ts}:a::"
+        per_page = 100
+        while True:
+            log.info(f"Requesting {per_page} purchases using token {token}")
+            data = {
+                "fan_id": self.user_id,
+                "count": per_page,
+                "older_than_token": token,
+            }
+            url = self._construct_url("collection_items")
+            data = self._request("POST", url, json_data=data, is_json=True)
+            try:
+                items = data["items"]
+            except KeyError:
+                raise BandcampError(
+                    "Failed to extract items from collection results page"
+                )
+            if not items:
+                log.info("Reached end of items")
+                break
+            try:
+                redownload_urls = data["redownload_urls"]
+            except KeyError:
+                raise BandcampError(
+                    "Failed to extract redownload_urls from collection results page"
+                )
+            for item_data in items:
+                try:
+                    band_name = item_data["band_name"]
+                except KeyError:
+                    log.error(
+                        "Failed to locate band name in item metadata, skipping item..."
+                    )
+                    continue
+                try:
+                    title = item_data["album_title"]
+                except KeyError:
+                    log.error(
+                        f'Failed to locate title in item metadata (possibly a subscription?) for "{band_name}", skipping item...'
+                    )
+                    continue
+                sale_item_type = item_data["sale_item_type"]
+                sale_item_id = item_data["sale_item_id"]
+                download_url_key = f"{sale_item_type}{sale_item_id}"
+                try:
+                    download_url = redownload_urls[download_url_key]
+                except KeyError:
+                    log.error(
+                        f"Failed to locate download URL for {band_name} / {title} "
+                        f"(key:{download_url_key}), skipping item..."
+                    )
+                    continue
+                item_data["download_url"] = download_url
+                item = BandcampItem(item_data)
+                token = item.token
+                log.info(f"Found item: {band_name} / {title} (id:{item.item_id})")
+                self.purchases.append(item)
+        log.info(f"Loaded {len(self.purchases)} purchases")
+        return True
+
+    def get_download_file_url(self, item, encoding="flac"):
+        soup = self._request("get", item.download_url)
+        pagedata = self._extract_pagedata_from_soup(soup)
+        download_url = None
+        if not pagedata:
+            raise ValueError('Either "url" or "pagedata" must be supplied')
+        try:
+            digital_items = pagedata["digital_items"]
+        except KeyError as e:
+            raise BandcampError(
+                "Failed to parse pagedata JSON, does not contain an "
+                '"digital_items" key'
+            ) from e
+        for digital_item in digital_items:
+            try:
+                digital_item_id = digital_item["item_id"]
+            except KeyError as e:
+                raise BandcampError(
+                    "Failed to parse pagedata JSON, does not contain an "
+                    '"digital_items[].art_id" key'
+                ) from e
+            if digital_item_id == item.item_id:
+                try:
+                    downloads = digital_item["downloads"]
+                except KeyError as e:
+                    raise BandcampError(
+                        "Failed to parse pagedata JSON, does not contain an "
+                        '"digital_items.downloads" key'
+                    ) from e
+                try:
+                    download_format = downloads[encoding]
+                except KeyError as e:
+                    encodings = downloads.keys()
+                    raise BandcampError(
+                        f"Download formats does not contain requested encoding: {encoding} "
+                        f"(available encodings: {encodings})"
+                    ) from e
+                try:
+                    download_url = download_format["url"]
+                except KeyError as e:
+                    raise BandcampError(
+                        "Failed to parse pagedata JSON, does not contain an "
+                        '"digital_items.downloads.[encoding].url" key'
+                    ) from e
+                return download_url
+        return False
+
+    def check_download_stat(self, item, file_download_url):
+        """
+        Constructs the download "stat" URL and verifies the state of the download.
+        If the state is OK, return the existing URL (download is OK) otherwise wait
+        for the stat to complete and return the new download URL.
+        """
+        download_url_parts = urlsplit(file_download_url)
+        path = download_url_parts.path
+        path_parts = path.split("/")
+        if path_parts[1] == "download":
+            path_parts[1] = "statdownload"
+        stat_url = urlunsplit(
+            (
+                download_url_parts.scheme,
+                download_url_parts.netloc,
+                "/".join(path_parts),
+                download_url_parts.query,
+                "",
+            )
+        )
+        body = self._request("get", stat_url, as_raw=True)
+        return self._get_js_stat_url(body, file_download_url)
+
+
+class BandcampItem:
+    def __init__(self, data):
+        self._data = data
+
+    def __repr__(self):
+        return json.dumps(self._data, indent=4, sort_keys=True)
+
+    def __getattr__(self, key):
+        try:
+            return self._data[key]
+        except KeyError as e:
+            raise KeyError(f'BandcampItem value "{key}" does not exist') from e

bandcampsync/config.py

@@ -0,0 +1,2 @@
+VERSION = "0.6.0"
+INTERNAL_USER_AGENT = f"bandcampsync/{VERSION}"

bandcampsync/download.py

@@ -0,0 +1,101 @@
+import math
+import shutil
+from zipfile import ZipFile
+from curl_cffi import requests
+from .logger import get_logger
+
+
+log = get_logger("download")
+
+
+def mask_sig(url):
+    if "&sig=" not in url:
+        return url
+    url_parts = url.split("&")
+    for i, url_part in enumerate(url_parts):
+        if url_part[:4] == "sig=":
+            url_parts[i] = "sig=[masked]"
+        elif url_part[:6] == "token=":
+            url_parts[i] = "token=[masked]"
+    return "&".join(url_parts)
+
+
+class DownloadBadStatusCode(ValueError):
+    pass
+
+
+class DownloadInvalidContentType(ValueError):
+    pass
+
+
+def download_file(
+    url,
+    target,
+    mode="wb",
+    chunk_size=8192,
+    logevery=10,
+    disallow_content_type="text/html",
+):
+    """
+    Attempts to stream a download to an open target file handle in chunks. If the
+    request returns a disallowed content type then return a failed state with the
+    response content.
+    """
+    text = True if "t" in mode else False
+    data_streamed = 0
+    last_log = 0
+    r = requests.get(url, stream=True, impersonate="chrome")
+    try:
+        # r.raise_for_status()
+        if r.status_code != 200:
+            raise DownloadBadStatusCode(f"Got non-200 status code: {r.status_code}")
+        try:
+            content_type = r.headers.get("Content-Type", "")
+        except (ValueError, KeyError):
+            content_type = ""
+        content_type_parts = content_type.split(";")
+        major_content_type = content_type_parts[0].strip()
+        if major_content_type == disallow_content_type:
+            raise DownloadInvalidContentType(
+                f"Invalid content type: {major_content_type}"
+            )
+        try:
+            content_length = int(r.headers.get("Content-Length", "0"))
+        except (ValueError, KeyError):
+            content_length = 0
+        for chunk in r.iter_content(chunk_size=chunk_size):
+            data_streamed += len(chunk)
+            if text:
+                chunk = chunk.decode()
+            target.write(chunk)
+            if content_length > 0 and logevery > 0:
+                percent_complete = math.floor((data_streamed / content_length) * 100)
+                if percent_complete % logevery == 0 and percent_complete > last_log:
+                    log.info(f"Downloading {mask_sig(url)}: {percent_complete}%")
+                    last_log = percent_complete
+    finally:
+        r.close()
+    return True
+
+
+def is_zip_file(file_path):
+    try:
+        with ZipFile(file_path) as z:
+            z.infolist()
+        return True
+    except Exception:
+        return False
+
+
+def unzip_file(decompress_from, decompress_to):
+    with ZipFile(decompress_from) as z:
+        z.extractall(decompress_to)
+    return True
+
+
+def move_file(src, dst):
+    return shutil.move(src, dst)
+
+
+def copy_file(src, dst):
+    return shutil.copyfile(src, dst)