DiscogsDataProcessorCLI 1.5.9__tar.gz → 1.6.1__tar.gz

{discogsdataprocessorcli-1.5.9 → discogsdataprocessorcli-1.6.1}/DiscogsDataProcessorCLI.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: DiscogsDataProcessorCLI
-Version: 1.5.9
+Version: 1.6.1
 Summary: A CLI to download, extract and convert Discogs data dumps
 Home-page: https://github.com/ofurkancoban/DiscogsCLI
 Author: Furkan Çoban
@@ -14,6 +14,7 @@ Description-Content-Type: text/markdown
 Requires-Dist: rich
 Requires-Dist: pandas
 Requires-Dist: typer
+Requires-Dist: requests
 
 # 🎧 Discogs CLI — Data Processor Tool 💿
 <p align="center">

{discogsdataprocessorcli-1.5.9 → discogsdataprocessorcli-1.6.1}/DiscogsDataProcessorCLI.egg-info/requires.txt

@@ -1,3 +1,4 @@
 rich
 pandas
 typer
+requests

{discogsdataprocessorcli-1.5.9 → discogsdataprocessorcli-1.6.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: DiscogsDataProcessorCLI
-Version: 1.5.9
+Version: 1.6.1
 Summary: A CLI to download, extract and convert Discogs data dumps
 Home-page: https://github.com/ofurkancoban/DiscogsCLI
 Author: Furkan Çoban
@@ -14,6 +14,7 @@ Description-Content-Type: text/markdown
 Requires-Dist: rich
 Requires-Dist: pandas
 Requires-Dist: typer
+Requires-Dist: requests
 
 # 🎧 Discogs CLI — Data Processor Tool 💿
 <p align="center">

{discogsdataprocessorcli-1.5.9 → discogsdataprocessorcli-1.6.1}/discogs/deleter.py

@@ -29,7 +29,7 @@ def delete_files():
 
     for i in selected:
         row = df.iloc[i]
-        filename = Path(row["url"]).name
+        filename = row["filename"]
         year_month = row["month"]
         data_dir = download_dir / "Datasets" / year_month
 

{discogsdataprocessorcli-1.5.9 → discogsdataprocessorcli-1.6.1}/discogs/downloader.py

@@ -4,7 +4,6 @@ import time
 import requests
 from time import sleep
 from pathlib import Path
-from urllib.parse import urlparse
 from rich.console import Console
 from rich.progress import (
     Progress, BarColumn, DownloadColumn, TransferSpeedColumn,
@@ -15,27 +14,50 @@ from datetime import datetime
 
 console = Console()
 
+# Discogs' data listing host rate-limits aggressively, so keep concurrency low
+# and stagger requests to avoid tripping it.
+MAX_WORKERS = 3
+STAGGER_SECONDS = 1.0
+REQUEST_HEADERS = {
+    "User-Agent": "Mozilla/5.0 (compatible; DiscogsCLI/1.6; +https://github.com/ofurkancoban/DiscogsCLI)"
+}
+
+
 def _download_file(url: str, target_path: Path, progress, task_id, retries: int = 5) -> Path:
     """
     Downloads a file with support for resume and retry.
     Updates a Rich progress bar during download.
     """
-    headers = {}
-    downloaded = 0
-
-    # If file exists, resume from where it left off
-    if target_path.exists():
-        downloaded = target_path.stat().st_size
-        headers["Range"] = f"bytes={downloaded}-"
-
-    total_size = int(requests.head(url).headers.get("Content-Length", 0))
+    downloaded = target_path.stat().st_size if target_path.exists() else 0
+    if downloaded:
+        progress.update(task_id, completed=downloaded)
 
     for attempt in range(retries):
+        headers = dict(REQUEST_HEADERS)
+        if downloaded:
+            headers["Range"] = f"bytes={downloaded}-"
+
         try:
-            with requests.get(url, headers=headers, stream=True, timeout=10) as response:
+            with requests.get(url, headers=headers, stream=True, timeout=30) as response:
+                if response.status_code == 429:
+                    wait = int(response.headers.get("Retry-After", 30))
+                    console.print(f"[yellow]⚠ Rate limited by server, waiting {wait}s...[/]")
+                    sleep(wait)
+                    continue
+
+                if response.status_code == 416:
+                    # Requested range not satisfiable: file is already complete
+                    return target_path
+
                 response.raise_for_status()
 
-                mode = "ab" if downloaded else "wb"
+                # Server may ignore the Range header and resend the whole file
+                resumed = downloaded > 0 and response.status_code == 206
+                if not resumed:
+                    downloaded = 0
+                    progress.update(task_id, completed=0)
+
+                mode = "ab" if resumed else "wb"
                 with open(target_path, mode) as f:
                     for chunk in response.iter_content(chunk_size=1024 * 64):
                         if chunk:
@@ -48,17 +70,23 @@ def _download_file(url: str, target_path: Path, progress, task_id, retries: int
         except requests.RequestException as e:
             # Retry a few times if download fails
             if attempt < retries - 1:
-                sleep(1.5)
+                sleep(3)
                 continue
             else:
                 raise RuntimeError(f"Download failed after {retries} retries: {e}")
 
+    raise RuntimeError(f"Download failed after {retries} retries due to rate limiting")
+
+
 def download_files_threaded(df, selected_indexes, download_dir: Path) -> list[Path]:
     """
     Downloads multiple files concurrently using threads.
     Displays a combined progress bar for all downloads.
     """
-    urls = [df.iloc[i]["url"] for i in selected_indexes]
+    items = [
+        (row["url"], row["filename"], int(row["size_bytes"]))
+        for row in (df.iloc[i] for i in selected_indexes)
+    ]
     paths = []
     total_bytes = 0
     start_time = time.time()
@@ -75,12 +103,10 @@ def download_files_threaded(df, selected_indexes, download_dir: Path) -> list[Pa
         "•",
         TimeRemainingColumn(),
     ) as progress:
-        with ThreadPoolExecutor(max_workers=8) as executor:
+        with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
             futures = []
 
-            for url in urls:
-                filename = Path(urlparse(url).path).name
-
+            for url, filename, size_bytes in items:
                 # Extract date from filename and create target folder
                 date_str = filename.split("_")[1]
                 year_month = datetime.strptime(date_str, "%Y%m%d").strftime("%Y-%m")
@@ -89,17 +115,16 @@ def download_files_threaded(df, selected_indexes, download_dir: Path) -> list[Pa
                 target_path = target_folder / filename
 
                 # Skip already downloaded files
-                if target_path.exists():
+                if target_path.exists() and (size_bytes == 0 or target_path.stat().st_size >= size_bytes):
                     console.print(f"[yellow]⚠ Already downloaded:[/] {filename}")
                     paths.append(target_path)
                     continue
 
-                # Prepare progress bar for this file
-                total = int(requests.head(url).headers.get("Content-Length", 0))
-                total_bytes += total
-                task_id = progress.add_task("Downloading", filename=filename, total=total)
+                total_bytes += size_bytes
+                task_id = progress.add_task("Downloading", filename=filename, total=size_bytes)
                 future = executor.submit(_download_file, url, target_path, progress, task_id)
                 futures.append(future)
+                sleep(STAGGER_SECONDS)  # avoid bursting requests against the rate limiter
 
             # Wait for all downloads to finish
             for future in as_completed(futures):
@@ -117,4 +142,4 @@ def download_files_threaded(df, selected_indexes, download_dir: Path) -> list[Pa
     console.print(f"[cyan]💾 Total size:[/] {size_mb:.1f} MB")
     console.print(f"[cyan]⏱ Duration:[/] {duration:.1f} seconds")
 
-    return paths
+    return paths

{discogsdataprocessorcli-1.5.9 → discogsdataprocessorcli-1.6.1}/discogs/main.py

@@ -57,7 +57,6 @@ def run():
     typer.secho(f"\n✅ Done in {duration:.1f} seconds!", fg="green")
     open_folder(download_dir)
 
-@app.command()
 @app.command()
 def download():
     """
@@ -117,7 +116,7 @@ def delete(all: bool = typer.Option(False, "--all", help="Delete all downloaded,
     for i in selected:
         row = df.iloc[i]
         year_month = row["month"]
-        filename = Path(row["url"]).name
+        filename = row["filename"]
         data_dir = download_dir / "Datasets" / year_month
 
         gz_file = data_dir / filename

discogsdataprocessorcli-1.6.1/discogs/scraper.py

@@ -0,0 +1,156 @@
+# discogs/scraper.py
+
+import re
+import requests
+import pandas as pd
+from datetime import datetime
+from pathlib import Path
+from urllib.parse import quote, unquote
+from discogs.config import get_download_dir
+
+# Discogs replaced direct S3 bucket listing with this CDN-backed listing page.
+BASE_URL = "https://data.discogs.com/"
+DATA_PREFIX = "data/"
+
+REQUEST_HEADERS = {
+    "User-Agent": "Mozilla/5.0 (compatible; DiscogsCLI/1.6; +https://github.com/ofurkancoban/DiscogsCLI)"
+}
+
+# Matches one listing row, e.g.:
+# 2026-01-15 16:40:37             388 B          <a href="?download=data%2F2026%2Fdiscogs_20260101_CHECKSUM.txt">discogs_20260101_CHECKSUM.txt</a>
+#                                 -              <a href="?prefix=data%2F2026%2F">2026/</a>
+_ROW_RE = re.compile(
+    r'(?:(?P<date>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})\s*)?'
+    r'(?P<size>[\d.]+\s?[KMGT]?B|-)\s+'
+    r'<a href="\?(?P<kind>download|prefix)=(?P<key>[^"]+)">(?P<name>[^<]+)</a>'
+)
+
+_SIZE_MULTIPLIERS = {"B": 1, "KB": 1024, "MB": 1024 ** 2, "GB": 1024 ** 3, "TB": 1024 ** 4}
+
+
+def _parse_size(size_str: str) -> int:
+    """Converts a human-readable size string (e.g. "455.8 MB") to bytes."""
+    size_str = size_str.strip()
+    match = re.match(r"([\d.]+)\s*([KMGT]?B)", size_str)
+    if not match:
+        return 0
+    value, unit = match.groups()
+    return int(float(value) * _SIZE_MULTIPLIERS.get(unit, 1))
+
+
+def _fetch_listing(prefix: str) -> str:
+    """Fetches the raw HTML listing page for a given key prefix."""
+    url = f"{BASE_URL}?prefix={quote(prefix, safe='')}"
+    r = requests.get(url, headers=REQUEST_HEADERS, timeout=30)
+    r.raise_for_status()
+    return r.text
+
+
+def list_directories() -> list[str]:
+    """
+    Lists available yearly folders on the Discogs data listing page.
+    Example: data/2024/
+    """
+    html = _fetch_listing(DATA_PREFIX)
+
+    dirs = set()
+    for match in _ROW_RE.finditer(html):
+        if match.group("kind") != "prefix":
+            continue
+        key = unquote(match.group("key"))
+        if re.match(r"^data/\d{4}/$", key):
+            dirs.add(key)
+
+    return sorted(dirs)
+
+
+def list_files(directory_prefix: str) -> pd.DataFrame:
+    """
+    Lists files in the specified year folder and extracts metadata like size,
+    last modified date, type (artists, labels, etc.), and generates their URLs.
+    """
+    html = _fetch_listing(directory_prefix)
+
+    data = []
+    for match in _ROW_RE.finditer(html):
+        if match.group("kind") != "download":
+            continue
+
+        key = unquote(match.group("key"))
+        filename = Path(key).name
+
+        ctype = "unknown"
+        lname = filename.lower()
+        if "artist" in lname:
+            ctype = "artists"
+        elif "label" in lname:
+            ctype = "labels"
+        elif "master" in lname:
+            ctype = "masters"
+        elif "release" in lname:
+            ctype = "releases"
+
+        # Filter only usable .gz files
+        if ctype != "unknown" and filename.endswith(".gz"):
+            month = get_month_from_key(filename)
+            data.append({
+                "key": key,
+                "size_bytes": _parse_size(match.group("size")),
+                "last_modified": match.group("date"),
+                "month": month,
+                "content": ctype,
+                "filename": filename,
+                "url": f"{BASE_URL}?download={quote(key, safe='')}",
+            })
+
+    df = pd.DataFrame(data)
+    if df.empty:
+        return df
+
+    # Add download/extracted/converted status columns
+    download_dir = get_download_dir()
+    df["downloaded"] = df["filename"].apply(
+        lambda fn: (download_dir / "Datasets" / get_month_from_key(fn) / fn).exists()
+    )
+    df["extracted"] = df["filename"].apply(
+        lambda fn: (download_dir / "Datasets" / get_month_from_key(fn) / fn.replace(".gz", "")).exists()
+    )
+    df["converted"] = df["filename"].apply(
+        lambda fn: (download_dir / "Datasets" / get_month_from_key(fn) / fn.replace(".gz", ".csv")).exists()
+    )
+
+    return df
+
+
+def get_month_from_key(key: str) -> str:
+    """
+    Extracts the year and month from the Discogs filename.
+    Example: discogs_20240101_artist.gz → 2024-01
+    """
+    match = re.search(r"discogs_(\d{6})\d{2}", key)
+    if match:
+        try:
+            return datetime.strptime(match.group(1), "%Y%m").strftime("%Y-%m")
+        except Exception:
+            return ""
+    return ""
+
+
+def get_latest_files() -> pd.DataFrame:
+    """
+    Fetches and returns a DataFrame with files from the most recent available folder.
+    """
+    dirs = list_directories()
+    if not dirs:
+        return pd.DataFrame()
+
+    latest_dir = dirs[-1]
+    df = list_files(latest_dir)
+
+    if df.empty:
+        return df
+
+    # Parse dates and sort by month and type
+    df["last_modified"] = pd.to_datetime(df["last_modified"])
+    df = df.sort_values(by=["month", "content"], ascending=[False, True]).reset_index(drop=True)
+    return df

{discogsdataprocessorcli-1.5.9 → discogsdataprocessorcli-1.6.1}/discogs/selector.py

@@ -99,7 +99,7 @@ def display_status_table(df, download_dir: Path):
     table.add_column("Converted", justify="center")
 
     for idx, row in df.iterrows():
-        filename = Path(row["url"]).name
+        filename = row["filename"]
         year_month = row["month"]
         data_dir = download_dir / "Datasets" / year_month
         gz_path = data_dir / filename
@@ -153,7 +153,7 @@ def show_welcome():
 
     console.print(Panel.fit(
         ascii_logo,
-        title="Discogs Data Processor CLI (v1.5)",
+        title="Discogs Data Processor CLI (v1.6.1)",
         subtitle="by ofurkancoban",
         style="bold cyan"
     ))

{discogsdataprocessorcli-1.5.9 → discogsdataprocessorcli-1.6.1}/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = DiscogsDataProcessorCLI
-version = 1.5.9
+version = 1.6.1
 author = Furkan Çoban
 author_email = ofurkancoban@gmail.com
 description = A CLI to download, extract and convert Discogs data dumps
@@ -21,6 +21,7 @@ install_requires =
 	rich
 	pandas
 	typer
+	requests
 
 [options.entry_points]
 console_scripts = 

discogsdataprocessorcli-1.5.9/discogs/scraper.py

@@ -1,125 +0,0 @@
-# discogs/scraper.py
-
-import re
-import requests
-import pandas as pd
-import xml.etree.ElementTree as ET
-from datetime import datetime
-from pathlib import Path
-from discogs.config import get_download_dir
-
-# Base URL of the Discogs S3 bucket
-S3_BASE_URL = "https://discogs-data-dumps.s3.us-west-2.amazonaws.com/"
-S3_PREFIX = "data/"  # Prefix for data folders inside the bucket
-
-def list_directories() -> list[str]:
-    """
-    Lists available yearly folders on the Discogs S3 bucket.
-    Example: data/2024/
-    """
-    url = f"{S3_BASE_URL}?prefix={S3_PREFIX}&delimiter=/"
-    r = requests.get(url)
-    r.raise_for_status()
-
-    ns = "{http://s3.amazonaws.com/doc/2006-03-01/}"
-    root = ET.fromstring(r.text)
-
-    dirs = []
-    for cp in root.findall(ns + 'CommonPrefixes'):
-        p = cp.find(ns + 'Prefix').text
-        if re.match(r"data/\d{4}/", p):  # Match folders like "data/2023/"
-            dirs.append(p)
-
-    return sorted(dirs)
-
-def list_files(directory_prefix: str) -> pd.DataFrame:
-    """
-    Lists files in the specified S3 folder and extracts metadata like size,
-    last modified date, type (artists, labels, etc.), and generates their URLs.
-    """
-    url = f"{S3_BASE_URL}?prefix={directory_prefix}"
-    r = requests.get(url)
-    r.raise_for_status()
-
-    ns = "{http://s3.amazonaws.com/doc/2006-03-01/}"
-    root = ET.fromstring(r.text)
-
-    data = []
-    for content in root.findall(ns + 'Contents'):
-        key = content.find(ns + 'Key').text
-        size = int(content.find(ns + 'Size').text)
-        last_modified = content.find(ns + 'LastModified').text
-
-        # Determine content type from filename
-        ctype = "unknown"
-        lname = key.lower()
-        if "artist" in lname:
-            ctype = "artists"
-        elif "label" in lname:
-            ctype = "labels"
-        elif "master" in lname:
-            ctype = "masters"
-        elif "release" in lname:
-            ctype = "releases"
-
-        # Filter only usable .gz files
-        if ctype != "unknown" and key.endswith(".gz"):
-            month = get_month_from_key(key)
-            filename = Path(key).name
-            data.append({
-                "key": key,
-                "size_bytes": size,
-                "last_modified": last_modified,
-                "month": month,
-                "content": ctype,
-                "filename": filename,
-                "url": S3_BASE_URL + key,
-            })
-
-    df = pd.DataFrame(data)
-
-    # Add download/extracted/converted status columns
-    download_dir = get_download_dir()
-    df["downloaded"] = df["filename"].apply(
-        lambda fn: (download_dir / "Datasets" / get_month_from_key(fn) / fn).exists()
-    )
-    df["extracted"] = df["filename"].apply(
-        lambda fn: (download_dir / "Datasets" / get_month_from_key(fn) / fn.replace(".gz", "")).exists()
-    )
-    df["converted"] = df["filename"].apply(
-        lambda fn: (download_dir / "Datasets" / get_month_from_key(fn) / fn.replace(".gz", ".csv")).exists()
-    )
-
-    return df
-
-def get_month_from_key(key: str) -> str:
-    """
-    Extracts the year and month from the Discogs filename.
-    Example: discogs_20240101_artist.gz → 2024-01
-    """
-    match = re.search(r"discogs_(\d{6})\d{2}", key)
-    if match:
-        try:
-            return datetime.strptime(match.group(1), "%Y%m").strftime("%Y-%m")
-        except Exception:
-            return ""
-    return ""
-
-def get_latest_files() -> pd.DataFrame:
-    """
-    Fetches and returns a DataFrame with files from the most recent available S3 folder.
-    """
-    dirs = list_directories()
-    if not dirs:
-        return pd.DataFrame()
-
-    latest_dir = dirs[-1]
-    df = list_files(latest_dir)
-
-    if df.empty:
-        return df
-
-    # Parse dates and sort by month and type
-    df["last_modified"] = pd.to_datetime(df["last_modified"])
-    df = df.sort_values(by=["month", "content"], ascending=[False, True]).reset_index(drop=True)
-    return df