DiscogsDataProcessorCLI 1.5.0__py3-none-any.whl

discogs/main.py

@@ -0,0 +1,170 @@
+# discogs/main.py
+
+import typer
+from discogs.selector import show_welcome, display_status_table, select_indices
+from discogs.scraper import get_latest_files
+from discogs.downloader import download_files_threaded
+from discogs.extractor import extract_gz_files
+from discogs.converter import convert_xml_to_csv
+from discogs.config import get_download_dir
+from discogs.utils import open_folder
+from pathlib import Path
+from rich.console import Console
+import time
+
+# Initialize CLI app with help text
+app = typer.Typer(
+    help="📦 Discogs CLI - Download, extract, and convert Discogs data dumps.",
+    invoke_without_command=True
+)
+
+console = Console()
+
+@app.command(help="One-click pipeline: Fetch latest files, download, extract, and convert to CSV.")
+def run():
+    """
+    Full automated pipeline: shows welcome screen, fetches files,
+    lets user choose which ones to download, then downloads, extracts,
+    and converts them to CSV.
+    """
+    show_welcome()
+    download_dir = get_download_dir()
+
+    typer.echo("\U0001F50D Fetching available Discogs files...")
+    df = get_latest_files()
+
+    if df.empty:
+        typer.echo("No data found.")
+        raise typer.Exit()
+
+    display_status_table(df, download_dir)
+    indices = select_indices(df)
+
+    if not indices:
+        typer.echo("No selection made.")
+        raise typer.Exit()
+
+    start = time.time()
+
+    downloaded = download_files_threaded(df, indices, download_dir)
+    extracted = extract_gz_files(downloaded)
+
+    for xml_file in extracted:
+        content_type = xml_file.stem.split("_")[-1]
+        convert_xml_to_csv(xml_file, content_type)
+
+    duration = time.time() - start
+    typer.secho(f"\n✅ Done in {duration:.1f} seconds!", fg="green")
+    open_folder(download_dir)
+
+@app.command()
+@app.command()
+def download():
+    """
+    Download selected Discogs data files only (no extract or convert).
+    """
+    download_dir = get_download_dir()
+    typer.echo("\U0001F50D Fetching available Discogs files...")
+    df = get_latest_files()
+
+    if df.empty:
+        typer.echo("No data found.")
+        raise typer.Exit()
+
+    display_status_table(df, download_dir)
+    indices = select_indices(df)
+    if not indices:
+        typer.echo("No files selected.")
+        raise typer.Exit()
+
+    download_files_threaded(df, indices, download_dir)
+
+    open_folder(download_dir)
+
+@app.command()
+def convert():
+    """Convert extracted XML files to CSV (interactive mode)."""
+    from discogs.converter import convert_interactively
+    convert_interactively()
+
+@app.command()
+def extract():
+    """Extract downloaded .gz files (interactive mode)."""
+    from discogs.extractor import extract_interactively
+    extract_interactively()
+
+@app.command("delete")
+def delete(all: bool = typer.Option(False, "--all", help="Delete all downloaded, extracted and converted files.")):
+    """
+    Deletes selected or all downloaded, extracted, and converted files.
+    """
+    download_dir = get_download_dir()
+    df = get_latest_files()
+
+    if df.empty:
+        console.print("[red]No files found.[/red]")
+        raise typer.Exit()
+
+    display_status_table(df, download_dir)
+
+    # If --all is passed, select all files
+    selected = list(range(len(df))) if all else select_indices(df, allow_all=True)
+
+    if not selected:
+        console.print("[yellow]No files selected.[/yellow]")
+        raise typer.Exit()
+
+    for i in selected:
+        row = df.iloc[i]
+        year_month = row["month"]
+        filename = Path(row["url"]).name
+        data_dir = download_dir / "Datasets" / year_month
+
+        gz_file = data_dir / filename
+        xml_file = gz_file.with_suffix("")
+        csv_file = xml_file.with_suffix(".csv")
+
+        for file in [gz_file, xml_file, csv_file]:
+            if file.exists():
+                try:
+                    file.unlink()
+                    console.print(f"[green]✔ Deleted:[/] {file.name}")
+                except Exception as e:
+                    console.print(f"[red]✗ Failed to delete {file.name}:[/] {e}")
+            else:
+                console.print(f"[dim]• Not found:[/] {file.name}")
+
+@app.command()
+def show():
+    """
+    Displays the list of available Discogs dump files.
+    """
+    from discogs.selector import show_welcome
+    from discogs.scraper import get_latest_files
+    from discogs.config import get_download_dir
+
+    show_welcome()
+    df = get_latest_files()
+    display_status_table(df, get_download_dir())
+
+@app.command()
+def config():
+    """Launches the download folder configuration prompt."""
+    from discogs.config import set_download_dir
+    set_download_dir()
+
+@app.callback()
+def main(ctx: typer.Context):
+    """
+    If no subcommand is provided, defaults to running the `run` pipeline.
+    """
+    if ctx.invoked_subcommand is None:
+        run()
+
+# Automatically run the `run` pipeline if no arguments are provided
+if __name__ == "__main__":
+    import sys
+    if len(sys.argv) == 1:
+        app(prog_name="discogs", args=["run"])
+    else:
+        app()

discogs/scraper.py

@@ -0,0 +1,125 @@
+# discogs/scraper.py
+
+import re
+import requests
+import pandas as pd
+import xml.etree.ElementTree as ET
+from datetime import datetime
+from pathlib import Path
+from discogs.config import get_download_dir
+
+# Base URL of the Discogs S3 bucket
+S3_BASE_URL = "https://discogs-data-dumps.s3.us-west-2.amazonaws.com/"
+S3_PREFIX = "data/"  # Prefix for data folders inside the bucket
+
+def list_directories() -> list[str]:
+    """
+    Lists available yearly folders on the Discogs S3 bucket.
+    Example: data/2024/
+    """
+    url = f"{S3_BASE_URL}?prefix={S3_PREFIX}&delimiter=/"
+    r = requests.get(url)
+    r.raise_for_status()
+
+    ns = "{http://s3.amazonaws.com/doc/2006-03-01/}"
+    root = ET.fromstring(r.text)
+
+    dirs = []
+    for cp in root.findall(ns + 'CommonPrefixes'):
+        p = cp.find(ns + 'Prefix').text
+        if re.match(r"data/\d{4}/", p):  # Match folders like "data/2023/"
+            dirs.append(p)
+
+    return sorted(dirs)
+
+def list_files(directory_prefix: str) -> pd.DataFrame:
+    """
+    Lists files in the specified S3 folder and extracts metadata like size,
+    last modified date, type (artists, labels, etc.), and generates their URLs.
+    """
+    url = f"{S3_BASE_URL}?prefix={directory_prefix}"
+    r = requests.get(url)
+    r.raise_for_status()
+
+    ns = "{http://s3.amazonaws.com/doc/2006-03-01/}"
+    root = ET.fromstring(r.text)
+
+    data = []
+    for content in root.findall(ns + 'Contents'):
+        key = content.find(ns + 'Key').text
+        size = int(content.find(ns + 'Size').text)
+        last_modified = content.find(ns + 'LastModified').text
+
+        # Determine content type from filename
+        ctype = "unknown"
+        lname = key.lower()
+        if "artist" in lname:
+            ctype = "artists"
+        elif "label" in lname:
+            ctype = "labels"
+        elif "master" in lname:
+            ctype = "masters"
+        elif "release" in lname:
+            ctype = "releases"
+
+        # Filter only usable .gz files
+        if ctype != "unknown" and key.endswith(".gz"):
+            month = get_month_from_key(key)
+            filename = Path(key).name
+            data.append({
+                "key": key,
+                "size_bytes": size,
+                "last_modified": last_modified,
+                "month": month,
+                "content": ctype,
+                "filename": filename,
+                "url": S3_BASE_URL + key,
+            })
+
+    df = pd.DataFrame(data)
+
+    # Add download/extracted/converted status columns
+    download_dir = get_download_dir()
+    df["downloaded"] = df["filename"].apply(
+        lambda fn: (download_dir / "Datasets" / get_month_from_key(fn) / fn).exists()
+    )
+    df["extracted"] = df["filename"].apply(
+        lambda fn: (download_dir / "Datasets" / get_month_from_key(fn) / fn.replace(".gz", "")).exists()
+    )
+    df["converted"] = df["filename"].apply(
+        lambda fn: (download_dir / "Datasets" / get_month_from_key(fn) / fn.replace(".gz", ".csv")).exists()
+    )
+
+    return df
+
+def get_month_from_key(key: str) -> str:
+    """
+    Extracts the year and month from the Discogs filename.
+    Example: discogs_20240101_artist.gz → 2024-01
+    """
+    match = re.search(r"discogs_(\d{6})\d{2}", key)
+    if match:
+        try:
+            return datetime.strptime(match.group(1), "%Y%m").strftime("%Y-%m")
+        except Exception:
+            return ""
+    return ""
+
+def get_latest_files() -> pd.DataFrame:
+    """
+    Fetches and returns a DataFrame with files from the most recent available S3 folder.
+    """
+    dirs = list_directories()
+    if not dirs:
+        return pd.DataFrame()
+
+    latest_dir = dirs[-1]
+    df = list_files(latest_dir)
+
+    if df.empty:
+        return df
+
+    # Parse dates and sort by month and type
+    df["last_modified"] = pd.to_datetime(df["last_modified"])
+    df = df.sort_values(by=["month", "content"], ascending=[False, True]).reset_index(drop=True)
+    return df

discogs/selector.py

@@ -0,0 +1,197 @@
+# discogs/selector.py
+
+from rich.prompt import Prompt
+from typing import List
+import pandas as pd
+from rich.panel import Panel
+from rich.markdown import Markdown
+from rich.table import Table
+from rich.console import Console
+from discogs.utils import human_readable_size
+from pathlib import Path
+
+console = Console()
+
+def display_table(df: pd.DataFrame) -> None:
+    """
+    Displays a Rich-formatted table of available Discogs files.
+    Shows basic info: index, month, content type, file size, and URL.
+    """
+    table = Table(title="Available Discogs Files", show_lines=True)
+
+    table.add_column("No", style="cyan", justify="right")
+    table.add_column("Month", style="magenta")
+    table.add_column("Type", style="green")
+    table.add_column("Size (MB)", justify="right")
+    table.add_column("URL", style="dim", overflow="fold")
+
+    for i, row in df.iterrows():
+        size_mb = f"{row['size_bytes'] / (1024 ** 2):.2f}"
+        table.add_row(
+            str(i + 1),
+            row["month"],
+            row["content"],
+            size_mb,
+            row["url"]
+        )
+
+    console.print(table)
+
+def select_indices(df: pd.DataFrame, allow_all: bool = False) -> List[int]:
+    """
+    Prompts user to select files by number (comma-separated list or 'all' if allowed).
+    Returns a list of selected row indices.
+    """
+    while True:
+        selection = Prompt.ask(
+            "[bold green]Select file(s) by number (comma-separated)[/]",
+            default="1"
+        )
+
+        if allow_all and selection.strip().lower() == "all":
+            return list(range(len(df)))
+
+        try:
+            selected = [int(x.strip()) - 1 for x in selection.split(",")]
+            if all(0 <= i < len(df) for i in selected):
+                return selected
+            else:
+                raise ValueError
+        except ValueError:
+            console.print("[red]Invalid selection. Try again.[/red]")
+
+def select_files(df: pd.DataFrame) -> List[int]:
+    """
+    Allows user to select files using basic printed list.
+    Returns selected row indices.
+    """
+    if df.empty:
+        console.print("[red]No files to select.[/red]")
+        return []
+
+    for i, row in df.iterrows():
+        size_mb = f"{row['size_bytes'] / (1024 ** 2):.2f} MB"
+        console.print(f"[{i + 1}] {row['month']} | {row['content']} | {size_mb}")
+
+    while True:
+        selection = Prompt.ask("Select file(s) by number (comma-separated)", default="1")
+        try:
+            indices = [int(x.strip()) - 1 for x in selection.split(",")]
+            if all(0 <= i < len(df) for i in indices):
+                return indices
+        except Exception:
+            pass
+
+        console.print("[red]Invalid selection. Try again.[/red]")
+
+def display_status_table(df, download_dir: Path):
+    """
+    Displays the full download/extract/convert status of all files in a table.
+    Includes ✔/✗ markers for each status column.
+    """
+    table = Table(title="Available Discogs Files", show_lines=True)
+    table.add_column("No", justify="right", style="cyan", no_wrap=True)
+    table.add_column("Month", style="magenta")
+    table.add_column("Type", style="yellow")
+    table.add_column("Size", justify="right")
+    table.add_column("Downloaded", justify="center")
+    table.add_column("Extracted", justify="center")
+    table.add_column("Converted", justify="center")
+
+    for idx, row in df.iterrows():
+        filename = Path(row["url"]).name
+        year_month = row["month"]
+        data_dir = download_dir / "Datasets" / year_month
+        gz_path = data_dir / filename
+        xml_path = gz_path.with_suffix("")
+        csv_path = xml_path.with_suffix(".csv")
+
+        is_downloaded = gz_path.exists()
+        is_extracted = xml_path.exists()
+        is_converted = csv_path.exists()
+
+        check = lambda b: "[green]✔[/green]" if b else "[red]✗[/red]"
+
+        table.add_row(
+            str(idx + 1),
+            row["month"],
+            row["content"],
+            human_readable_size(row["size_bytes"]),
+            check(is_downloaded),
+            check(is_extracted),
+            check(is_converted),
+        )
+
+    console.print(table)
+
+def show_welcome():
+    """
+    Displays the ASCII welcome screen with a summary of available commands and features.
+    """
+    ascii_logo = r"""
+            ██████╗ ██╗███████╗ ██████╗ ██████╗  ██████╗ ███████╗             
+            ██╔══██╗██║██╔════╝██╔════╝██╔═══██╗██╔════╝ ██╔════╝             
+            ██║  ██║██║███████╗██║     ██║   ██║██║  ███╗███████╗             
+            ██║  ██║██║╚════██║██║     ██║   ██║██║   ██║╚════██║             
+            ██████╔╝██║███████║╚██████╗╚██████╔╝╚██████╔╝███████║             
+            ╚═════╝ ╚═╝╚══════╝ ╚═════╝ ╚═════╝  ╚═════╝ ╚══════╝             
+
+                        ██████╗  █████╗ ████████╗ █████╗                      
+                        ██╔══██╗██╔══██╗╚══██╔══╝██╔══██╗                     
+                        ██║  ██║███████║   ██║   ███████║                     
+                        ██║  ██║██╔══██║   ██║   ██╔══██║                     
+                        ██████╔╝██║  ██║   ██║   ██║  ██║                     
+                        ╚═════╝ ╚═╝  ╚═╝   ╚═╝   ╚═╝  ╚═╝                     
+
+    ██████╗ ██████╗  ██████╗  ██████╗███████╗███████╗███████╗ ██████╗ ██████╗ 
+    ██╔══██╗██╔══██╗██╔═══██╗██╔════╝██╔════╝██╔════╝██╔════╝██╔═══██╗██╔══██╗
+    ██████╔╝██████╔╝██║   ██║██║     █████╗  ███████╗███████╗██║   ██║██████╔╝
+    ██╔═══╝ ██╔══██╗██║   ██║██║     ██╔══╝  ╚════██║╚════██║██║   ██║██╔══██╗
+    ██║     ██║  ██║╚██████╔╝╚██████╗███████╗███████║███████║╚██████╔╝██║  ██║
+    ╚═╝     ╚═╝  ╚═╝ ╚═════╝  ╚═════╝╚══════╝╚══════╝╚══════╝ ╚═════╝ ╚═╝  ╚═╝
+        """
+
+    console.print(Panel.fit(
+        ascii_logo,
+        title="Discogs Data Processor CLI",
+        subtitle="by ofurkancoban",
+        style="bold cyan"
+    ))
+
+    md = Markdown("""
+Welcome to the **Discogs CLI**!
+
+This tool allows you to:
+- 🧠 Scrape the latest data dump list from Discogs
+- ⬇️  Download selected files
+- 📦 Extract `.gz` files
+- ✂️  Chunk large XML into smaller files
+- 📄 Convert everything into tidy CSV
+- 🗑 Delete downloaded/extracted/converted files
+- ⚙️  Configure your download folder
+
+---
+
+**Available Commands:**
+
+- `python -m discogs.main run` — Full auto mode (download → extract → convert)
+- `python -m discogs.main show` — Display available Discogs files
+- `python -m discogs.main download` — Download selected files
+- `python -m discogs.main extract` — Extract previously downloaded `.gz` files
+- `python -m discogs.main convert` — Convert extracted `.xml` files to `.csv`
+- `python -m discogs.main delete` — Delete files by selection (or `--all`)
+- `python -m discogs.main config` — Set or change your download folder
+
+---
+
+**Connect with me:**
+
+- 🌐 GitHub: [github.com/ofurkancoban](https://github.com/ofurkancoban)
+- 💼 LinkedIn: [linkedin.com/in/ofurkancoban](https://linkedin.com/in/ofurkancoban)
+- 📊 Kaggle: [kaggle.com/ofurkancoban](https://www.kaggle.com/ofurkancoban)
+
+---
+
+""")
+
+    console.print(md)

discogs/utils.py

@@ -0,0 +1,96 @@
+# discogs/utils.py
+
+import json
+from rich.prompt import Prompt
+from rich.console import Console
+import subprocess
+import platform
+from pathlib import Path
+
+console = Console()
+
+CONFIG_PATH = Path.home() / "Downloads" / "Discogs" / ".discogs_config.json"
+DEFAULT_DOWNLOAD_PATH = Path.home() / "Downloads" / "Discogs"
+
+def load_config() -> dict:
+    """
+    Loads the config JSON from disk.
+    Returns default config if file doesn't exist or can't be read.
+    """
+    if CONFIG_PATH.exists():
+        try:
+            with CONFIG_PATH.open("r") as f:
+                return json.load(f)
+        except Exception as e:
+            console.print(f"[red]⚠ Error reading config:[/] {e}")
+    return {"download_dir": str(DEFAULT_DOWNLOAD_PATH)}
+
+def save_config(config: dict):
+    """
+    Saves the provided config dictionary as JSON to disk.
+    """
+    try:
+        with CONFIG_PATH.open("w") as f:
+            json.dump(config, f, indent=2)
+        console.print(f"[green]✔ Config saved:[/] {CONFIG_PATH}")
+    except Exception as e:
+        console.print(f"[red]⚠ Failed to save config:[/] {e}")
+
+def get_download_dir() -> Path:
+    """
+    Returns the current download directory from config.
+    Falls back to default if not configured.
+    """
+    config = load_config()
+    return Path(config.get("download_dir", str(DEFAULT_DOWNLOAD_PATH)))
+
+def open_folder(path: Path):
+    """
+    Opens the given folder path using the system's default file explorer.
+    Supports macOS, Windows, and Linux.
+    """
+    try:
+        if platform.system() == "Darwin":  # macOS
+            subprocess.run(["open", str(path)])
+        elif platform.system() == "Windows":
+            subprocess.run(["explorer", str(path)])
+        else:  # Linux (assumes xdg-open is available)
+            subprocess.run(["xdg-open", str(path)])
+    except Exception as e:
+        print(f"Error opening folder: {e}")
+
+def human_readable_size(size_bytes: int) -> str:
+    """
+    Converts a file size in bytes into a human-readable string (KB, MB, GB, etc).
+    """
+    if size_bytes == 0:
+        return "0 B"
+
+    size_name = ("B", "KB", "MB", "GB", "TB")
+    i = 0
+    double_size = float(size_bytes)
+    while double_size >= 1024 and i < len(size_name) - 1:
+        double_size /= 1024
+        i += 1
+    return f"{double_size:.2f} {size_name[i]}"
+
+def set_download_dir():
+    """
+    Prompts the user to enter a new download folder and updates the config.
+    Creates the folder if it doesn't exist.
+    """
+    current = get_download_dir()
+    new_path = Prompt.ask("Download folder", default=str(current)).strip()
+    path = Path(new_path).expanduser()
+
+    if not path.exists():
+        try:
+            path.mkdir(parents=True)
+            console.print(f"[green]✔ Created directory:[/] {path}")
+        except Exception as e:
+            console.print(f"[red]Failed to create directory:[/] {e}")
+            return
+
+    config = load_config()
+    config["download_dir"] = str(path)
+    save_config(config)