DiscogsDataProcessorCLI 1.5.0__py3-none-any.whl

discogs/chunker.py

@@ -0,0 +1,95 @@
+# discogs/chunker.py
+
+import re
+from pathlib import Path
+from rich.console import Console
+from rich.progress import Progress, BarColumn, TimeElapsedColumn, TextColumn
+
+
+def sanitize_line(line: str) -> str:
+    """
+    Removes invalid XML characters and fixes unescaped ampersands.
+    This helps ensure the XML is well-formed before processing.
+    """
+    line = re.sub(r'[^\x09\x0A\x0D\x20-\uD7FF\uE000-\uFFFD]', '', line)  # Remove illegal XML characters
+    line = re.sub(r'&(?![a-zA-Z0-9#]+;)', '&', line)  # Escape unescaped '&' characters
+    return line
+
+
+def chunk_xml_by_type(xml_file: Path, content_type: str, records_per_file: int = 10000) -> Path:
+    """
+    Splits a large XML file into smaller, valid XML files (chunks).
+    Each chunk contains up to `records_per_file` XML records.
+    Returns the folder path where chunked files are stored.
+    """
+    record_tag = content_type[:-1].lower()  # e.g., "releases" → "release"
+    start_pat = re.compile(fr'<{record_tag}\b', re.IGNORECASE)  # Match opening tag
+    end_pat = re.compile(fr'</{record_tag}>', re.IGNORECASE)    # Match closing tag
+
+    chunk_folder = xml_file.parent / f"chunked_{content_type}"  # Output folder
+    chunk_folder.mkdir(parents=True, exist_ok=True)
+
+    console = Console()
+    chunk_count = 0
+    record_count = 0
+    inside_record = False
+    buffer_lines = []  # Stores lines of current XML record
+    current_chunk_file = None
+
+    # Helper function to open a new chunk file
+    def open_new_chunk():
+        nonlocal chunk_count, current_chunk_file, record_count
+        chunk_count += 1
+        chunk_path = chunk_folder / f"chunk_{chunk_count:05}.xml"
+        current_chunk_file = open(chunk_path, "w", encoding="utf-8")
+        current_chunk_file.write(f'<?xml version="1.0" encoding="utf-8"?>\n<{content_type}>\n')
+        record_count = 0
+
+    # Helper function to close the current chunk file
+    def close_chunk():
+        nonlocal current_chunk_file
+        if current_chunk_file:
+            current_chunk_file.write(f"</{content_type}>")
+            current_chunk_file.close()
+            current_chunk_file = None
+
+    open_new_chunk()
+
+    # Setup progress bar for visual feedback
+    with Progress(
+        TextColumn("[progress.description]{task.description}"),
+        BarColumn(),
+        "[progress.percentage]{task.percentage:.1f}%",
+        "•",
+        TimeElapsedColumn()
+    ) as progress:
+        task = progress.add_task(f"Chunking {xml_file.name}", total=xml_file.stat().st_size)
+
+        # Read and process XML file line by line
+        with xml_file.open("r", encoding="utf-8", errors="ignore") as f:
+            for raw_line in f:
+                line = sanitize_line(raw_line)
+                progress.update(task, advance=len(raw_line))
+
+                if not inside_record:
+                    # Detect start of a record
+                    if start_pat.search(line):
+                        inside_record = True
+                        buffer_lines = [line]
+                else:
+                    buffer_lines.append(line)
+                    if end_pat.search(line):
+                        # Write complete record to current chunk
+                        current_chunk_file.write("".join(buffer_lines) + "\n")
+                        record_count += 1
+                        inside_record = False
+                        buffer_lines = []
+
+                        # If chunk is full, start a new one
+                        if record_count >= records_per_file:
+                            close_chunk()
+                            open_new_chunk()
+
+    close_chunk()
+    console.print(f"[green]✔ Chunked into {chunk_count} file(s): {chunk_folder}")
+    return chunk_folder

discogs/config.py

@@ -0,0 +1,75 @@
+# discogs/config.py
+
+import json
+from pathlib import Path
+from rich.console import Console
+from rich.prompt import Prompt
+
+CONFIG_PATH = Path.home() / ".discogs_config.json"  # Path to the user's config file
+DEFAULT_DOWNLOAD_PATH = Path.home() / "Downloads" / "Discogs"  # Default download location
+
+console = Console()
+
+def get_download_dir() -> Path:
+    """
+    Returns the download directory from the config file if it exists,
+    otherwise returns the default download path.
+    """
+    if CONFIG_PATH.exists():
+        path = CONFIG_PATH.read_text().strip()
+        if path:
+            return Path(path)
+    return DEFAULT_DOWNLOAD_PATH
+
+def set_download_dir(path: str) -> None:
+    """
+    Sets (writes) the download directory path to the config file.
+    """
+    CONFIG_PATH.write_text(path.strip())
+
+def load_config() -> dict:
+    """
+    Loads configuration data from the config file.
+    If the config cannot be read, it returns a dictionary with default values.
+    """
+    if CONFIG_PATH.exists():
+        try:
+            with CONFIG_PATH.open("r") as f:
+                return json.load(f)
+        except Exception as e:
+            console.print(f"[red]Error reading config:[/] {e}")
+    return {"download_dir": str(DEFAULT_DOWNLOAD_PATH)}
+
+def save_config(config: dict) -> None:
+    """
+    Saves the given configuration dictionary to the config file.
+    Creates the parent directory if it doesn't exist.
+    """
+    CONFIG_PATH.parent.mkdir(parents=True, exist_ok=True)
+    with CONFIG_PATH.open("w") as f:
+        json.dump(config, f, indent=4)
+
+def get_download_dir() -> Path:
+    """
+    Gets the download directory from config, falling back to default if missing.
+    """
+    config = load_config()
+    return Path(config.get("download_dir", str(DEFAULT_DOWNLOAD_PATH)))
+
+def configure_download_folder() -> None:
+    """
+    Prompts the user to configure the download folder via CLI.
+    Updates the config if a new folder is provided.
+    """
+    console.print("[bold]Configure Discogs download folder[/bold]")
+    current = get_download_dir()
+    console.print(f"Current folder: [green]{current}[/green]")
+
+    new_path = Prompt.ask("Enter new folder path or leave empty to keep current", default=str(current)).strip()
+    if new_path:
+        new_path = Path(new_path).expanduser()
+        new_path.mkdir(parents=True, exist_ok=True)
+        save_config({"download_dir": str(new_path)})
+        console.print(f"[green]✅ Download folder updated to:[/] {new_path}")
+    else:
+        console.print("[yellow]No changes made.[/yellow]")

discogs/converter.py

@@ -0,0 +1,190 @@
+# discogs/converter.py
+
+import shutil
+import json
+import csv
+import xml.etree.ElementTree as ET
+from pathlib import Path
+from rich.console import Console
+from rich.progress import (
+    Progress,
+    SpinnerColumn,
+    BarColumn,
+    TextColumn,
+    TimeElapsedColumn,
+    DownloadColumn,
+    TransferSpeedColumn,
+)
+
+from discogs.chunker import chunk_xml_by_type
+
+console = Console()
+
+def _scan_columns(chunk_file: Path, record_tag: str, column_set: set):
+    """
+    Scans an XML chunk file to identify all unique tag paths and attributes.
+    Adds these as potential CSV columns.
+    """
+    current_path = []
+    for event, elem in ET.iterparse(chunk_file, events=("start", "end")):
+        if event == "start":
+            current_path.append(elem.tag)
+            # Add all attributes of the current tag to the column set
+            for attr in elem.attrib:
+                key = "_".join(current_path[-2:] + [attr]) if len(current_path) >= 2 else f"{elem.tag}_{attr}"
+                column_set.add(key)
+        elif event == "end":
+            if elem.text and not elem.text.isspace():
+                # Add tag path for text content
+                key = "_".join(current_path[-2:] + [elem.tag]) if len(current_path) >= 2 else elem.tag
+                column_set.add(key)
+            current_path.pop()
+            elem.clear()
+
+def _write_rows(chunk_file: Path, writer: csv.DictWriter, columns: list, record_tag: str):
+    """
+    Parses an XML chunk and writes each record as a CSV row using the given column list.
+    """
+    current_path = []
+    record_data = {}
+    nested = {}
+
+    for event, elem in ET.iterparse(chunk_file, events=("start", "end")):
+        if event == "start":
+            current_path.append(elem.tag)
+            # Collect attribute values
+            for attr, val in elem.attrib.items():
+                key = "_".join(current_path[-2:] + [attr]) if len(current_path) >= 2 else f"{elem.tag}_{attr}"
+                nested.setdefault(key, []).append(val)
+        elif event == "end":
+            if elem.text and not elem.text.isspace():
+                # Collect text values
+                key = "_".join(current_path[-2:] + [elem.tag]) if len(current_path) >= 2 else elem.tag
+                nested.setdefault(key, []).append(elem.text.strip())
+
+            # End of a full record → flush it to CSV
+            if elem.tag == record_tag:
+                for k, v in nested.items():
+                    record_data[k] = v[0] if len(v) == 1 else json.dumps(v)  # Use first or serialize list
+                writer.writerow({col: record_data.get(col, "") for col in columns})
+                record_data.clear()
+                nested.clear()
+
+            current_path.pop()
+            elem.clear()
+
+from time import perf_counter
+
+def convert_chunks_to_csv(chunk_dir: Path, output_csv: Path, content_type: str):
+    """
+    Converts all chunked XML files in a given folder into a single CSV file.
+    The function discovers all columns, parses each chunk, and writes rows.
+    """
+    record_tag = content_type[:-1]  # e.g. "releases" → "release"
+    chunks = sorted(chunk_dir.glob("chunk_*.xml"))
+
+    if not chunks:
+        console.print(f"[red]No XML chunks found in {chunk_dir}[/red]")
+        return
+
+    start_time = perf_counter()
+
+    # Step 1: Scan all chunks to detect all column names
+    column_set = set()
+    console.print("[bold]Step 1:[/] Scanning tags...")
+
+    with Progress(
+        SpinnerColumn(),
+        TextColumn("[progress.description]{task.description}"),
+        BarColumn(),
+        "[progress.percentage]{task.percentage:.1f}%",
+        "•",
+        TimeElapsedColumn()
+    ) as p:
+        task = p.add_task("Scanning...", total=len(chunks))
+        for chunk in chunks:
+            _scan_columns(chunk, record_tag, column_set)
+            p.update(task, advance=1)
+
+    columns = sorted(column_set)
+
+    # Step 2: Write rows into CSV
+    console.print(f"[bold]Step 2:[/] Writing [green]{output_csv.name}[/green] with {len(columns)} columns...")
+
+    with open(output_csv, "w", newline="", encoding="utf-8") as f:
+        writer = csv.DictWriter(f, fieldnames=columns)
+        writer.writeheader()
+
+        with Progress(
+            SpinnerColumn(),
+            TextColumn("[progress.description]{task.description}"),
+            BarColumn(),
+            "[progress.percentage]{task.percentage:.1f}%",
+            "•",
+            TimeElapsedColumn()
+        ) as p:
+            task = p.add_task("Converting...", total=len(chunks))
+            for chunk in chunks:
+                _write_rows(chunk, writer, columns, record_tag)
+                p.update(task, advance=1)
+
+    duration = perf_counter() - start_time
+    output_size_mb = output_csv.stat().st_size / (1024 * 1024)
+
+    # Final status output
+    console.print(f"\n[green]✔ CSV saved:[/] {output_csv}")
+    console.print("[bold green]✔ Conversion completed[/bold green]")
+    console.print(f"[bold white]📄 Chunks processed:[/] {len(chunks)} files")
+    console.print(f"[bold white]🧩 Output CSV:[/] {output_csv.name}")
+    console.print(f"[bold white]💾 Output size:[/] {output_size_mb:.2f} MB")
+    console.print(f"[bold white]🗂 Saved to:[/] {output_csv.parent}")
+    console.print(f"[bold white]⏱ Duration:[/] {duration:.1f} seconds")
+
+def convert_xml_to_csv(xml_path: Path, content_type: str) -> Path:
+    """
+    Full pipeline: chunk an XML file and convert the chunks to a CSV file.
+    Temporary chunked files are deleted after the process.
+    """
+    chunk_dir = xml_path.parent / f"chunked_{content_type}"
+    output_csv = xml_path.with_suffix(".csv")
+
+    chunk_xml_by_type(xml_path, content_type)  # Split large XML into smaller parts
+    convert_chunks_to_csv(chunk_dir, output_csv, content_type)  # Convert chunks to CSV
+    shutil.rmtree(chunk_dir, ignore_errors=True)  # Cleanup
+
+    return output_csv
+
+def convert_interactively():
+    """
+    Prompts user to select XML files for conversion.
+    """
+    from rich.prompt import Prompt
+    from discogs.config import get_download_dir
+    from discogs.utils import open_folder
+
+    download_dir = get_download_dir()
+    dataset_dir = download_dir / "Datasets"
+
+    xml_files = list(dataset_dir.rglob("*.xml"))
+    if not xml_files:
+        console.print("[red]No XML files found to convert.[/red]")
+        return
+
+    console.print("[bold]Select XML file to convert:[/bold]")
+    for i, file in enumerate(xml_files):
+        console.print(f"[{i + 1}] {file.relative_to(download_dir)}")
+
+    choice = Prompt.ask("Enter number", default="1")
+    try:
+        idx = int(choice.strip()) - 1
+        if 0 <= idx < len(xml_files):
+            file = xml_files[idx]
+            content_type = file.stem.split("_")[-1]
+            convert_xml_to_csv(file, content_type)
+            open_folder(file.parent)
+        else:
+            console.print("[red]Invalid selection.[/red]")
+    except:
+        console.print("[red]Invalid input.[/red]")
+
+__all__ = ["convert_xml_to_csv"]  # Exported symbols

discogs/deleter.py

@@ -0,0 +1,51 @@
+# discogs/deleter.py
+
+from pathlib import Path
+from rich.console import Console
+from discogs.selector import display_status_table, select_indices
+from discogs.config import get_download_dir
+from discogs.scraper import get_latest_files
+
+console = Console()
+
+
+def delete_files():
+    """
+    Interactive function that allows the user to delete selected 
+    .gz, .xml, and .csv files from the dataset directory.
+    """
+    download_dir = get_download_dir()  # Get the base download directory from config
+    df = get_latest_files()  # Load the latest file list (as a DataFrame)
+
+    if df.empty:
+        console.print("[red]No files found.[/red]")
+        return
+
+    # Show the current file status table
+    display_status_table(df, download_dir)
+
+    # Let the user select which files to delete
+    selected = select_indices(df)
+
+    for i in selected:
+        row = df.iloc[i]
+        filename = Path(row["url"]).name
+        year_month = row["month"]
+        data_dir = download_dir / "Datasets" / year_month
+
+        gz_path = data_dir / filename  # Original .gz file
+        xml_path = gz_path.with_suffix("")  # Extracted .xml file
+        csv_path = xml_path.with_suffix(".csv")  # Converted .csv file
+
+        # Try deleting each file, one by one
+        for file in [gz_path, xml_path, csv_path]:
+            if file.exists():
+                file.unlink()  # Delete the file
+                console.print(f"[red]🗑 Deleted:[/] {file.name}")
+            else:
+                console.print(f"[dim]• Not found:[/] {file.name}")
+
+
+# Allow this script to be run directly
+if __name__ == "__main__":
+    delete_files()

discogs/downloader.py

@@ -0,0 +1,120 @@
+# discogs/downloader.py
+
+import time
+import requests
+from time import sleep
+from pathlib import Path
+from urllib.parse import urlparse
+from rich.console import Console
+from rich.progress import (
+    Progress, BarColumn, DownloadColumn, TransferSpeedColumn,
+    TimeRemainingColumn, TextColumn, SpinnerColumn
+)
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from datetime import datetime
+
+console = Console()
+
+def _download_file(url: str, target_path: Path, progress, task_id, retries: int = 5) -> Path:
+    """
+    Downloads a file with support for resume and retry.
+    Updates a Rich progress bar during download.
+    """
+    headers = {}
+    downloaded = 0
+
+    # If file exists, resume from where it left off
+    if target_path.exists():
+        downloaded = target_path.stat().st_size
+        headers["Range"] = f"bytes={downloaded}-"
+
+    total_size = int(requests.head(url).headers.get("Content-Length", 0))
+
+    for attempt in range(retries):
+        try:
+            with requests.get(url, headers=headers, stream=True, timeout=10) as response:
+                response.raise_for_status()
+
+                mode = "ab" if downloaded else "wb"
+                with open(target_path, mode) as f:
+                    for chunk in response.iter_content(chunk_size=1024 * 64):
+                        if chunk:
+                            f.write(chunk)
+                            downloaded += len(chunk)
+                            progress.update(task_id, completed=downloaded)
+
+            return target_path  # Download completed
+
+        except requests.RequestException as e:
+            # Retry a few times if download fails
+            if attempt < retries - 1:
+                sleep(1.5)
+                continue
+            else:
+                raise RuntimeError(f"Download failed after {retries} retries: {e}")
+
+def download_files_threaded(df, selected_indexes, download_dir: Path) -> list[Path]:
+    """
+    Downloads multiple files concurrently using threads.
+    Displays a combined progress bar for all downloads.
+    """
+    urls = [df.iloc[i]["url"] for i in selected_indexes]
+    paths = []
+    total_bytes = 0
+    start_time = time.time()
+
+    with Progress(
+        SpinnerColumn(),
+        TextColumn("[progress.description]{task.description} → [bold blue]{task.fields[filename]}", justify="right"),
+        BarColumn(),
+        "[progress.percentage]{task.percentage:>3.1f}%",
+        "•",
+        DownloadColumn(),
+        "•",
+        TransferSpeedColumn(),
+        "•",
+        TimeRemainingColumn(),
+    ) as progress:
+        with ThreadPoolExecutor(max_workers=8) as executor:
+            futures = []
+
+            for url in urls:
+                filename = Path(urlparse(url).path).name
+
+                # Extract date from filename and create target folder
+                date_str = filename.split("_")[1]
+                year_month = datetime.strptime(date_str, "%Y%m%d").strftime("%Y-%m")
+                target_folder = download_dir / "Datasets" / year_month
+                target_folder.mkdir(parents=True, exist_ok=True)
+                target_path = target_folder / filename
+
+                # Skip already downloaded files
+                if target_path.exists():
+                    console.print(f"[yellow]⚠ Already downloaded:[/] {filename}")
+                    paths.append(target_path)
+                    continue
+
+                # Prepare progress bar for this file
+                total = int(requests.head(url).headers.get("Content-Length", 0))
+                total_bytes += total
+                task_id = progress.add_task("Downloading", filename=filename, total=total)
+                future = executor.submit(_download_file, url, target_path, progress, task_id)
+                futures.append(future)
+
+            # Wait for all downloads to finish
+            for future in as_completed(futures):
+                try:
+                    result = future.result()
+                    paths.append(result)
+                except Exception as e:
+                    console.print(f"[red]Error downloading file:[/] {e}")
+
+    duration = time.time() - start_time
+    size_mb = total_bytes / (1024 ** 2)
+
+    # Final summary
+    console.print(f"\n[green]📥 {len(paths)} file(s) downloaded[/]")
+    console.print(f"[cyan]💾 Total size:[/] {size_mb:.1f} MB")
+    console.print(f"[cyan]⏱ Duration:[/] {duration:.1f} seconds")
+
+    return paths

discogs/extractor.py

@@ -0,0 +1,92 @@
+# discogs/extractor.py
+
+import gzip
+from pathlib import Path
+from rich.console import Console
+from rich.progress import Progress, SpinnerColumn, BarColumn, TimeElapsedColumn, TextColumn
+
+console = Console()  # Global console instance for consistent output
+
+def extract_gz(gz_path: Path, delete_original: bool = False) -> Path:
+    """
+    Extracts a single .gz file into its original XML format.
+    Optionally deletes the .gz file after extraction.
+    """
+    if gz_path.suffix != ".gz":
+        raise ValueError("File is not a .gz file")
+
+    xml_path = gz_path.with_suffix("")  # Remove ".gz" to get .xml filename
+    total_size = gz_path.stat().st_size
+
+    # Display progress bar while extracting
+    with Progress(
+        SpinnerColumn(),
+        TextColumn("[progress.description]{task.description}"),
+        BarColumn(),
+        "[progress.percentage]{task.percentage:>3.1f}%",
+        "•",
+        TimeElapsedColumn()
+    ) as progress:
+        task = progress.add_task(f"Extracting {gz_path.name}", total=total_size)
+
+        with gzip.open(gz_path, 'rb') as f_in, open(xml_path, 'wb') as f_out:
+            while True:
+                chunk = f_in.read(1024 * 1024)  # Read in 1MB chunks
+                if not chunk:
+                    break
+                f_out.write(chunk)
+                progress.update(task, advance=len(chunk))
+
+    console.print(f"[green]✔ Extracted:[/] {xml_path}")
+
+    # Optionally remove the original .gz file after extraction
+    if delete_original:
+        gz_path.unlink()
+        console.print(f"[yellow]🗑 Deleted original:[/] {gz_path}")
+
+    return xml_path
+
+def extract_gz_files(files: list[Path], delete_original: bool = False) -> list[Path]:
+    """
+    Extracts multiple .gz files in sequence.
+    Returns a list of extracted XML file paths.
+    """
+    return [extract_gz(file, delete_original=delete_original) for file in files]
+
+def get_extracted_path(gz_path: Path) -> Path:
+    """
+    Returns the path of the extracted XML file for a given .gz file.
+    """
+    return gz_path.with_suffix("")
+
+def extract_interactively():
+    """
+    Prompts user to select .gz files for extraction.
+    """
+    from rich.prompt import Prompt
+    from discogs.config import get_download_dir
+    from discogs.utils import open_folder
+
+    download_dir = get_download_dir()
+    dataset_dir = download_dir / "Datasets"
+
+    gz_files = list(dataset_dir.rglob("*.gz"))
+    if not gz_files:
+        console.print("[red]No .gz files found to extract.[/red]")
+        return
+
+    console.print("[bold]Select .gz file to extract:[/bold]")
+    for i, file in enumerate(gz_files):
+        console.print(f"[{i + 1}] {file.relative_to(download_dir)}")
+
+    choice = Prompt.ask("Enter number", default="1")
+    try:
+        idx = int(choice.strip()) - 1
+        if 0 <= idx < len(gz_files):
+            file = gz_files[idx]
+            extract_gz(file)
+            open_folder(file.parent)
+        else:
+            console.print("[red]Invalid selection.[/red]")
+    except:
+        console.print("[red]Invalid input.[/red]")