DiscogsDataProcessorCLI 1.5.0__tar.gz

discogsdataprocessorcli-1.5.0/DiscogsDataProcessorCLI.egg-info/PKG-INFO

@@ -0,0 +1,119 @@
+Metadata-Version: 2.4
+Name: DiscogsDataProcessorCLI
+Version: 1.5.0
+Summary: A CLI to download, extract and convert Discogs data dumps
+Home-page: https://github.com/ofurkancoban/DiscogsCLI
+Author: Furkan Çoban
+Author-email: ofurkancoban@gmail.com
+License: MIT
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+Requires-Dist: rich
+Requires-Dist: pandas
+Requires-Dist: typer
+
+# 🎧 Discogs CLI — Data Processor Tool 💿
+<p align="center">
+  <img src="img/logo.png" alt="Discogs Logo" width="200"/>
+</p>
+A modern command-line tool to **download**, **extract**, and **convert** Discogs data dumps into structured CSV files.
+<p align="center">
+  <img src="img/preview.gif" />
+</p>
+
+---
+
+## 🚀 Features
+
+- 🧠 Scrape latest available data dump list from Discogs S3
+- ⬇️ Download `.gz` files for artists, labels, releases, masters
+- 📦 Extract `.gz` files to raw XML
+- ✂️ Chunk large XML into smaller files
+- 📄 Convert XML to clean, flat CSV files
+- 🗑 Delete selected or all files
+- ⚙️ Set custom download folder
+- 🧪 Easy to use from terminal with friendly UI
+
+---
+
+## 🧩 Installation
+
+### 🍻 Install with Homebrew
+```bash
+brew tap ofurkancoban/discogs
+brew install discogs
+```
+### or
+```bash
+git clone https://github.com/ofurkancoban/DiscogsCLI.git
+cd DiscogsCLI
+pip install -e .
+```
+
+---
+
+## 💻 Usage
+
+```bash
+discogs run        # Auto: download → extract → convert
+discogs show       # List available Discogs data
+discogs download   # Just download selected files
+discogs extract    # Extract downloaded .gz files
+discogs convert    # Convert extracted XML to CSV
+discogs delete     # Delete files by selection or --all
+discogs config     # Set download folder
+```
+
+---
+
+## 📁 Folder Structure
+
+```
+~/Downloads/Discogs/
+├── .discogs_config.json
+└── Datasets/
+    └── 2025-04/
+        ├── discogs_20250401_artists.gz
+        ├── discogs_20250401_artists     ← .xml
+        └── discogs_20250401_artists.csv ← converted
+```
+
+---
+
+## 🧠 Example Workflow
+
+```bash
+discogs show
+# [1] 2025-04 | releases | 950 MB
+# [2] 2025-04 | artists  | 320 MB
+
+discogs download
+# Select 1,2
+# Downloads only
+
+discogs extract
+# Select file to extract
+
+discogs convert
+# Select XML to convert
+```
+
+---
+
+## 🧑‍💻 Author
+
+- GitHub: [github.com/ofurkancoban](https://github.com/ofurkancoban)
+- LinkedIn: [linkedin.com/in/ofurkancoban](https://linkedin.com/in/ofurkancoban)
+- Kaggle: [kaggle.com/ofurkancoban](https://www.kaggle.com/ofurkancoban)
+
+---
+
+## 📜 License
+
+MIT — use freely, mention when you do something cool 😎
+---
+
+Built with ❤️ by [@ofurkancoban](https://github.com/ofurkancoban)

discogsdataprocessorcli-1.5.0/DiscogsDataProcessorCLI.egg-info/SOURCES.txt

@@ -0,0 +1,20 @@
+README.md
+pyproject.toml
+setup.cfg
+DiscogsDataProcessorCLI.egg-info/PKG-INFO
+DiscogsDataProcessorCLI.egg-info/SOURCES.txt
+DiscogsDataProcessorCLI.egg-info/dependency_links.txt
+DiscogsDataProcessorCLI.egg-info/entry_points.txt
+DiscogsDataProcessorCLI.egg-info/requires.txt
+DiscogsDataProcessorCLI.egg-info/top_level.txt
+discogs/__init__.py
+discogs/chunker.py
+discogs/config.py
+discogs/converter.py
+discogs/deleter.py
+discogs/downloader.py
+discogs/extractor.py
+discogs/main.py
+discogs/scraper.py
+discogs/selector.py
+discogs/utils.py

discogsdataprocessorcli-1.5.0/DiscogsDataProcessorCLI.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

discogsdataprocessorcli-1.5.0/DiscogsDataProcessorCLI.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+discogs = discogs.main:main

discogsdataprocessorcli-1.5.0/DiscogsDataProcessorCLI.egg-info/requires.txt

@@ -0,0 +1,3 @@
+rich
+pandas
+typer

discogsdataprocessorcli-1.5.0/DiscogsDataProcessorCLI.egg-info/top_level.txt

@@ -0,0 +1 @@
+discogs

discogsdataprocessorcli-1.5.0/PKG-INFO

@@ -0,0 +1,119 @@
+Metadata-Version: 2.4
+Name: DiscogsDataProcessorCLI
+Version: 1.5.0
+Summary: A CLI to download, extract and convert Discogs data dumps
+Home-page: https://github.com/ofurkancoban/DiscogsCLI
+Author: Furkan Çoban
+Author-email: ofurkancoban@gmail.com
+License: MIT
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+Requires-Dist: rich
+Requires-Dist: pandas
+Requires-Dist: typer
+
+# 🎧 Discogs CLI — Data Processor Tool 💿
+<p align="center">
+  <img src="img/logo.png" alt="Discogs Logo" width="200"/>
+</p>
+A modern command-line tool to **download**, **extract**, and **convert** Discogs data dumps into structured CSV files.
+<p align="center">
+  <img src="img/preview.gif" />
+</p>
+
+---
+
+## 🚀 Features
+
+- 🧠 Scrape latest available data dump list from Discogs S3
+- ⬇️ Download `.gz` files for artists, labels, releases, masters
+- 📦 Extract `.gz` files to raw XML
+- ✂️ Chunk large XML into smaller files
+- 📄 Convert XML to clean, flat CSV files
+- 🗑 Delete selected or all files
+- ⚙️ Set custom download folder
+- 🧪 Easy to use from terminal with friendly UI
+
+---
+
+## 🧩 Installation
+
+### 🍻 Install with Homebrew
+```bash
+brew tap ofurkancoban/discogs
+brew install discogs
+```
+### or
+```bash
+git clone https://github.com/ofurkancoban/DiscogsCLI.git
+cd DiscogsCLI
+pip install -e .
+```
+
+---
+
+## 💻 Usage
+
+```bash
+discogs run        # Auto: download → extract → convert
+discogs show       # List available Discogs data
+discogs download   # Just download selected files
+discogs extract    # Extract downloaded .gz files
+discogs convert    # Convert extracted XML to CSV
+discogs delete     # Delete files by selection or --all
+discogs config     # Set download folder
+```
+
+---
+
+## 📁 Folder Structure
+
+```
+~/Downloads/Discogs/
+├── .discogs_config.json
+└── Datasets/
+    └── 2025-04/
+        ├── discogs_20250401_artists.gz
+        ├── discogs_20250401_artists     ← .xml
+        └── discogs_20250401_artists.csv ← converted
+```
+
+---
+
+## 🧠 Example Workflow
+
+```bash
+discogs show
+# [1] 2025-04 | releases | 950 MB
+# [2] 2025-04 | artists  | 320 MB
+
+discogs download
+# Select 1,2
+# Downloads only
+
+discogs extract
+# Select file to extract
+
+discogs convert
+# Select XML to convert
+```
+
+---
+
+## 🧑‍💻 Author
+
+- GitHub: [github.com/ofurkancoban](https://github.com/ofurkancoban)
+- LinkedIn: [linkedin.com/in/ofurkancoban](https://linkedin.com/in/ofurkancoban)
+- Kaggle: [kaggle.com/ofurkancoban](https://www.kaggle.com/ofurkancoban)
+
+---
+
+## 📜 License
+
+MIT — use freely, mention when you do something cool 😎
+---
+
+Built with ❤️ by [@ofurkancoban](https://github.com/ofurkancoban)

discogsdataprocessorcli-1.5.0/README.md

@@ -0,0 +1,102 @@
+# 🎧 Discogs CLI — Data Processor Tool 💿
+<p align="center">
+  <img src="img/logo.png" alt="Discogs Logo" width="200"/>
+</p>
+A modern command-line tool to **download**, **extract**, and **convert** Discogs data dumps into structured CSV files.
+<p align="center">
+  <img src="img/preview.gif" />
+</p>
+
+---
+
+## 🚀 Features
+
+- 🧠 Scrape latest available data dump list from Discogs S3
+- ⬇️ Download `.gz` files for artists, labels, releases, masters
+- 📦 Extract `.gz` files to raw XML
+- ✂️ Chunk large XML into smaller files
+- 📄 Convert XML to clean, flat CSV files
+- 🗑 Delete selected or all files
+- ⚙️ Set custom download folder
+- 🧪 Easy to use from terminal with friendly UI
+
+---
+
+## 🧩 Installation
+
+### 🍻 Install with Homebrew
+```bash
+brew tap ofurkancoban/discogs
+brew install discogs
+```
+### or
+```bash
+git clone https://github.com/ofurkancoban/DiscogsCLI.git
+cd DiscogsCLI
+pip install -e .
+```
+
+---
+
+## 💻 Usage
+
+```bash
+discogs run        # Auto: download → extract → convert
+discogs show       # List available Discogs data
+discogs download   # Just download selected files
+discogs extract    # Extract downloaded .gz files
+discogs convert    # Convert extracted XML to CSV
+discogs delete     # Delete files by selection or --all
+discogs config     # Set download folder
+```
+
+---
+
+## 📁 Folder Structure
+
+```
+~/Downloads/Discogs/
+├── .discogs_config.json
+└── Datasets/
+    └── 2025-04/
+        ├── discogs_20250401_artists.gz
+        ├── discogs_20250401_artists     ← .xml
+        └── discogs_20250401_artists.csv ← converted
+```
+
+---
+
+## 🧠 Example Workflow
+
+```bash
+discogs show
+# [1] 2025-04 | releases | 950 MB
+# [2] 2025-04 | artists  | 320 MB
+
+discogs download
+# Select 1,2
+# Downloads only
+
+discogs extract
+# Select file to extract
+
+discogs convert
+# Select XML to convert
+```
+
+---
+
+## 🧑‍💻 Author
+
+- GitHub: [github.com/ofurkancoban](https://github.com/ofurkancoban)
+- LinkedIn: [linkedin.com/in/ofurkancoban](https://linkedin.com/in/ofurkancoban)
+- Kaggle: [kaggle.com/ofurkancoban](https://www.kaggle.com/ofurkancoban)
+
+---
+
+## 📜 License
+
+MIT — use freely, mention when you do something cool 😎
+---
+
+Built with ❤️ by [@ofurkancoban](https://github.com/ofurkancoban)

discogsdataprocessorcli-1.5.0/discogs/chunker.py

@@ -0,0 +1,95 @@
+# discogs/chunker.py
+
+import re
+from pathlib import Path
+from rich.console import Console
+from rich.progress import Progress, BarColumn, TimeElapsedColumn, TextColumn
+
+
+def sanitize_line(line: str) -> str:
+    """
+    Removes invalid XML characters and fixes unescaped ampersands.
+    This helps ensure the XML is well-formed before processing.
+    """
+    line = re.sub(r'[^\x09\x0A\x0D\x20-\uD7FF\uE000-\uFFFD]', '', line)  # Remove illegal XML characters
+    line = re.sub(r'&(?![a-zA-Z0-9#]+;)', '&', line)  # Escape unescaped '&' characters
+    return line
+
+
+def chunk_xml_by_type(xml_file: Path, content_type: str, records_per_file: int = 10000) -> Path:
+    """
+    Splits a large XML file into smaller, valid XML files (chunks).
+    Each chunk contains up to `records_per_file` XML records.
+    Returns the folder path where chunked files are stored.
+    """
+    record_tag = content_type[:-1].lower()  # e.g., "releases" → "release"
+    start_pat = re.compile(fr'<{record_tag}\b', re.IGNORECASE)  # Match opening tag
+    end_pat = re.compile(fr'</{record_tag}>', re.IGNORECASE)    # Match closing tag
+
+    chunk_folder = xml_file.parent / f"chunked_{content_type}"  # Output folder
+    chunk_folder.mkdir(parents=True, exist_ok=True)
+
+    console = Console()
+    chunk_count = 0
+    record_count = 0
+    inside_record = False
+    buffer_lines = []  # Stores lines of current XML record
+    current_chunk_file = None
+
+    # Helper function to open a new chunk file
+    def open_new_chunk():
+        nonlocal chunk_count, current_chunk_file, record_count
+        chunk_count += 1
+        chunk_path = chunk_folder / f"chunk_{chunk_count:05}.xml"
+        current_chunk_file = open(chunk_path, "w", encoding="utf-8")
+        current_chunk_file.write(f'<?xml version="1.0" encoding="utf-8"?>\n<{content_type}>\n')
+        record_count = 0
+
+    # Helper function to close the current chunk file
+    def close_chunk():
+        nonlocal current_chunk_file
+        if current_chunk_file:
+            current_chunk_file.write(f"</{content_type}>")
+            current_chunk_file.close()
+            current_chunk_file = None
+
+    open_new_chunk()
+
+    # Setup progress bar for visual feedback
+    with Progress(
+        TextColumn("[progress.description]{task.description}"),
+        BarColumn(),
+        "[progress.percentage]{task.percentage:.1f}%",
+        "•",
+        TimeElapsedColumn()
+    ) as progress:
+        task = progress.add_task(f"Chunking {xml_file.name}", total=xml_file.stat().st_size)
+
+        # Read and process XML file line by line
+        with xml_file.open("r", encoding="utf-8", errors="ignore") as f:
+            for raw_line in f:
+                line = sanitize_line(raw_line)
+                progress.update(task, advance=len(raw_line))
+
+                if not inside_record:
+                    # Detect start of a record
+                    if start_pat.search(line):
+                        inside_record = True
+                        buffer_lines = [line]
+                else:
+                    buffer_lines.append(line)
+                    if end_pat.search(line):
+                        # Write complete record to current chunk
+                        current_chunk_file.write("".join(buffer_lines) + "\n")
+                        record_count += 1
+                        inside_record = False
+                        buffer_lines = []
+
+                        # If chunk is full, start a new one
+                        if record_count >= records_per_file:
+                            close_chunk()
+                            open_new_chunk()
+
+    close_chunk()
+    console.print(f"[green]✔ Chunked into {chunk_count} file(s): {chunk_folder}")
+    return chunk_folder

discogsdataprocessorcli-1.5.0/discogs/config.py

@@ -0,0 +1,75 @@
+# discogs/config.py
+
+import json
+from pathlib import Path
+from rich.console import Console
+from rich.prompt import Prompt
+
+CONFIG_PATH = Path.home() / ".discogs_config.json"  # Path to the user's config file
+DEFAULT_DOWNLOAD_PATH = Path.home() / "Downloads" / "Discogs"  # Default download location
+
+console = Console()
+
+def get_download_dir() -> Path:
+    """
+    Returns the download directory from the config file if it exists,
+    otherwise returns the default download path.
+    """
+    if CONFIG_PATH.exists():
+        path = CONFIG_PATH.read_text().strip()
+        if path:
+            return Path(path)
+    return DEFAULT_DOWNLOAD_PATH
+
+def set_download_dir(path: str) -> None:
+    """
+    Sets (writes) the download directory path to the config file.
+    """
+    CONFIG_PATH.write_text(path.strip())
+
+def load_config() -> dict:
+    """
+    Loads configuration data from the config file.
+    If the config cannot be read, it returns a dictionary with default values.
+    """
+    if CONFIG_PATH.exists():
+        try:
+            with CONFIG_PATH.open("r") as f:
+                return json.load(f)
+        except Exception as e:
+            console.print(f"[red]Error reading config:[/] {e}")
+    return {"download_dir": str(DEFAULT_DOWNLOAD_PATH)}
+
+def save_config(config: dict) -> None:
+    """
+    Saves the given configuration dictionary to the config file.
+    Creates the parent directory if it doesn't exist.
+    """
+    CONFIG_PATH.parent.mkdir(parents=True, exist_ok=True)
+    with CONFIG_PATH.open("w") as f:
+        json.dump(config, f, indent=4)
+
+def get_download_dir() -> Path:
+    """
+    Gets the download directory from config, falling back to default if missing.
+    """
+    config = load_config()
+    return Path(config.get("download_dir", str(DEFAULT_DOWNLOAD_PATH)))
+
+def configure_download_folder() -> None:
+    """
+    Prompts the user to configure the download folder via CLI.
+    Updates the config if a new folder is provided.
+    """
+    console.print("[bold]Configure Discogs download folder[/bold]")
+    current = get_download_dir()
+    console.print(f"Current folder: [green]{current}[/green]")
+
+    new_path = Prompt.ask("Enter new folder path or leave empty to keep current", default=str(current)).strip()
+    if new_path:
+        new_path = Path(new_path).expanduser()
+        new_path.mkdir(parents=True, exist_ok=True)
+        save_config({"download_dir": str(new_path)})
+        console.print(f"[green]✅ Download folder updated to:[/] {new_path}")
+    else:
+        console.print("[yellow]No changes made.[/yellow]")