rcdl 3.0.0b18__py3-none-any.whl

rcdl/interface/cli.py

@@ -0,0 +1,216 @@
+# interface/cli.py
+
+"""Hold all cli commands"""
+
+import logging
+import subprocess
+import sys
+import inspect
+
+import click
+
+from rcdl.core import downloader as dl
+from rcdl.interface.ui import UI, print_db_info
+from rcdl.core.fuse import fuse_medias
+from rcdl.core.opti import optimize
+from rcdl.core.config import Config
+from rcdl.core.parser import (
+    get_creators,
+    get_creator_from_line,
+    parse_creator_input,
+    append_creator,
+)
+from rcdl.core.db import DB
+from rcdl.utils import clean_all
+
+from rcdl import __version__
+
+
+@click.command(help="Refresh video to be downloaded")
+def refresh():
+    """Refresh database with creators videos
+
+    - get all creators from creators.txt
+    - for each creators find all videos and put them in the database
+    No download is done in this command
+    """
+
+    UI.info("Welcome to RCDL refresh")
+    dl.refresh_creators_videos()
+
+    with DB() as db:
+        info = db.get_nb_per_status()
+    print_db_info(info)
+
+
+@click.command(help="Download all videos from all creator")
+@click.option(
+    "--max-fail-count",
+    type=int,
+    help="Set maximum number of failed attempts. Take precedence over config.toml",
+)
+def dlsf(max_fail_count: int | None):
+    """Download video based on DB information
+
+    - read databse
+    - for each video NOT_DOWNLOADED or FAILED & fail_count < settings, dl video
+    """
+    UI.info("Welcome to RCDL dlsf")
+    dl.download_videos_to_be_dl(max_fail_count)
+
+
+@click.command("fuse", help="Fuse part video into one")
+def fuse():
+    """Fuse videos"""
+    UI.info("Fuse/Concat video together")
+    fuse_medias()
+
+
+@click.command(help="Optimized video size")
+def opti():
+    """Optimized video size"""
+    UI.info("Optimized video")
+    optimize()
+
+
+@click.command(help="Clean patial download file, cache, etc...")
+@click.option("--all", is_flag=True, help="Act as if al lthe flags are true")
+@click.option(
+    "--partial", is_flag=True, help="Remove partial file from download (.aria2, .part)"
+)
+@click.option(
+    "--cache", is_flag=True, help="remove cache from yt-dlp and kill aria2 process"
+)
+@click.option(
+    "--medias-deleted",
+    is_flag=True,
+    help="Remove media marked for deletion (TO_BE_DELETED status)",
+)
+def clean(all: bool, partial: bool, cache: bool, medias_deleted: bool):
+    """Remove partial download, clear subprocesses cache"""
+    clean_all(all, partial, cache, medias_deleted)
+
+
+@click.command(help="Discover videos/creators with tags")
+@click.option("--tag", required=True, type=str, help="Tag to search for")
+@click.option(
+    "--max-page", default=10, type=int, help="Maximum number of pages to fetch"
+)
+def discover(tag, max_page):
+    """Discover new creators/videos
+    currently WIP. Do not use in prod"""
+    msg = f"[cdl] discover with tag={tag} max_page={max_page}"
+    click.echo(msg)
+    logging.info(msg)
+    UI.info("WIP - UNIMPLEMENTED")
+
+
+@click.command("add", help="Add a creator")
+@click.argument("creator_input")
+def add_creator(creator_input):
+    """Add a creator (URL or str) to creators.txt"""
+    service, creator_id = parse_creator_input(creator_input)
+    line = f"{service}/{creator_id}"
+    creator = get_creator_from_line(line)
+    if creator is not None:
+        append_creator(creator)
+        UI.info(f"Added {line} to creators.txt")
+    else:
+        UI.warning("Could not extract creator from input. Please check input is valid")
+
+
+@click.command("remove", help="Remove a creator")
+@click.option("--db", is_flag=True)
+@click.argument("creator_input")
+def remove_creator(db, creator_input):
+    """Remove a creator (excat line) from creators.txt"""
+    _service, creator_id = parse_creator_input(str(creator_input))
+
+    creators = get_creators()
+    all_creators = []
+    matched_creator = None
+    for creator in creators:
+        if creator.id == creator_id:
+            matched_creator = creator
+            continue
+        all_creators.append(creator)
+
+    if matched_creator is None:
+        UI.error(f"Could not find creator from {creator_input}")
+        return
+
+    try:
+        with open(Config.CREATORS_FILE, "w", encoding="utf-8"):
+            pass
+    except OSError as e:
+        UI.error(
+            f"Failed to create creators file at {Config.CREATORS_FILE} due to: {e}"
+        )
+        return
+    for c in all_creators:
+        append_creator(c)
+    UI.info(f"Removed creator {matched_creator.id}@({matched_creator.service})")
+    if db:
+        UI.info("Not yet implemented")
+
+
+@click.command("list", help="List all creators")
+def list_creators():
+    """List all creators in creators.txt"""
+    creators = get_creators()
+    UI.table_creators(creators)
+
+
+@click.command("status", help="Print db info")
+def db_status():
+    """Print number of entry per status per tables in the database"""
+    with DB() as db:
+        info = db.get_nb_per_status()
+    print_db_info(info)
+
+
+@click.command("gui", help="Launch GUI")
+def launch_gui():
+    UI.info("Launching GUI...")
+    import importlib.resources as resources
+
+    gui_path = resources.files("rcdl.gui").joinpath("gui.py")
+    subprocess.run([sys.executable, "-m", "streamlit", "run", str(gui_path)])
+
+
+@click.command("show-config")
+def show_config():
+    """Show all value in Config"""
+    for name, value in vars(Config).items():
+        if not name.startswith("__") and not inspect.isroutine(value):
+            UI.success(f"{name}: {value}")
+
+
+# --- CLI GROUP ---
+@click.group()
+@click.option("--debug", is_flag=True)
+@click.option("--dry-run", is_flag=True)
+@click.version_option(version=__version__, prog_name=Config.APP_NAME)
+def cli(debug, dry_run):
+    """Init cli app. Assign Config var depending on flag used when calling prgm"""
+    Config.set_debug(debug)
+    Config.set_dry_run(dry_run)
+
+
+# main commands
+cli.add_command(dlsf)
+cli.add_command(discover)
+cli.add_command(refresh)
+cli.add_command(fuse)
+cli.add_command(opti)
+cli.add_command(launch_gui)
+
+# creators command
+cli.add_command(add_creator)
+cli.add_command(remove_creator)
+cli.add_command(list_creators)
+
+# helper command
+cli.add_command(clean)
+cli.add_command(db_status)
+cli.add_command(show_config)

rcdl/interface/ui.py

@@ -0,0 +1,194 @@
+# interface/ui.py
+
+import logging
+import click
+from rich.console import Console, Group
+from rich.table import Table
+from rich.progress import (
+    Progress,
+    SpinnerColumn,
+    BarColumn,
+    TextColumn,
+    TimeRemainingColumn,
+    TimeElapsedColumn,
+)
+from rich import box
+from rich.live import Live
+from rich.text import Text
+from rcdl.core.models import Creator, FusedStatus, Status
+
+
+class NestedProgress:
+    def __init__(self, console: Console):
+        self.console = console
+        self.global_progress: Progress | None = None
+        self.current_progress: Progress | None = None
+        self.live: Live | None = None
+
+        self.total_task: int | None = None
+        self.current_task: int | None = None
+        self.status_text = Text("", style="cyan")
+        self.current_label = ""
+
+    def start(
+        self, *, total: int, total_label: str = "Total", current_label: str = "Current"
+    ):
+        # Global progress (elapsed only)
+        self.global_progress = Progress(
+            SpinnerColumn(),
+            TextColumn("[bold cyan]{task.description}"),
+            BarColumn(),
+            TextColumn("{task.completed}/{task.total}"),
+            TimeElapsedColumn(),
+            console=self.console,
+            transient=False,
+        )
+
+        # Current task progress (ETA included)
+        self.current_progress = Progress(
+            SpinnerColumn(),
+            TextColumn("[bold cyan]{task.description}"),
+            BarColumn(),
+            TextColumn("{task.completed}/{task.total}"),
+            TimeRemainingColumn(),
+            console=self.console,
+            transient=False,
+        )
+
+        self.current_label = current_label
+        group = Group(self.global_progress, self.current_progress, self.status_text)
+        self.live = Live(group, console=self.console)
+        self.live.__enter__()
+
+        self.total_task = self.global_progress.add_task(total_label, total=total)
+        self.current_task = self.current_progress.add_task(
+            current_label, total=1, visible=False
+        )
+
+    # total task helpers
+    def advance_total(self, step: int = 1):
+        if self.global_progress and self.total_task is not None:
+            self.global_progress.advance(self.total_task, step)  # type: ignore
+
+    def reset_current(self):
+        if not self.current_progress or self.current_task is None:
+            return
+
+        self.current_progress.remove_task(self.current_task)  # type: ignore
+
+        self.current_task = self.current_progress.add_task(
+            self.current_label,
+            total=1,
+            visible=False,
+        )
+
+    def start_current(self, description: str, total: int | None = None):
+        if not self.current_progress or self.current_task is None:
+            return
+
+        self.reset_current()
+
+        self.current_progress.update(
+            self.current_task,  # type: ignore
+            description=description,
+            total=total or 1,
+            completed=0,
+            visible=True,
+        )
+
+    def advance_current(self, step: int | float = 1):
+        if self.current_progress and self.current_task is not None:
+            self.current_progress.advance(self.current_task, step)  # type: ignore
+
+    def finish_current(self):
+        if self.current_progress and self.current_task is not None:
+            self.current_progress.update(self.current_task, visible=False)  # type: ignore
+
+    # Status line
+    def set_status(self, cyan: str, green: str = ""):
+        self.status_text.plain = ""
+        self.status_text.append(cyan, style="cyan")
+        if green:
+            self.status_text.append(green, style="green")
+
+    # Close
+    def close(self):
+        if self.live:
+            self.live.__exit__(None, None, None)
+            self.live = None
+
+
+class UI:
+    console = Console()
+    logger = logging.getLogger()
+
+    _video_progress_text: Text | None = None
+    _concat_progress_text: Text | None = None
+    _live: Live | None = None
+    _generic_text: Text | None = None
+
+    @staticmethod
+    def _log_to_file(log_level, msg: str):
+        log_level(msg)
+
+    @classmethod
+    def success(cls, msg: str):
+        """Print success msg"""
+        cls.console.print(f"[green]{msg}[/]")
+
+    @classmethod
+    def info(cls, msg: str):
+        """Print & log info msg"""
+        cls.console.print(msg)
+        cls._log_to_file(cls.logger.info, msg)
+
+    @classmethod
+    def debug(cls, msg: str):
+        """Log debug msg"""
+        # cls.console.print(f"[dim]{msg}[/]")
+        cls._log_to_file(cls.logger.debug, msg)
+
+    @classmethod
+    def warning(cls, msg: str):
+        """Print & log warning msg"""
+        cls.console.print(f"[yellow]{msg}[/]")
+        cls._log_to_file(cls.logger.debug, msg)
+
+    @classmethod
+    def error(cls, msg: str):
+        """Print & log error msg"""
+        cls.console.print(f"[red]{msg}[/]")
+        cls._log_to_file(cls.logger.debug, msg)
+
+    @classmethod
+    def table_creators(cls, creators: list[Creator]):
+        """Print to cli a table with all creators in creators.txt. Format is Creator ID | Service"""
+        table = Table(title="Creators", box=box.MINIMAL, show_lines=True)
+        table.add_column("Creators ID")
+        table.add_column("Service")
+        for creator in creators:
+            table.add_row(creator.id, creator.service)
+        cls.console.print(table)
+
+    @classmethod
+    def progress_posts_fetcher(cls, max_pages: int):
+        progress = Progress(
+            SpinnerColumn(),
+            TextColumn("[progress.description]{task.description}"),
+            BarColumn(),
+            console=cls.console,
+            transient=False,  # remove progress bar after finish
+        )
+        return progress
+
+
+def print_db_info(info: dict):
+    click.echo("--- TABLES ---")
+    click.echo("Posts:")
+    click.echo(f"\t{info['posts']} total")
+    click.echo("FusedMedias:")
+    for status in FusedStatus:
+        click.echo(f"\t{info['fuses'][status]} {status}")
+    click.echo("Medias:")
+    for status in Status:
+        click.echo(f"\t{info['medias'][status]} {status}")

rcdl/utils.py

@@ -0,0 +1,180 @@
+# utils.py
+
+"""Hold collection of useful functions"""
+
+from datetime import datetime, timezone
+import hashlib
+import json
+from pathlib import Path
+import os
+import time
+
+from rcdl.core.config import Config
+from rcdl.interface.ui import UI
+
+
+def get_date_now() -> str:
+    """Return an str of current datetime in isoformat"""
+    return datetime.now(timezone.utc).isoformat()
+
+
+def get_media_metadata(full_path: str) -> tuple[int, int, str]:
+    """Get duration (if possible), file_size and checksum"""
+    from rcdl.core.downloader_subprocess import ffprobe_get_duration
+
+    path = Path(full_path)
+    if not os.path.exists(path):
+        UI.error(f"{path} path should exist but does not.")
+
+    file_size = path.stat().st_size
+    checksum = get_media_hash(path)
+    if checksum is None:
+        checksum = ""
+    duration = ffprobe_get_duration(path)
+    if duration is None:
+        duration = 0
+    return duration, file_size, checksum
+
+
+def get_media_hash(path: Path, retries: int = Config.CHECKSUM_RETRY) -> str | None:
+    """Return a hash of a video file"""
+    for attempt in range(retries + 1):
+        try:
+            sha256_hash = hashlib.sha256()
+            with path.open("rb") as f:
+                for chunk in iter(lambda: f.read(8192), b""):
+                    sha256_hash.update(chunk)
+            checksum = sha256_hash.hexdigest()
+            return checksum
+        except OSError as e:
+            if attempt >= retries:
+                UI.error(
+                    f"OSError: Failed to checksum {path} "
+                    f"at attempt {attempt + 1} due to: {e}"
+                )
+                return None
+            UI.warning(
+                f"OSError: Failed to checksum {path} "
+                f"at attempt {attempt + 1} due to: {e}"
+            )
+            time.sleep(1.0)
+
+
+def get_json_hash(data: dict) -> tuple[str, str]:
+    """Return a hash of a dict"""
+    raw_json = json.dumps(data, sort_keys=True)
+    json_hash = hashlib.sha256(raw_json.encode("utf-8")).hexdigest()
+    return raw_json, json_hash
+
+
+def bytes_to_mb(bytes: float | int) -> float:
+    return bytes / (1024 * 1024)
+
+
+def bytes_to_str(bytes: float | int) -> str:
+    mb = bytes_to_mb(bytes)
+
+    if mb > 1024:
+        gb = mb / 1024
+        return f"{gb:.1f} GB"
+    return f"{mb:.1f} MB"
+
+
+def clean_all(all: bool, partial: bool, cache: bool, medias_deleted: bool):
+    """Remove partial file & external programm cache"""
+
+    from rcdl.core.db import DB
+    from rcdl.core.models import Status
+    from rcdl.core.downloader_subprocess import ytdlp_clear_cache, kill_aria2c
+
+    if all:
+        partial = True
+        cache = True
+        medias_deleted = True
+
+    # remove all partial file
+    if partial:
+        path = Config.BASE_DIR
+        folders = os.listdir(path)
+
+        for folder in folders:
+            if folder.startswith("."):
+                continue
+
+            folder_full_path = os.path.join(path, folder)
+            files = os.listdir(folder_full_path)
+
+            for file in files:
+                if (
+                    file.endswith(".aria2")
+                    or file.endswith(".part")
+                    or file.endswith(".opti.mp4")
+                ):
+                    full_path = os.path.join(folder_full_path, file)
+                    os.remove(full_path)
+                    UI.info(f"Removed {full_path}")
+
+    # cache
+    if cache:
+        # clear yt-dlp cache
+        ytdlp_clear_cache()
+        UI.info("Cleared yt-dlp cache dir")
+
+        # kill aria2c
+        kill_aria2c()
+        UI.info("Kill aria2c")
+
+    # medias with status deleted
+    if medias_deleted:
+        with DB() as db:
+            medias = db.query_media_by_status(Status.TO_BE_DELETED)
+        UI.info(f"Found {len(medias)} medias with DELETED status")
+
+        if len(medias) == 0:
+            return
+
+        total_size = 0.0
+        total_vid = 0
+        for media in medias:
+            with DB() as db:
+                post = db.query_post_by_id(media.post_id)
+            if post is None:
+                UI.warning(f"Could not match post id {media.post_id} to a post")
+                continue
+
+            path = os.path.join(Config.creator_folder(post.user), media.file_path)
+            if not os.path.exists:
+                UI.error(f"Path {path} should exist. Does not")
+                continue
+
+            try:
+                os.remove(path)
+                UI.info(f"Removed '{path}' ({bytes_to_mb(media.file_size)} MB)")
+                total_size += media.file_size
+                total_vid += 1
+            except Exception as e:
+                UI.error(f"Failed to rm media {media.post_id}/{media.url} due to: {e}")
+                continue
+
+            with DB() as db:
+                media.status = Status.DELETED
+                db.update_media(media)
+
+        UI.info(
+            f"Removed a total of {total_vid} videos ({bytes_to_mb(total_size)}MB,"
+            f" {round(bytes_to_mb(total_size) / 100, 1)}GB)"
+        )
+
+
+def format_seconds(seconds: int | float) -> str:
+    seconds = int(seconds)
+    h = seconds // 3600
+    m = (seconds % 3600) // 60
+    s = seconds % 60
+
+    if h > 0:
+        return f"{h}h:{m:02d}m:{s:02d}s"
+    elif m > 0:
+        return f"{m}m:{s:02d}s"
+    else:
+        return f"{s}s"

rcdl-3.0.0b18.dist-info/METADATA

@@ -0,0 +1,122 @@
+Metadata-Version: 2.4
+Name: rcdl
+Version: 3.0.0b18
+Summary: Coomer/Kemono CLI Downloader
+Keywords: downloader,video,media
+Author: Anonymous
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Dist: click>=8.2
+Requires-Dist: requests>=2.32
+Requires-Dist: pathvalidate==3.3.1
+Requires-Dist: rich==14.2.0
+Requires-Dist: streamlit==1.52.2
+
+# RCDL
+
+Riton Coomer Download Manager  
+`rcdl` is a tool to automatically download the videos of your favorites creators from [coomer.st](https://coomer.st) and [kemono.cr](https://kemono.cr)
+
+
+## Install
+### Dependencies
+- [yt-dlp](https://github.com/yt-dlp/yt-dlp)
+- [aria2](https://github.com/aria2/aria2)
+- [ffmpeg](https://www.ffmpeg.org/download.html) (Only for `fuse` command)  
+- [HandBrakeCLI](https://handbrake.fr/docs/en/latest/cli/cli-options.html) (Only for `opti` command)
+Recommended install:
+```bash
+pipx install yt-dlp streamlit
+sudo apt update
+sudo apt install aria2 ffmpeg handbrake-cli python3-tk
+```
+### Install RCDL
+It is recommended to use `pipx` to install `rcdl`
+```bash
+pipx install rcdl
+```
+or else:  
+```bash
+pip install rcdl
+```
+
+## How to use
+
+Run the CLI with:
+
+```bash
+rcdl --help
+```
+
+By default all files will live in `~/Videos/rcdl/`. Cache, configuration and log file will be in a hidden `rcdl/.cache/` folder.
+
+Main function:  
+```bash
+rcdl refresh    # look creators.json and find all possible videos
+rcdl dlsf       # download all found videos
+rcdl discover   # Discover new creator (WIP)
+rcdl opti       # Optimized video to reduce disk storage usage
+rcdl fuse       # Fuse all videos within a same post if they are fully downloaded
+```
+
+Manage creators:  
+```bash
+rcdl list   # list all current creators
+rcdl add [URL]
+rcdl add [service]/[creator_id]
+rcdl remove [creator_id]
+```
+
+Helper function:  
+```bash
+rcdl status     # give number of entry in the database per tables and status
+rcdl clean --all      # remove all partially downloaded file, external dependencies cache, etc...
+rcdl show-config        # print all config var and theirs value (paths, etc...)
+```
+
+### Settings
+Default settings file:
+```toml
+[app]
+default_max_page = 10
+max_fail_count = 7
+timeout = 10
+
+[fuse]
+max_width = 1920
+max_height = 1080
+fps = 30
+preset = "veryfast"
+threads = 0
+
+[paths]
+base_dir = "~/Videos/rcdl"
+handbrake_run_cmd = "HandBrakeCLI"
+```
+
+In `rcdl/.cache/config.toml`:
+```toml
+[paths]
+handbrake_run_cmd = "HandBrakeCLI"  # if installed via apt
+handbrake_run_cmd = "flatpak run --command=HandBrakeCLI fr.handbrake.ghb"   # if installed via flatpak
+```
+
+## Dev
+### Install
+```bash
+git clone https://github.com/ritonun/cdl.git rcdl
+cd rcdl
+python3 -m venv .venv
+source .venv/bin/activate
+pip install -e .
+```
+
+## Deploy
+```bash
+python3 -m pip install --upgrade build
+python3 -m pip install --upgrade twine
+```
+