rcdl 2.2.2__py3-none-any.whl → 3.0.0b23__py3-none-any.whl

rcdl/interface/cli.py

@@ -1,10 +1,18 @@
 # interface/cli.py
 
+"""Hold all cli commands"""
+
 import logging
+import subprocess
+import sys
+import inspect
 
 import click
 
 from rcdl.core import downloader as dl
+from rcdl.interface.ui import UI, print_db_info
+from rcdl.core.fuse import fuse_medias
+from rcdl.core.opti import optimize
 from rcdl.core.config import Config
 from rcdl.core.parser import (
     get_creators,
@@ -13,9 +21,7 @@ from rcdl.core.parser import (
     append_creator,
 )
 from rcdl.core.db import DB
-from .ui import UI
-from rcdl.core.fuse import fuse_videos
-
+from rcdl.utils import clean_all
 
 from rcdl import __version__
 
@@ -33,27 +39,56 @@ def refresh():
     dl.refresh_creators_videos()
 
     with DB() as db:
-        info = db.get_db_videos_info()
-
-    UI.db_videos_status_table(info)
+        info = db.get_nb_per_status()
+    print_db_info(info)
 
 
 @click.command(help="Download all videos from all creator")
-def dlsf():
+@click.option(
+    "--max-fail-count",
+    type=int,
+    help="Set maximum number of failed attempts. Take precedence over config.toml",
+)
+def dlsf(max_fail_count: int | None):
     """Download video based on DB information
 
     - read databse
     - for each video NOT_DOWNLOADED or FAILED & fail_count < settings, dl video
     """
     UI.info("Welcome to RCDL dlsf")
-    dl.download_videos_to_be_dl()
+    dl.download_videos_to_be_dl(max_fail_count)
 
 
 @click.command("fuse", help="Fuse part video into one")
 def fuse():
     """Fuse videos"""
-    UI.info("fuse")
-    fuse_videos()
+    UI.info("Fuse/Concat video together")
+    fuse_medias()
+
+
+@click.command(help="Optimized video size")
+def opti():
+    """Optimized video size"""
+    UI.info("Optimized video")
+    optimize()
+
+
+@click.command(help="Clean patial download file, cache, etc...")
+@click.option("--all", is_flag=True, help="Act as if al lthe flags are true")
+@click.option(
+    "--partial", is_flag=True, help="Remove partial file from download (.aria2, .part)"
+)
+@click.option(
+    "--cache", is_flag=True, help="remove cache from yt-dlp and kill aria2 process"
+)
+@click.option(
+    "--medias-deleted",
+    is_flag=True,
+    help="Remove media marked for deletion (TO_BE_DELETED status)",
+)
+def clean(all: bool, partial: bool, cache: bool, medias_deleted: bool):
+    """Remove partial download, clear subprocesses cache"""
+    clean_all(all, partial, cache, medias_deleted)
 
 
 @click.command(help="Discover videos/creators with tags")
@@ -67,7 +102,7 @@ def discover(tag, max_page):
     msg = f"[cdl] discover with tag={tag} max_page={max_page}"
     click.echo(msg)
     logging.info(msg)
-    dl.discover(tag, max_page)
+    UI.info("WIP - UNIMPLEMENTED")
 
 
 @click.command("add", help="Add a creator")
@@ -85,8 +120,9 @@ def add_creator(creator_input):
 
 
 @click.command("remove", help="Remove a creator")
+@click.option("--db", is_flag=True)
 @click.argument("creator_input")
-def remove_creator(creator_input):
+def remove_creator(db, creator_input):
     """Remove a creator (excat line) from creators.txt"""
     _service, creator_id = parse_creator_input(str(creator_input))
 
@@ -94,7 +130,7 @@ def remove_creator(creator_input):
     all_creators = []
     matched_creator = None
     for creator in creators:
-        if creator.creator_id == creator_id:
+        if creator.id == creator_id:
             matched_creator = creator
             continue
         all_creators.append(creator)
@@ -102,35 +138,79 @@ def remove_creator(creator_input):
     if matched_creator is None:
         UI.error(f"Could not find creator from {creator_input}")
         return
-    else:
-        open(Config.CREATORS_FILE, "w").close()
-        for c in all_creators:
-            append_creator(c)
-        UI.info(
-            f"Removed creator {matched_creator.creator_id}@({matched_creator.service})"
+
+    try:
+        with open(Config.CREATORS_FILE, "w", encoding="utf-8"):
+            pass
+    except OSError as e:
+        UI.error(
+            f"Failed to create creators file at {Config.CREATORS_FILE} due to: {e}"
         )
+        return
+    for c in all_creators:
+        append_creator(c)
+    UI.info(f"Removed creator {matched_creator.id}@({matched_creator.service})")
+    if db:
+        UI.info("Not yet implemented")
 
 
 @click.command("list", help="List all creators")
 def list_creators():
+    """List all creators in creators.txt"""
     creators = get_creators()
     UI.table_creators(creators)
 
 
+@click.command("status", help="Print db info")
+def db_status():
+    """Print number of entry per status per tables in the database"""
+    with DB() as db:
+        info = db.get_nb_per_status()
+    print_db_info(info)
+
+
+@click.command("gui", help="Launch GUI")
+def launch_gui():
+    UI.info("Launching GUI...")
+    import importlib.resources as resources
+
+    gui_path = resources.files("rcdl.gui").joinpath("gui.py")
+    subprocess.run([sys.executable, "-m", "streamlit", "run", str(gui_path)])
+
+
+@click.command("show-config")
+def show_config():
+    """Show all value in Config"""
+    for name, value in vars(Config).items():
+        if not name.startswith("__") and not inspect.isroutine(value):
+            UI.success(f"{name}: {value}")
+
+
 # --- CLI GROUP ---
 @click.group()
 @click.option("--debug", is_flag=True)
 @click.option("--dry-run", is_flag=True)
 @click.version_option(version=__version__, prog_name=Config.APP_NAME)
 def cli(debug, dry_run):
+    """Init cli app. Assign Config var depending on flag used when calling prgm"""
     Config.set_debug(debug)
     Config.set_dry_run(dry_run)
 
 
+# main commands
 cli.add_command(dlsf)
 cli.add_command(discover)
 cli.add_command(refresh)
+cli.add_command(fuse)
+cli.add_command(opti)
+cli.add_command(launch_gui)
+
+# creators command
 cli.add_command(add_creator)
 cli.add_command(remove_creator)
 cli.add_command(list_creators)
-cli.add_command(fuse)
+
+# helper command
+cli.add_command(clean)
+cli.add_command(db_status)
+cli.add_command(show_config)

rcdl/interface/ui.py

@@ -1,6 +1,7 @@
 # interface/ui.py
 
 import logging
+import click
 from rich.console import Console, Group
 from rich.table import Table
 from rich.progress import (
@@ -9,11 +10,112 @@ from rich.progress import (
     BarColumn,
     TextColumn,
     TimeRemainingColumn,
+    TimeElapsedColumn,
 )
 from rich import box
 from rich.live import Live
 from rich.text import Text
-from rcdl.core.models import VideoStatus, Creator
+from rcdl.core.models import Creator, FusedStatus, Status
+
+
+class NestedProgress:
+    def __init__(self, console: Console):
+        self.console = console
+        self.global_progress: Progress | None = None
+        self.current_progress: Progress | None = None
+        self.live: Live | None = None
+
+        self.total_task: int | None = None
+        self.current_task: int | None = None
+        self.status_text = Text("", style="cyan")
+        self.current_label = ""
+
+    def start(
+        self, *, total: int, total_label: str = "Total", current_label: str = "Current"
+    ):
+        # Global progress (elapsed only)
+        self.global_progress = Progress(
+            SpinnerColumn(),
+            TextColumn("[bold cyan]{task.description}"),
+            BarColumn(),
+            TextColumn("{task.completed}/{task.total}"),
+            TimeElapsedColumn(),
+            console=self.console,
+            transient=False,
+        )
+
+        # Current task progress (ETA included)
+        self.current_progress = Progress(
+            SpinnerColumn(),
+            TextColumn("[bold cyan]{task.description}"),
+            BarColumn(),
+            TextColumn("{task.completed}/{task.total}"),
+            TimeRemainingColumn(),
+            console=self.console,
+            transient=False,
+        )
+
+        self.current_label = current_label
+        group = Group(self.global_progress, self.current_progress, self.status_text)
+        self.live = Live(group, console=self.console)
+        self.live.__enter__()
+
+        self.total_task = self.global_progress.add_task(total_label, total=total)
+        self.current_task = self.current_progress.add_task(
+            current_label, total=1, visible=False
+        )
+
+    # total task helpers
+    def advance_total(self, step: int = 1):
+        if self.global_progress and self.total_task is not None:
+            self.global_progress.advance(self.total_task, step)  # type: ignore
+
+    def reset_current(self):
+        if not self.current_progress or self.current_task is None:
+            return
+
+        self.current_progress.remove_task(self.current_task)  # type: ignore
+
+        self.current_task = self.current_progress.add_task(
+            self.current_label,
+            total=1,
+            visible=False,
+        )
+
+    def start_current(self, description: str, total: int | None = None):
+        if not self.current_progress or self.current_task is None:
+            return
+
+        self.reset_current()
+
+        self.current_progress.update(
+            self.current_task,  # type: ignore
+            description=description,
+            total=total or 1,
+            completed=0,
+            visible=True,
+        )
+
+    def advance_current(self, step: int | float = 1):
+        if self.current_progress and self.current_task is not None:
+            self.current_progress.advance(self.current_task, step)  # type: ignore
+
+    def finish_current(self):
+        if self.current_progress and self.current_task is not None:
+            self.current_progress.update(self.current_task, visible=False)  # type: ignore
+
+    # Status line
+    def set_status(self, cyan: str, green: str = ""):
+        self.status_text.plain = ""
+        self.status_text.append(cyan, style="cyan")
+        if green:
+            self.status_text.append(green, style="green")
+
+    # Close
+    def close(self):
+        if self.live:
+            self.live.__exit__(None, None, None)
+            self.live = None
 
 
 class UI:
@@ -23,6 +125,7 @@ class UI:
     _video_progress_text: Text | None = None
     _concat_progress_text: Text | None = None
     _live: Live | None = None
+    _generic_text: Text | None = None
 
     @staticmethod
     def _log_to_file(log_level, msg: str):
@@ -41,8 +144,8 @@ class UI:
 
     @classmethod
     def debug(cls, msg: str):
-        """Print & log debug msg"""
-        cls.console.print(f"[dim]{msg}[/]")
+        """Log debug msg"""
+        # cls.console.print(f"[dim]{msg}[/]")
         cls._log_to_file(cls.logger.debug, msg)
 
     @classmethod
@@ -57,26 +160,6 @@ class UI:
         cls.console.print(f"[red]{msg}[/]")
         cls._log_to_file(cls.logger.debug, msg)
 
-    @classmethod
-    def db_videos_status_table(cls, info: dict):
-        """
-        Print to cli a table with info of numbers of videos per status.
-        Take in arg a dict: {
-            "not_downloaded": int,
-            "downloaded": int, "failed: int", "skipped: int", "ignored: int"}
-        """
-
-        table = Table(title="DB Videos status")
-
-        table.add_column("Video status")
-        table.add_column("Number of videos")
-
-        for status in VideoStatus:
-            name = status.value.replace("_", " ").capitalize()
-            table.add_row(name, str(info[status.value]))
-
-        cls.console.print(table)
-
     @classmethod
     def table_creators(cls, creators: list[Creator]):
         """Print to cli a table with all creators in creators.txt. Format is Creator ID | Service"""
@@ -84,7 +167,7 @@ class UI:
         table.add_column("Creators ID")
         table.add_column("Service")
         for creator in creators:
-            table.add_row(creator.creator_id, creator.service)
+            table.add_row(creator.id, creator.service)
         cls.console.print(table)
 
     @classmethod
@@ -98,96 +181,14 @@ class UI:
         )
         return progress
 
-    @classmethod
-    def video_progress(cls, total: int):
-        """Create video download progress output"""
-        progress = Progress(
-            SpinnerColumn(),
-            TextColumn("[bold cyan]{task.description}"),
-            BarColumn(),
-            TextColumn("{task.completed}/{task.total}"),
-            TimeRemainingColumn(),
-            console=cls.console,
-            transient=False,  # remove the bar after completion
-        )
-
-        cls._video_progress_text = Text("Waiting...", style="Cyan")
-        group = Group(progress, cls._video_progress_text)
-        cls._live = Live(group, console=cls.console)
-        cls._live.__enter__()
-
-        task = progress.add_task("Downloading videos", total=total)
-        return progress, task
-
-    @classmethod
-    def set_current_video_progress(cls, creator_info: str, filename: str):
-        """Update video download output
-        args:
-            creator_info: str =  'creator_id@(service)'
-            filename: str = video.relative_path
-        """
-        if cls._video_progress_text is None:
-            return
-        cls._video_progress_text.plain = ""
-        cls._video_progress_text.append(f"{creator_info} -> ", style="Cyan")
-        cls._video_progress_text.append(filename, style="green")
-
-    @classmethod
-    def close_video_progress(cls):
-        """Close video progress"""
-        if cls._live:
-            cls._live.__exit__(None, None, None)
-            cls._live = None
-
-    @classmethod
-    def concat_progress(cls, total: int):
-        """Create concat progress bat"""
-        progress = Progress(
-            SpinnerColumn(),
-            TextColumn("[bold cyan]{task.description}"),
-            BarColumn(),
-            TextColumn("{task.completed}/{task.total}"),
-            TimeRemainingColumn(),
-            console=cls.console,
-            transient=False,  # remove the bar after completion
-        )
-
-        cls._concat_progress_text = Text("Waiting...", style="Cyan")
-        group = Group(progress, cls._concat_progress_text)
-        cls._live = Live(group, console=cls.console)
-        cls._live.__enter__()
-
-        task = progress.add_task("Concatenating videos", total=total)
-        return progress, task
-
-    @classmethod
-    def set_current_concat_progress(cls, msg: str, filename: str):
-        """Update video download output
-        args:
-            creator_info: str =  'creator_id@(service)'
-            filename: str = video.relative_path
-        """
-        if cls._concat_progress_text is None:
-            return
-        cls._concat_progress_text.plain = ""
-        cls._concat_progress_text.append(f"{msg} -> ", style="Cyan")
-        cls._concat_progress_text.append(filename, style="green")
-
-    @classmethod
-    def close_concat_progress(cls):
-        """Close video progress"""
-        if cls._live:
-            cls._live.__exit__(None, None, None)
-            cls._live = None
 
-    @classmethod
-    def progress_total_concat(cls):
-        progress = Progress(
-            SpinnerColumn(),
-            TextColumn("[progress.description]{task.description}"),
-            BarColumn(),
-            TextColumn("{task.completed}/{task.total}"),
-            console=cls.console,
-            transient=False,  # remove progress bar after finish
-        )
-        return progress
+def print_db_info(info: dict):
+    click.echo("--- TABLES ---")
+    click.echo("Posts:")
+    click.echo(f"\t{info['posts']} total")
+    click.echo("FusedMedias:")
+    for status in FusedStatus:
+        click.echo(f"\t{info['fuses'][status]} {status}")
+    click.echo("Medias:")
+    for status in Status:
+        click.echo(f"\t{info['medias'][status]} {status}")

rcdl/utils.py

@@ -1,11 +1,180 @@
 # utils.py
 
-import click
+"""Hold collection of useful functions"""
 
+from datetime import datetime, timezone
+import hashlib
+import json
+from pathlib import Path
+import os
+import time
 
-def echo(text: str):
-    click.echo(text)
+from rcdl.core.config import Config
+from rcdl.interface.ui import UI
 
 
-def echoc(text: str, color: str):
-    click.echo(click.style(text, fg=color))
+def get_date_now() -> str:
+    """Return an str of current datetime in isoformat"""
+    return datetime.now(timezone.utc).isoformat()
+
+
+def get_media_metadata(full_path: str) -> tuple[int, int, str]:
+    """Get duration (if possible), file_size and checksum"""
+    from rcdl.core.downloader_subprocess import ffprobe_get_duration
+
+    path = Path(full_path)
+    if not os.path.exists(path):
+        UI.error(f"{path} path should exist but does not.")
+
+    file_size = path.stat().st_size
+    checksum = get_media_hash(path)
+    if checksum is None:
+        checksum = ""
+    duration = ffprobe_get_duration(path)
+    if duration is None:
+        duration = 0
+    return duration, file_size, checksum
+
+
+def get_media_hash(path: Path, retries: int = Config.CHECKSUM_RETRY) -> str | None:
+    """Return a hash of a video file"""
+    for attempt in range(retries + 1):
+        try:
+            sha256_hash = hashlib.sha256()
+            with path.open("rb") as f:
+                for chunk in iter(lambda: f.read(8192), b""):
+                    sha256_hash.update(chunk)
+            checksum = sha256_hash.hexdigest()
+            return checksum
+        except OSError as e:
+            if attempt >= retries:
+                UI.error(
+                    f"OSError: Failed to checksum {path} "
+                    f"at attempt {attempt + 1} due to: {e}"
+                )
+                return None
+            UI.warning(
+                f"OSError: Failed to checksum {path} "
+                f"at attempt {attempt + 1} due to: {e}"
+            )
+            time.sleep(1.0)
+
+
+def get_json_hash(data: dict) -> tuple[str, str]:
+    """Return a hash of a dict"""
+    raw_json = json.dumps(data, sort_keys=True)
+    json_hash = hashlib.sha256(raw_json.encode("utf-8")).hexdigest()
+    return raw_json, json_hash
+
+
+def bytes_to_mb(bytes: float | int) -> float:
+    return bytes / (1024 * 1024)
+
+
+def bytes_to_str(bytes: float | int) -> str:
+    mb = bytes_to_mb(bytes)
+
+    if mb > 1024:
+        gb = mb / 1024
+        return f"{gb:.1f} GB"
+    return f"{mb:.1f} MB"
+
+
+def clean_all(all: bool, partial: bool, cache: bool, medias_deleted: bool):
+    """Remove partial file & external programm cache"""
+
+    from rcdl.core.db import DB
+    from rcdl.core.models import Status
+    from rcdl.core.downloader_subprocess import ytdlp_clear_cache, kill_aria2c
+
+    if all:
+        partial = True
+        cache = True
+        medias_deleted = True
+
+    # remove all partial file
+    if partial:
+        path = Config.BASE_DIR
+        folders = os.listdir(path)
+
+        for folder in folders:
+            if folder.startswith("."):
+                continue
+
+            folder_full_path = os.path.join(path, folder)
+            files = os.listdir(folder_full_path)
+
+            for file in files:
+                if (
+                    file.endswith(".aria2")
+                    or file.endswith(".part")
+                    or file.endswith(".opti.mp4")
+                ):
+                    full_path = os.path.join(folder_full_path, file)
+                    os.remove(full_path)
+                    UI.info(f"Removed {full_path}")
+
+    # cache
+    if cache:
+        # clear yt-dlp cache
+        ytdlp_clear_cache()
+        UI.info("Cleared yt-dlp cache dir")
+
+        # kill aria2c
+        kill_aria2c()
+        UI.info("Kill aria2c")
+
+    # medias with status deleted
+    if medias_deleted:
+        with DB() as db:
+            medias = db.query_media_by_status(Status.TO_BE_DELETED)
+        UI.info(f"Found {len(medias)} medias with DELETED status")
+
+        if len(medias) == 0:
+            return
+
+        total_size = 0.0
+        total_vid = 0
+        for media in medias:
+            with DB() as db:
+                post = db.query_post_by_id(media.post_id)
+            if post is None:
+                UI.warning(f"Could not match post id {media.post_id} to a post")
+                continue
+
+            path = os.path.join(Config.creator_folder(post.user), media.file_path)
+            if not os.path.exists:
+                UI.error(f"Path {path} should exist. Does not")
+                continue
+
+            try:
+                os.remove(path)
+                UI.info(f"Removed '{path}' ({bytes_to_mb(media.file_size)} MB)")
+                total_size += media.file_size
+                total_vid += 1
+            except Exception as e:
+                UI.error(f"Failed to rm media {media.post_id}/{media.url} due to: {e}")
+                continue
+
+            with DB() as db:
+                media.status = Status.DELETED
+                db.update_media(media)
+
+        UI.info(
+            f"Removed a total of {total_vid} videos ({bytes_to_mb(total_size)}MB,"
+            f" {round(bytes_to_mb(total_size) / 100, 1)}GB)"
+        )
+
+
+def format_seconds(seconds: int | float) -> str:
+    seconds = int(seconds)
+    h = seconds // 3600
+    m = (seconds % 3600) // 60
+    s = seconds % 60
+
+    if h > 0:
+        return f"{h}h:{m:02d}m:{s:02d}s"
+    elif m > 0:
+        return f"{m}m:{s:02d}s"
+    else:
+        return f"{s}s"