rcdl 3.0.0b18__py3-none-any.whl

rcdl/core/downloader_subprocess.py

@@ -0,0 +1,366 @@
+# core/downloader_subprocess.py
+
+"""
+Handle all subprocess call to external tool (yt-dlp, ffmpeg, ...)
+"""
+
+import subprocess
+import logging
+from pathlib import Path
+import os
+
+
+from rcdl.interface.ui import UI, NestedProgress
+from rcdl.core import parser
+from rcdl.core.models import Media, Post
+from rcdl.core.config import Config
+from rcdl.utils import bytes_to_str
+
+
+def ytdlp_clear_cache():
+    """Clear yt-dlp cache"""
+    cmd = ["yt-dlp", "--rm-cache-dir"]
+    subprocess.run(cmd, check=False)
+
+
+def kill_aria2c():
+    """Kill all aria2c process"""
+    cmd = ["pkill", "-f", "aria2c"]
+    subprocess.run(cmd, check=False)
+
+
+def ytdlp_subprocess(
+    url: str,
+    filepath: Path | str,
+):
+    """Call yt-dlp in a subprocess to download a video"""
+
+    cmd = [
+        "yt-dlp",
+        "-q",
+        "--progress",
+        url,
+        "-o",
+        filepath,
+        "--external-downloader",
+        "aria2c",
+    ]
+
+    logging.info("CMD: %s", " ".join(cmd))
+
+    result = subprocess.run(cmd, capture_output=True, text=True, check=False)
+    if result.returncode != 0:
+        logging.error("yt-dlp failed to dl vid: %s", result.stderr)
+
+    return result.returncode
+
+
+def ffprobe_get_duration(path: Path) -> int | None:
+    """Get duration of a video in seconds with ffprobe
+    Return an int or None if command failed"""
+    cmd = [
+        "ffprobe",
+        "-v",
+        "error",
+        "-show_entries",
+        "format=duration",
+        "-of",
+        "default=noprint_wrappers=1:nokey=1",
+        str(path),
+    ]
+    try:
+        result = subprocess.run(
+            cmd,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.DEVNULL,
+            text=True,
+            check=True,
+        )
+        return int(float(result.stdout.strip()))
+    except subprocess.CalledProcessError as e:
+        UI.error(f"Failed to use ffprobe on {path} due to {e}")
+        return None
+    except (AttributeError, ValueError, OverflowError) as e:
+        UI.error(f"Failed to parse duration result of {path} due to {e}")
+        return None
+
+
+def get_max_width_height(medias: list[Media], post: Post) -> tuple[int, int]:
+    """Get width and height of all media in list. Return max within video found and config"""
+
+    cmd = [
+        "ffprobe",
+        "-v",
+        "error",
+        "-select_streams",
+        "v:0",
+        "-show_entries",
+        "stream=width,height",
+        "-of",
+        "csv=p=0",
+    ]
+    width = 0
+    height = 0
+    max_width = 1920
+    max_height = 1080
+    for m in medias:
+        path = os.path.join(Config.creator_folder(post.user), m.file_path)
+        full_cmd = cmd + [path]
+
+        try:
+            result = subprocess.run(
+                full_cmd, capture_output=True, text=True, check=True
+            )
+            w_str, h_str = result.stdout.strip().split(",")
+
+            width = min(int(w_str), max_width)
+            height = min(int(h_str), max_height)
+        except subprocess.CalledProcessError as e:
+            UI.error(f"Fail to use ffprobe to get width, height on {path} due to {e}")
+        except (AttributeError, ValueError, OverflowError) as e:
+            UI.error(f"Failed to parse duration for {path} due to {e}")
+    return (width, height)
+
+
+def get_total_duration(medias: list[Media], post: Post) -> int:
+    """Get total duration in ms of all medias in list"""
+
+    def _get_duration(path: str) -> int:
+        """Get video duration in ms"""
+
+        cmd = [
+            "ffprobe",
+            "-v",
+            "error",
+            "-select_streams",
+            "v:0",
+            "-show_entries",
+            "format=duration",
+            "-of",
+            "default=noprint_wrappers=1:nokey=1",
+            path,
+        ]
+
+        result = subprocess.run(cmd, capture_output=True, text=True, check=True)
+        return int(float(result.stdout.strip()) * 1000)
+
+    duration = 0
+    for m in medias:
+        path = os.path.join(Config.creator_folder(post.user), m.file_path)
+        duration += _get_duration(path)
+    return duration
+
+
+def ffmpeg_concat_build_command(medias: list[Media], post: Post) -> dict:
+    """Build the ffmpeg concat command"""
+
+    width, height = get_max_width_height(medias, post)
+    logging.info("Found (%s, %s) (width, height) for this group.", width, height)
+    if width == 0:
+        width = Config.MAX_WIDTH
+    if height == 0:
+        height = Config.MAX_HEIGHT
+
+    # output path
+    output_filename = parser.get_filename_fuse(post)
+    output_path = os.path.join(Config.creator_folder(post.user), output_filename)
+
+    # build cmd
+    cmd = ["ffmpeg", "-y", "-progress", "pipe:2", "-nostats"]
+
+    # inputs
+    for media in medias:
+        input_path = os.path.join(Config.creator_folder(post.user), media.file_path)
+        cmd.extend(["-i", input_path])
+
+    # filter complex
+    filter_lines = []
+    for idx in range(len(medias)):
+        filter_lines.append(
+            f"[{idx}:v]"
+            f"scale={width}:{height}:force_original_aspect_ratio=decrease,"
+            f"pad={width}:{height}:(ow-iw)/2:(oh-ih)/2,"
+            f"fps={Config.FPS},setsar=1"
+            f"[v{idx}]"
+        )
+
+    # concat inputs
+    concat = []
+    for idx in range(len(medias)):
+        concat.append(f"[v{idx}][{idx}:a]")
+
+    filter_lines.append(f"{''.join(concat)}concat=n={len(medias)}:v=1:a=1[outv][outa]")
+    filter_complex = ";".join(filter_lines)
+
+    cmd.extend(
+        [
+            "-filter_complex",
+            filter_complex,
+            "-map",
+            "[outv]",
+            "-map",
+            "[outa]",
+            "-c:v",
+            "libx264",
+            "-preset",
+            Config.PRESET,
+            "-threads",
+            str(Config.THREADS),
+            "-c:a",
+            "aac",
+            "-movflags",
+            "+faststart",
+            output_path,
+        ]
+    )
+
+    return {"cmd": cmd, "output_path": output_path}
+
+
+def parse_line_ffmpeg_concat_into_advance(line: str) -> int | None:
+    line = line.strip()
+    if not line:
+        return None
+
+    progres_key = "out_time_ms"
+    if line.startswith(progres_key):
+        current_progress_str = line.replace(f"{progres_key}=", "").strip()
+        try:
+            current_progress_us = int(current_progress_str)
+            current_progress_ms = current_progress_us // 1000
+            return current_progress_ms
+        except ValueError as e:
+            logging.warning(
+                "Skipping invalid progress line: %r (%s)",
+                current_progress_str,
+                e,
+            )
+            return None
+        except Exception as e:
+            UI.error(f"Unexpected error while updating progress: {e}")
+            return None
+    return None
+
+
+def ffmpeg_concat(medias: list[Media], post: Post, progress: NestedProgress):
+    """Run ffmpeg concat command to merge video together"""
+
+    command_builder = ffmpeg_concat_build_command(medias, post)
+    cmd = command_builder["cmd"]
+
+    logging.info("CMD: %s", " ".join(cmd))
+
+    ffmpeg_log = Config.CACHE_DIR / "ffmpeg.log"
+    with open(ffmpeg_log, "w", encoding="utf-8") as log_file:
+        print(cmd, file=log_file)
+        # run cmd
+        process = subprocess.Popen(
+            cmd,
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.PIPE,
+            text=True,
+            bufsize=1,
+        )
+
+        assert process.stderr is not None
+        total_duration = get_total_duration(medias, post)
+        progress.start_current(
+            description=f"{post.user}->{medias[0].file_path}", total=total_duration
+        )
+
+        last_progress = 0
+
+        for line in process.stderr:
+            line = line.strip()
+            print(line, file=log_file)
+            current_progress_ms = parse_line_ffmpeg_concat_into_advance(line)
+            if current_progress_ms is None:
+                continue
+            delta = current_progress_ms - last_progress
+            progress.advance_current(step=delta)
+            last_progress = current_progress_ms
+
+        process.wait()
+        progress.finish_current()
+
+    UI.debug(f"Result: {process.returncode}")
+    if process.returncode != 0:
+        UI.error(f"Failed to concat videos. See ffmpeg log file {ffmpeg_log}")
+        with open(ffmpeg_log, "r", encoding="utf-8") as f:
+            lines = f.read()
+        logging.warning("---FFMPEG LOG---")
+        logging.warning(lines)
+        logging.warning("---END FFMPEG LOG---")
+        return process.returncode
+
+    return 0
+
+
+def parse_line_into_pourcent(line: str) -> float | None:
+    line = line.strip()
+    if not line:
+        return None
+
+    if "%" in line:
+        try:
+            parts = line.split("%")
+            parts = parts[0].strip().split(" ")
+            pourcent = parts[-1]
+            flt_prcnt = float(pourcent)
+            return flt_prcnt
+        except Exception as e:
+            UI.error(f"Error parsing line {line}: {e}")
+            return None
+    return None
+
+
+def handbrake_optimized(media: Media, user: str, progress: NestedProgress):
+    """Optimize video size with handbrake software"""
+
+    handbrake_process = Config.HANDBRAKE_RUN_CMD.split(" ")
+
+    folder_path = Config.creator_folder(user)
+    video_path = os.path.join(folder_path, media.file_path)
+
+    output_path = video_path + ".opti.mp4"
+
+    cmd = ["-i", video_path, "-o", output_path, "--preset", "HQ 1080p30 Surround"]
+
+    full_cmd = handbrake_process + cmd
+    UI.debug(f"Running cmd '{full_cmd}'")
+
+    # -- process
+    process = subprocess.Popen(
+        full_cmd,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.DEVNULL,
+        text=True,
+    )
+
+    assert process.stdout is not None
+    progress.start_current(description="Optimizing", total=100)
+    progress.set_status(
+        f"{user}@({media.service}) -> ",
+        f"{media.file_path} ({bytes_to_str(media.file_size)})",
+    )
+
+    current_progress = 0.0
+
+    for line in process.stdout:
+        float_pourcent = parse_line_into_pourcent(line)
+        if float_pourcent is None:
+            continue
+        delta = float_pourcent - current_progress
+        current_progress = float_pourcent
+        progress.advance_current(step=delta)
+
+    process.wait()
+    progress.finish_current()
+    # -- end process
+
+    if process.returncode == 0:
+        UI.debug("Return code: 0")
+    else:
+        UI.error(f"Return code: {process.returncode}")
+
+    return process.returncode

rcdl/core/file_io.py

@@ -0,0 +1,41 @@
+# core/file_io.py
+
+"""All write/read to file function (excluding sqlite database)"""
+
+import json
+
+
+def write_json(path, data, mode="w"):
+    """Write dict data to json"""
+    with open(path, mode, encoding="utf-8") as f:
+        json.dump(data, f, indent=4)
+
+
+def load_json(path) -> dict:
+    """Load data from json"""
+    with open(path, "r", encoding="utf-8") as f:
+        data = json.load(f)
+    return data
+
+
+def load_txt(path) -> list[str]:
+    """Read text from a .txt file.
+    Return list of stripped lines"""
+    with open(path, "r", encoding="utf-8") as f:
+        lines = f.readlines()
+    for i, line in enumerate(lines):
+        lines[i] = line.strip()
+    return lines
+
+
+def write_txt(path, lines: list[str] | str, mode: str = "a"):
+    """Write txt to .txt file"""
+    if isinstance(lines, str):
+        lines = [lines]
+
+    with open(path, mode, encoding="utf-8") as f:
+        for line in lines:
+            if not line.endswith("\n"):
+                f.write(line + "\n")
+            else:
+                f.write(line)

rcdl/core/fuse.py

@@ -0,0 +1,127 @@
+# core/fuse.py
+
+"""Handle merging videos from a same post"""
+
+import os
+import subprocess
+
+from rcdl.core.config import Config
+from rcdl.core.db import DB
+from rcdl.core.models import FusedStatus, Status, FusedMedia, Media, Post
+from rcdl.interface.ui import UI, NestedProgress
+import rcdl.core.downloader_subprocess as dls
+from rcdl.utils import get_media_metadata, get_date_now
+
+
+def update_db(fuse: FusedMedia, medias: list[Media], user: str, result):
+    """Update DB depending on subprocess result (SUCESS/FAILURE)"""
+    if result == 0:
+        path = os.path.join(Config.creator_folder(user), fuse.file_path)
+        duration, file_size, checksum = get_media_metadata(path)
+        fuse.duration = duration
+        fuse.status = FusedStatus.FUSED
+        fuse.checksum = checksum
+        fuse.created_at = get_date_now()
+        fuse.file_size = file_size
+        for media in medias:
+            media.status = Status.FUSED
+    else:
+        fuse.fail_count += 1
+    with DB() as db:
+        db.update_fuse(fuse)
+        for media in medias:
+            db.update_media(media)
+
+
+def get_medias_and_post(
+    post_id: str, total_parts: int
+) -> tuple[None, None] | tuple[list[Media], Post]:
+    """Get medias and post related to a fuse group.
+    Return a list[Media] and a Post
+    Handle Errors, return None, None"""
+    # get associated post
+    with DB() as db:
+        post = db.query_post_by_id(post_id)
+    if post is None:
+        UI.error(f"Could not match fuses post id {post_id} to a post in post tables")
+        return None, None
+
+    # get all videos of a post
+    with DB() as db:
+        medias = db.query_media_by_post_id(post_id)
+
+    # check number of media in db match total part expected in fused media
+    if len(medias) != total_parts:
+        UI.error(f"Found {len(medias)} videos part. Expected {total_parts}")
+        return None, None
+
+    # check all video are downloaded
+    allowed_status = [Status.DOWNLOADED, Status.OPTIMIZED]
+    if Config.DEBUG:
+        allowed_status.append(Status.FUSED)
+    ok = True
+    for media in medias:
+        if media.status not in allowed_status:
+            ok = False
+            break
+    if not ok:
+        return None, None
+
+    # sort medias list
+    sorted_medias = sorted(medias, key=lambda m: m.sequence)
+    return sorted_medias, post
+
+
+def fuse_medias():
+    """Fuse all media part of a fuse group with status PENDING in DB fuses"""
+    # get all fused media
+    with DB() as db:
+        fuses = db.query_fuses_by_status(FusedStatus.PENDING)
+    if Config.DEBUG:
+        with DB() as db:
+            ok_fuses = db.query_fuses_by_status(FusedStatus.FUSED)
+        fuses.extend(ok_fuses)
+
+    progress = NestedProgress(UI.console)
+    progress.start(
+        total=len(fuses), total_label="Fusing videos", current_label="Current fuse"
+    )
+
+    for fm in fuses:
+        medias, post = get_medias_and_post(fm.id, fm.total_parts)
+        if medias is None or post is None:
+            progress.advance_total()
+            continue
+
+        # concat medias
+        result = 1
+        try:
+            result = dls.ffmpeg_concat(medias, post, progress)
+        except (OSError, subprocess.SubprocessError, ValueError) as e:
+            UI.error(f"Failed to concat video (id:{post.id}) due to: {e}")
+
+        # update db
+        update_db(fm, medias, post.user, result)
+
+        progress.advance_total()
+
+        # remove part file
+        for media in medias:
+            media_full_path = os.path.join(
+                Config.creator_folder(post.user), media.file_path
+            )
+            try:
+                if Config.DEBUG:
+                    UI.info(f"Skipped '{media_full_path}' removal")
+                    continue
+                os.remove(media_full_path)
+                UI.info(f"Removed file '{media_full_path}'")
+            except (FileNotFoundError, PermissionError) as e:
+                UI.error(
+                    f"FileNotFound/PermissionError: Failed to "
+                    f"remove media '{media_full_path}' due to: {e}"
+                )
+            except OSError as e:
+                UI.error(f"Failed to remove media '{media_full_path}' due to: {e}")
+
+    progress.close()

rcdl/core/models.py

@@ -0,0 +1,105 @@
+# core/models.py
+
+"""Hold all dataclass models and enums"""
+
+from dataclasses import dataclass
+from enum import Enum
+
+
+class Status(Enum):
+    """Status for media"""
+
+    PENDING = "pending"  # to be downloaded
+    DOWNLOADED = "downloaded"  # video has been downloaded
+    FUSED = "fused"  # video has been fused, and impliitly removed
+    TO_BE_DELETED = "to_be_delete"  # video has been marked for delete
+    DELETED = "deleted"  # video has been deleted
+    OPTIMIZED = "optimized"  # video has been optimized (reduce file size)
+
+
+class FusedStatus(Enum):
+    """Status for fused group"""
+
+    PENDING = "pending"
+    FUSED = "fused"
+
+
+class CreatorStatus(Enum):
+    FAVORITED = "FAVORITED"
+    NA = "NA"
+
+
+@dataclass
+class Post:
+    """Post model that shadow post dict response of request
+    Partially used in posts db (check db_queries.py)
+    """
+
+    id: str
+    user: str
+    service: str
+    domain: str
+    title: str
+    substring: str
+    published: str
+    file: dict
+    attachments: list
+    json_hash: str
+    raw_json: str
+    fetched_at: str
+
+
+@dataclass
+class Media:
+    """Media model: use in medias DB"""
+
+    post_id: str
+    service: str
+    url: str
+    duration: float
+    sequence: int
+    status: Status
+    checksum: str
+    file_path: str
+    created_at: str
+    updated_at: str
+    file_size: int
+    fail_count: int = 0
+
+
+@dataclass
+class FusedMedia:
+    """Fuses group models.
+    Used in fuses db."""
+
+    id: str
+    duration: int
+    total_parts: int
+    status: FusedStatus
+    checksum: str
+    file_path: str
+    created_at: str
+    updated_at: str
+    file_size: int
+    fail_count: int = 0
+
+
+@dataclass
+class Creator:
+    """Creator model"""
+
+    id: str
+    name: str
+    service: str
+    domain: str
+    indexed: str
+    updated: str
+    favorited: int
+
+    status: CreatorStatus
+
+    # param
+    max_size: int
+    max_posts: int
+    min_date: str
+    max_date: str