rcdl 2.2.2__py3-none-any.whl → 3.0.0b23__py3-none-any.whl

rcdl/__init__.py

@@ -1,5 +1,10 @@
 # __init__.py
 
+"""
+Init file for package structure.
+Get the version from pyproject.toml version tag
+"""
+
 from importlib.metadata import version
 
 __version__ = version("rcdl")

rcdl/__main__.py

@@ -1,16 +1,29 @@
 # __main__.py
 
+"""
+__main__: entry in the programm
+Setup logging, create files/folders structures, check dependencies,
+init Config global variables, init database,
+Then call cli group
+"""
+
 import logging
 
-from rcdl.core.config import Config, setup_logging
+from rcdl.core.config import Config, setup_logging, check_dependencies
 
 # setup file structure
 Config.ensure_dirs()
 Config.ensure_files()
 
+# load config file settings
+Config.load_config()
+
 # setup logging
 setup_logging(Config.LOG_FILE, level=0)
 
+# check dependencies
+check_dependencies()
+
 logging.info("--- INIT ---")
 logging.info("Logger initialized")
 
@@ -18,8 +31,7 @@ logging.info("Logger initialized")
 from rcdl.core.db import DB  # noqa: E402
 
 db = DB()
-db.init_table()
-logging.info(f"DB version: {db.get_schema_version()}")
+db.init_database()
 db.close()
 
 from rcdl.interface.cli import cli  # noqa: E402, F401

rcdl/core/adapters.py

@@ -0,0 +1,241 @@
+# core/adapters.py
+
+"""Convert [Any] into proper Models from models.py"""
+
+import json
+import sqlite3
+from dataclasses import fields
+
+from rcdl.core import parser
+from rcdl.interface.ui import UI
+from rcdl.core.models import Post, Media, Status, FusedMedia
+from rcdl.utils import get_date_now, get_json_hash
+
+VALID_POST_KEYS = set(
+    [
+        "id",
+        "user",
+        "service",
+        "title",
+        "substring",
+        "published",
+        "file",
+        "attachments",
+    ]
+)
+
+
+def _compute_json_metadata(raw: dict) -> tuple[str, str, str]:
+    """From a json dict, return:
+    - raw_json: str
+    - json_hash: str
+    - fetched_at str (datetime)
+    """
+    raw_json, json_hash = get_json_hash(raw)
+    fetched_at = get_date_now()
+    return raw_json, json_hash, fetched_at
+
+
+def json_posts_to_posts(posts: list[dict]) -> list[Post]:
+    """Convert a list of json post (dict) into a list of Post model
+    Ignore if conversion failed"""
+    formatted_posts = []
+    for post in posts:
+        p = json_post_to_post(post)
+        if p is not None:
+            formatted_posts.append(p)
+    return formatted_posts
+
+
+def json_post_to_post(post: dict) -> Post | None:
+    """Convert a json post (dict) into a Post model
+    or return None if covnersion failed"""
+    post_keys = set(post)
+    if post_keys != VALID_POST_KEYS:
+        UI.error(
+            f"Post id {post.get('id')} of {post.get('user')} "
+            f"has invalid schema. "
+            f"Missing: {VALID_POST_KEYS - post_keys}, "
+            f"Extra: {post_keys - VALID_POST_KEYS}"
+        )
+        return None
+
+    try:
+        domain = parser.get_domain(post["service"])
+        raw_json, json_hash, fetched_at = _compute_json_metadata(post)
+        return Post(
+            **post,
+            domain=domain,
+            json_hash=json_hash,
+            raw_json=raw_json,
+            fetched_at=fetched_at,
+        )
+    except TypeError as e:
+        UI.error(
+            f"Post id {post.get('id')} from {post.get('user')} could not be parsed: {e}"
+        )
+        return None
+
+
+def row_to_post(row: sqlite3.Row) -> Post | None:
+    """Convert a sqlite3 row into a Post model.
+    Return None if conversion failed"""
+    try:
+        raw = json.loads(row["raw_json"])
+        return Post(
+            id=row["id"],
+            user=row["user"],
+            service=row["service"],
+            domain=row["domain"],
+            published=row["published"],
+            json_hash=row["json_hash"],
+            raw_json=row["raw_json"],
+            fetched_at=row["fetched_at"],
+            title=raw["title"],
+            substring=raw["substring"],
+            file=raw["file"],
+            attachments=raw["attachments"],
+        )
+    except KeyError as e:
+        UI.error(
+            f"KeyError: Failed to convert {row['id']} (row_id) into Post model due to: {e}"
+        )
+        return None
+    except TypeError as e:
+        UI.error(
+            f"TypeError: Failed to convert {row['id']} (row_id) into Post model due to: {e}"
+        )
+        return None
+    except ValueError as e:
+        UI.error(
+            f"ValueError/JSONDecodeError: Failed to convert "
+            f"{row['id']} (row_id) into Post model due to: {e}"
+        )
+        return None
+
+
+def rows_to_posts(rows: list[sqlite3.Row]) -> list[Post]:
+    """Convert a list of sqlite3 rows. Return a list of Post model.
+    Ignore the row if conversion fail"""
+    posts: list[Post] = []
+    for row in rows:
+        post = row_to_post(row)
+        if post is not None:
+            posts.append(post)
+
+    if len(posts) != len(rows):
+        UI.error(
+            f"From {len(rows)} rows, only converted {len(posts)}."
+            f" {len(rows) - len(posts)} error."
+        )
+
+    return posts
+
+
+def row_to_media(row: sqlite3.Row) -> Media | None:
+    """Convert a sqlite3 row into a Media model.
+    Return None if conversion failed"""
+    try:
+        # create a dict to hold column of row that are present in Media.
+        # Ignore column (like default autoincrement ID) that are not a field in Media
+        media_data = {}
+        for field in fields(Media):
+            field_name = field.name
+            if field_name in row.keys():
+                value = row[field_name]
+                if field_name == "status" and value is not None:
+                    value = Status(value)
+                media_data[field_name] = value
+        return Media(**media_data)
+    except (KeyError, TypeError, ValueError) as e:
+        UI.error(
+            f"Key/Type/Value Error: Failed to convert row {row['id']} into Post model due to {e}"
+        )
+        return None
+
+
+def rows_to_medias(rows: list[sqlite3.Row]) -> list[Media]:
+    """Convert a list of sqlite3 rows. Return a list of Media model.
+    Ignore row if conversion failed"""
+    medias: list[Media] = []
+    for row in rows:
+        media = row_to_media(row)
+        if media is not None:
+            medias.append(media)
+
+    if len(medias) != len(rows):
+        UI.error(
+            f"From {len(rows)} rows, only converted {len(medias)}."
+            f" {len(rows) - len(medias)} error."
+        )
+
+    return medias
+
+
+def row_to_fused_media(row: sqlite3.Row) -> FusedMedia | None:
+    """Convert a sqlite3 row into a FusedMedia model.
+    Return None if conversion fail"""
+    if row is None:
+        return None
+    try:
+        fuses_data = {}
+        for field in fields(FusedMedia):
+            field_name = field.name
+            if field_name in row.keys():
+                value = row[field_name]
+                if field_name == "status" and value is not None:
+                    value = Status(value)
+                fuses_data[field_name] = value
+        return FusedMedia(**fuses_data)
+    except (KeyError, TypeError, ValueError) as e:
+        UI.error(
+            f"Key/Type/Value Error: Failed to convert row "
+            f"{row['id']} into FusedMedia model due to {e}"
+        )
+        return None
+
+
+def rows_to_fuses(rows: list[sqlite3.Row]) -> list[FusedMedia]:
+    """Convert a lsit of sqlite3 rows into a list of FusedMedia model
+    Ignore row if conversion failed"""
+    fuses: list[FusedMedia] = []
+    for row in rows:
+        fuse = row_to_fused_media(row)
+        if fuse is not None:
+            fuses.append(fuse)
+
+    if len(fuses) != len(rows):
+        UI.error(
+            f"From {len(rows)} rows, only converted {len(fuses)}."
+            f" {len(rows) - len(fuses)} error."
+        )
+
+    return fuses
+
+
+def post_to_videos(post: Post) -> list[Media]:
+    """Extract a list of Media model from a Post model"""
+    json_post = json.loads(post.raw_json)
+
+    urls = parser.extract_video_urls(json_post)
+    sequence = 0
+    medias: list[Media] = []
+    for url in urls:
+        medias.append(
+            Media(
+                post_id=post.id,
+                service=post.service,
+                url=url,
+                duration=0.0,
+                sequence=sequence,
+                status=Status.PENDING,
+                checksum="",
+                file_path=parser.get_filename(json_post, url),
+                created_at="",
+                updated_at="",
+                file_size=0,
+                fail_count=0,
+            )
+        )
+        sequence += 1
+    return medias

rcdl/core/api.py

@@ -1,9 +1,13 @@
 # core/api.py
 
-from .models import Creator
+"""Build real URL for api request"""
+
+from rcdl.core.models import Creator
 
 
 class URL:
+    """Build real URL for api request"""
+
     DOMAINS_BASE_URL = {
         "coomer": "https://coomer.st/api/v1/",
         "kemono": "https://kemono.cr/api/v1/",
@@ -11,34 +15,47 @@ class URL:
 
     @staticmethod
     def get_base_url(domain: str) -> str:
+        """Return https://domain.com"""
         if domain not in URL.DOMAINS_BASE_URL:
             raise KeyError(f"{domain} not in known domains urls")
         return URL.DOMAINS_BASE_URL[domain]
 
     @staticmethod
     def get_post_revision(creator: Creator, post_id) -> str:
-        return f"{URL.get_base_url(creator.domain)}{creator.service}/user/{creator.creator_id}/post/{post_id}/revisions"
+        """Return post revision url"""
+        return (
+            f"{URL.get_base_url(creator.domain)}{creator.service}"
+            f"/user/{creator.id}/post/{post_id}/revisions"
+        )
 
     @staticmethod
     def get_headers() -> dict:
+        """Return necessary request header for successful request"""
         return {
-            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0 Safari/537.36",
+            "User-Agent": (
+                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+                "AppleWebKit/537.36 (KHTML, like Gecko) "
+                "Chrome/117.0 Safari/537.36"
+            ),
             "Accept": "text/css",
         }
 
     @staticmethod
     def get_url_from_file(domain: str, path_url: str):
+        """Add path_url to based domain url"""
         if domain == "coomer":
             return f"https://coomer.st{path_url}"
-        elif domain == "kemono":
+        if domain == "kemono":
             return f"https://kemono.cr{path_url}"
-        else:
-            raise ValueError(
-                f"Domain {domain} is not an accepted value/does not exist. Please check your creators.json file"
-            )
+
+        raise ValueError(
+            f"Domain {domain} is not an accepted value/does not exist. "
+            f"Please check your creators.json file"
+        )
 
     @staticmethod
     def add_params(url: str, params: dict):
+        """Create all parameters string (key=params&key=...)"""
         url += "?"
         for key in params:
             url += f"{key}={params[key]}&"
@@ -46,9 +63,14 @@ class URL:
 
     @staticmethod
     def get_creator_post_wo_param(creator: Creator) -> str:
-        return f"{URL.get_base_url(creator.domain)}{creator.service}/user/{creator.creator_id}/posts"
+        """Get creator post without parameters"""
+        return (
+            f"{URL.get_base_url(creator.domain)}{creator.service}"
+            f"/user/{creator.id}/posts"
+        )
 
     @staticmethod
     def get_posts_page_url_wo_param():
+        """Get posts page without parameters -> use in tag search"""
         domain = URL.DOMAINS_BASE_URL["coomer"]
         return f"{domain}posts"

rcdl/core/config.py

@@ -1,13 +1,22 @@
 # core/config.py
 
+"""
+Handle Config init, dependencies check, logging setup,
+files and folders structures, config settings parameters init
+"""
+
 from pathlib import Path
 import logging
 import os
+import tomllib
+import subprocess
 
-from .file_io import write_txt
+from rcdl.core.file_io import write_txt
 
 
 class Config:
+    """Global app var/parameters"""
+
     # paths
     APP_NAME = "rcdl"
 
@@ -16,62 +25,103 @@ class Config:
     CACHE_DIR = BASE_DIR / ".cache"
     DB_PATH = CACHE_DIR / "cdl.db"
     LOG_FILE = CACHE_DIR / "cdl.log"
-    FUSE_CSV_FILE = CACHE_DIR / "cdl_fuse.csv"
     CREATORS_FILE = CACHE_DIR / "creators.txt"
     DISCOVER_DIR = CACHE_DIR / "discover"
-
-    # default creators
-    DEFAULT_CREATORS = ["boixd/onlyfans"]
+    CONFIG_FILE = CACHE_DIR / "config.toml"
 
     DEBUG = False
     DRY_RUN = False
 
     # api settings
-    POST_PER_PAGE = 50
-    DEFAULT_MAX_PAGE = 10
-    MAX_FAIL_COUNT = 7
+    POST_PER_PAGE: int = 50
+    DEFAULT_MAX_PAGE: int = 10
+    MAX_FAIL_COUNT: int = 7
+    TIMEOUT: int = 10
+
+    # fuse settings
+    MAX_WIDTH: int = 1920
+    MAX_HEIGHT: int = 1080
+    FPS: int = 30
+    PRESET: str = "veryfast"
+    THREADS: int = 0
+
+    HANDBRAKE_RUN_CMD = "HandBrakeCLI"
+
+    CHECKSUM_RETRY = 2
 
     @classmethod
     def ensure_dirs(cls):
+        """Ensure directory exist"""
         cls.CACHE_DIR.mkdir(parents=True, exist_ok=True)
         cls.DISCOVER_DIR.mkdir(exist_ok=True)
 
     @classmethod
     def ensure_files(cls):
-        files = [
-            cls.DB_PATH,
-            cls.FUSE_CSV_FILE,
-            cls.CREATORS_FILE,
-        ]
+        """Ensure file exist, populate default if necessary"""
+        files = [cls.DB_PATH, cls.CREATORS_FILE, cls.CONFIG_FILE]
         for file in files:
             if not file.exists():
                 file.touch()
                 logging.info("Created file %s", file)
                 if file == cls.CREATORS_FILE:
-                    write_txt(cls.CREATORS_FILE, cls.DEFAULT_CREATORS, mode="w")
+                    write_txt(cls.CREATORS_FILE, DEFAULT_CREATORS, mode="w")
+                if file == cls.CONFIG_FILE:
+                    write_txt(cls.CONFIG_FILE, DEFAULT_CONFIG, mode="w")
 
     @classmethod
     def creator_folder(cls, creator_id: str) -> Path:
+        """Return creator folder path base on user/creator_id"""
         folder = cls.BASE_DIR / creator_id
         folder.mkdir(exist_ok=True)
         return folder
 
     @classmethod
     def cache_file(cls, filename: str, ext: str = ".json") -> Path:
+        """Return filepath of a file in the .cache/ folder"""
         file_name = filename + ext
         file = cls.CACHE_DIR / file_name
         return file
 
     @classmethod
     def set_debug(cls, debug: bool):
+        """Set class variable DEBUG"""
         cls.DEBUG = debug
 
     @classmethod
     def set_dry_run(cls, dry_run: bool):
+        """Set class variable DRY_RUN"""
         cls.DRY_RUN = dry_run
 
+    @classmethod
+    def load_config(cls):
+        """Load config.toml and set class var with value in config.toml"""
+        with open(cls.CONFIG_FILE, "rb") as f:
+            data = tomllib.load(f)
+        app = data.get("app", {})
+        cls.DEFAULT_MAX_PAGE = app.get("default_max_page", cls.DEFAULT_MAX_PAGE)
+        cls.MAX_FAIL_COUNT = app.get("max_fail_count", cls.MAX_FAIL_COUNT)
+        cls.TIMEOUT = app.get("timeout", cls.TIMEOUT)
+        cls.CHECKSUM_RETRY = app.get("checksum_retry", cls.CHECKSUM_RETRY)
+
+        video = data.get("video", {})
+        cls.MAX_WIDTH = video.get("max_width", cls.MAX_WIDTH)
+        cls.MAX_HEIGHT = video.get("max_height", cls.MAX_HEIGHT)
+        cls.FPS = video.get("fps", cls.FPS)
+        cls.PRESET = video.get("preset", cls.PRESET)
+        cls.THREADS = video.get("threads", cls.THREADS)
+
+        paths = data.get("paths", {})
+        if "base_dir" in paths:
+            cls.BASE_DIR = Path(
+                os.environ.get("RCDL_BASE_DIR", os.path.expanduser(paths["base_dir"]))
+            )
+            cls.CACHE_DIR = cls.BASE_DIR / ".cache"
+        if "handbrake_run_cmd" in paths:
+            cls.HANDBRAKE_RUN_CMD = paths.get("handbrake_run_cmd")
+
 
 def setup_logging(log_file: Path, level: int = 0):
+    """Setup logging for rcdl"""
     logger = logging.getLogger()
     logger.setLevel(level)
     logger.handlers.clear()  # avoid double handlers if called multiple times
@@ -91,3 +141,72 @@ def setup_logging(log_file: Path, level: int = 0):
     stream = logging.StreamHandler()
     stream.setLevel(logging.ERROR)  # only show warnings/errors from libraries
     logger.addHandler(stream)
+
+
+def check_dependencies():
+    """Check external program version against last tested working version"""
+    for prgrm, info in DEPENDENCIES_TEST_VERSION.items():
+        try:
+            result = subprocess.run(
+                info["cmd"],
+                capture_output=True,
+                text=True,
+                shell=True,
+                check=False,
+            )
+            version = result.stdout.strip()
+
+            if version != info["version"]:
+                print(
+                    f"Last tested version for {prgrm}:"
+                    f" {info['version']} -> yours: {version}"
+                )
+                if version == "":
+                    print(f"{prgrm} is not installed.")
+                print(f"Check {prgrm} is installed if your version is empty.")
+        except (OSError, subprocess.SubprocessError) as e:
+            print(
+                f"Failed to check {prgrm} version due to: {e}\nCheck {prgrm} is installed."
+            )
+
+
+DEPENDENCIES_TEST_VERSION = {
+    "yt-dlp": {"cmd": "yt-dlp --version", "version": "2025.12.08"},
+    "aria2c": {
+        "cmd": "aria2c -v | head -n 1",
+        "version": "aria2 version 1.37.0",
+    },
+    "ffmpeg": {
+        "cmd": 'ffmpeg -version | sed -n "s/ffmpeg version \\([-0-9.]*\\).*/\\1/p;"',
+        "version": "7.1.1-1",
+    },
+    "handbrake": {
+        "cmd": Config.HANDBRAKE_RUN_CMD
+        + ' --version 2>&1 | sed -n "s/HandBrake \\([0-9.]*\\).*/\\1/p"',
+        "version": "1.9.2",
+    },
+}
+
+
+# default creators
+DEFAULT_CREATORS = ["boixd/onlyfans"]
+
+# default config params
+DEFAULT_CONFIG: str = """\
+[app]
+default_max_page = 10
+max_fail_count = 7
+timeout = 10
+checksum_retry = 2
+
+[fuse]
+max_width = 1920
+max_height = 1080
+fps = 30
+preset = "veryfast"
+threads = 0
+
+[paths]
+base_dir = "~/Videos/rcdl"
+handbrake_run_cmd = "HandBrakeCLI"
+"""