rcdl 2.2.2__py3-none-any.whl

rcdl/__init__.py

@@ -0,0 +1,5 @@
+# __init__.py
+
+from importlib.metadata import version
+
+__version__ = version("rcdl")

rcdl/__main__.py

@@ -0,0 +1,25 @@
+# __main__.py
+
+import logging
+
+from rcdl.core.config import Config, setup_logging
+
+# setup file structure
+Config.ensure_dirs()
+Config.ensure_files()
+
+# setup logging
+setup_logging(Config.LOG_FILE, level=0)
+
+logging.info("--- INIT ---")
+logging.info("Logger initialized")
+
+# init database
+from rcdl.core.db import DB  # noqa: E402
+
+db = DB()
+db.init_table()
+logging.info(f"DB version: {db.get_schema_version()}")
+db.close()
+
+from rcdl.interface.cli import cli  # noqa: E402, F401

rcdl/core/api.py

@@ -0,0 +1,54 @@
+# core/api.py
+
+from .models import Creator
+
+
+class URL:
+    DOMAINS_BASE_URL = {
+        "coomer": "https://coomer.st/api/v1/",
+        "kemono": "https://kemono.cr/api/v1/",
+    }
+
+    @staticmethod
+    def get_base_url(domain: str) -> str:
+        if domain not in URL.DOMAINS_BASE_URL:
+            raise KeyError(f"{domain} not in known domains urls")
+        return URL.DOMAINS_BASE_URL[domain]
+
+    @staticmethod
+    def get_post_revision(creator: Creator, post_id) -> str:
+        return f"{URL.get_base_url(creator.domain)}{creator.service}/user/{creator.creator_id}/post/{post_id}/revisions"
+
+    @staticmethod
+    def get_headers() -> dict:
+        return {
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0 Safari/537.36",
+            "Accept": "text/css",
+        }
+
+    @staticmethod
+    def get_url_from_file(domain: str, path_url: str):
+        if domain == "coomer":
+            return f"https://coomer.st{path_url}"
+        elif domain == "kemono":
+            return f"https://kemono.cr{path_url}"
+        else:
+            raise ValueError(
+                f"Domain {domain} is not an accepted value/does not exist. Please check your creators.json file"
+            )
+
+    @staticmethod
+    def add_params(url: str, params: dict):
+        url += "?"
+        for key in params:
+            url += f"{key}={params[key]}&"
+        return url[:-1]
+
+    @staticmethod
+    def get_creator_post_wo_param(creator: Creator) -> str:
+        return f"{URL.get_base_url(creator.domain)}{creator.service}/user/{creator.creator_id}/posts"
+
+    @staticmethod
+    def get_posts_page_url_wo_param():
+        domain = URL.DOMAINS_BASE_URL["coomer"]
+        return f"{domain}posts"

rcdl/core/config.py

@@ -0,0 +1,93 @@
+# core/config.py
+
+from pathlib import Path
+import logging
+import os
+
+from .file_io import write_txt
+
+
+class Config:
+    # paths
+    APP_NAME = "rcdl"
+
+    BASE_DIR = Path(os.environ.get("RCDL_BASE_DIR", Path.home() / "Videos/rcdl"))
+
+    CACHE_DIR = BASE_DIR / ".cache"
+    DB_PATH = CACHE_DIR / "cdl.db"
+    LOG_FILE = CACHE_DIR / "cdl.log"
+    FUSE_CSV_FILE = CACHE_DIR / "cdl_fuse.csv"
+    CREATORS_FILE = CACHE_DIR / "creators.txt"
+    DISCOVER_DIR = CACHE_DIR / "discover"
+
+    # default creators
+    DEFAULT_CREATORS = ["boixd/onlyfans"]
+
+    DEBUG = False
+    DRY_RUN = False
+
+    # api settings
+    POST_PER_PAGE = 50
+    DEFAULT_MAX_PAGE = 10
+    MAX_FAIL_COUNT = 7
+
+    @classmethod
+    def ensure_dirs(cls):
+        cls.CACHE_DIR.mkdir(parents=True, exist_ok=True)
+        cls.DISCOVER_DIR.mkdir(exist_ok=True)
+
+    @classmethod
+    def ensure_files(cls):
+        files = [
+            cls.DB_PATH,
+            cls.FUSE_CSV_FILE,
+            cls.CREATORS_FILE,
+        ]
+        for file in files:
+            if not file.exists():
+                file.touch()
+                logging.info("Created file %s", file)
+                if file == cls.CREATORS_FILE:
+                    write_txt(cls.CREATORS_FILE, cls.DEFAULT_CREATORS, mode="w")
+
+    @classmethod
+    def creator_folder(cls, creator_id: str) -> Path:
+        folder = cls.BASE_DIR / creator_id
+        folder.mkdir(exist_ok=True)
+        return folder
+
+    @classmethod
+    def cache_file(cls, filename: str, ext: str = ".json") -> Path:
+        file_name = filename + ext
+        file = cls.CACHE_DIR / file_name
+        return file
+
+    @classmethod
+    def set_debug(cls, debug: bool):
+        cls.DEBUG = debug
+
+    @classmethod
+    def set_dry_run(cls, dry_run: bool):
+        cls.DRY_RUN = dry_run
+
+
+def setup_logging(log_file: Path, level: int = 0):
+    logger = logging.getLogger()
+    logger.setLevel(level)
+    logger.handlers.clear()  # avoid double handlers if called multiple times
+
+    # loggin format & file handler
+    file_handler = logging.FileHandler(log_file, encoding="utf-8", mode="a")
+    file_handler.setFormatter(
+        logging.Formatter(
+            "{asctime} - {levelname} - {message}",
+            style="{",
+            datefmt="%Y-%m-%d %H:%M:%S",
+        )
+    )
+    logger.addHandler(file_handler)
+
+    # log library warning/errors
+    stream = logging.StreamHandler()
+    stream.setLevel(logging.ERROR)  # only show warnings/errors from libraries
+    logger.addHandler(stream)

rcdl/core/db.py

@@ -0,0 +1,235 @@
+# core/db.py
+
+"""Handle SQL DB and DB Parsing"""
+
+import sqlite3
+import logging
+from typing import Iterable
+
+
+import rcdl.core.db_queries as queries
+from .config import Config
+from .models import Video, VideoStatus
+
+
+class DB:
+    def __init__(self):
+        self.conn = sqlite3.connect(Config.DB_PATH)
+        self.conn.row_factory = sqlite3.Row
+        self.conn.execute("PRAGMA foreign_keys = ON")  # check input into db
+        self.conn.execute("PRAGMA journal_mode = WAL")  #
+        self.conn.execute("PRAGMA synchronous = NORMAL")  # faster write speed
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.close()
+
+    def init_table(self):
+        # init table for videos to DL
+        self.conn.execute(queries.CREATE_VIDEOS_TABLE)
+        self.conn.execute(queries.CREATE_IDX_VIDEOS_STATUS)
+        self.conn.execute(queries.CREATE_IDX_VIDEOS_CREATOR)
+        self.conn.execute(queries.CREATE_IDX_VIDEOS_FAIL_COUNT)
+
+        # init table for version schem, easy migration if necessary
+        self.conn.execute(queries.CREATE_SCHEMA_VERSION_TABLE)
+
+        # Initialize version if empty
+        cur = self.conn.cursor()
+        cur.execute("SELECT COUNT(*) as cnt FROM schema_version")
+        row = cur.fetchone()
+        if row["cnt"] == 0:
+            cur.execute("INSERT INTO schema_version (version) VALUES (?)", (1,))
+
+        self.conn.commit()
+
+    def get_schema_version(self) -> int:
+        # get current db version -> for future update to db
+        cur = self.conn.cursor()
+        cur.execute("SELECT version FROM schema_version")
+        row = cur.fetchone()
+        return row["version"] if row else 0
+
+    def set_schema_version(self, version: int):
+        # set current db version
+        self.conn.execute("UPDATE schema_version SET version = ?", (version,))
+        self.conn.commit()
+
+    def _row_to_video(self, row: sqlite3.Row) -> Video:
+        # helper function to retunr Video model from a sql result
+        return Video(
+            post_id=row["post_id"],
+            creator_id=row["creator_id"],
+            service=row["service"],
+            domain=row["domain"],
+            relative_path=row["relative_path"],
+            url=row["url"],
+            part=row["part"],
+            status=VideoStatus(row["status"]),
+            fail_count=row["fail_count"],
+            published=row["published"],
+            title=row["title"],
+            substring=row["substring"],
+            downloaded_at=row["downloaded_at"],
+            file_size=row["file_size"],
+        )
+
+    def query_videos(
+        self,
+        *,
+        status: VideoStatus | Iterable[VideoStatus] | None = None,
+        creator_id: str | None = None,
+        post_id: str | None = None,
+        max_fail_count: int | None = None,
+        min_part_number: int | None = None,
+        pending: bool = False,
+    ) -> list[Video]:
+        """
+        Query the DB.
+        Parametes are 'AND' so if video status and creator_id are sepcified,
+        it will look for a match between the two
+
+        pending:
+        No parameters are taken into account if pending True.
+        Look for videos with: NOT DOWNLOADED status OR (FAILED & fail_count < Config.max_fail_count)
+        """
+        sql = "SELECT * FROM videos"
+        conditions = []
+        params = []
+
+        if pending:
+            sql += " WHERE status = ? OR (status = ? AND fail_count < ?)"
+            params.extend(
+                [
+                    VideoStatus.NOT_DOWNLOADED.value,
+                    VideoStatus.FAILED.value,
+                    max_fail_count or Config.MAX_FAIL_COUNT,
+                ]
+            )
+        else:
+            if status is not None:
+                if isinstance(status, VideoStatus):
+                    conditions.append("status = ?")
+                    params.append(status.value)
+                else:
+                    statuses = list(status)
+                    placeholders = ",".join("?" for _ in statuses)
+                    conditions.append(f"status IN ({placeholders})")
+                    params.extend(s.value for s in statuses)
+
+            if creator_id is not None:
+                conditions.append("creator_id = ?")
+                params.append(creator_id)
+
+            if post_id is not None:
+                conditions.append("post_id = ?")
+                params.append(post_id)
+
+            if max_fail_count is not None:
+                conditions.append("fail_count < ?")
+                params.append(max_fail_count)
+
+            if min_part_number is not None:
+                conditions.append("part > ?")
+                params.append(min_part_number)
+
+            if conditions:
+                sql += " WHERE " + " AND ".join(conditions)
+
+        logging.debug(f"SQL CMD: {sql} with params: {params}")
+        cur = self.conn.cursor()
+        cur.execute(sql, params)
+        rows = cur.fetchall()
+        if Config.DEBUG:
+            logging.debug(f"DB query returned {len(rows)} result")
+
+        return [self._row_to_video(r) for r in rows]
+
+    def get_db_videos_info(self):
+        """Return number of videos per status
+        return info: dict {
+            "not_downloaded": int,
+            "failed": int,
+            etc...
+        }
+        """
+        info = {}
+        for status in VideoStatus:
+            vids = self.query_videos(status=status)
+            info[status.value] = len(vids)
+        return info
+
+    def set_status(
+        self, video: Video, status: VideoStatus, *, fail_count: int | None = None
+    ):
+        """Set video status to specified status"""
+        video.status = status
+        if fail_count is not None:
+            video.fail_count = fail_count
+        self._upsert_video(video)
+
+    def insert_videos(self, videos: list[Video]):
+        """
+        Insert a video if not already present. Else ignore.
+        Does not modify any values
+        """
+        if not videos:
+            return
+
+        rows = []
+        for video in videos:
+            rows.append(
+                (
+                    video.post_id,
+                    video.creator_id,
+                    video.service,
+                    video.domain,
+                    video.relative_path,
+                    video.url,
+                    video.part,
+                    VideoStatus.NOT_DOWNLOADED.value,
+                    0,
+                    video.published,
+                    video.title,
+                    video.substring,
+                    None,
+                    None,
+                )
+            )
+
+        self.conn.executemany(queries.INSERT_IGNORE_VIDEO_UPSERT, rows)
+        self.conn.commit()
+
+    def _upsert_video(self, video: Video):
+        """Upsert a video.
+        If video already in DB, update specifics fields:
+             status, fail_count, relative path, file_size, downloaded_at
+        """
+        if video.status is None:
+            video.status = VideoStatus.NOT_DOWNLOADED
+
+        self.conn.execute(
+            queries.INSERT_VIDEO_UPSERT,
+            (
+                video.post_id,
+                video.creator_id,
+                video.service,
+                video.domain,
+                video.relative_path,
+                video.url,
+                video.part,
+                video.status.value,
+                video.fail_count,
+                video.published,
+                video.title,
+                video.substring,
+                video.downloaded_at,
+                video.file_size,
+            ),
+        )
+        self.conn.commit()
+
+    def close(self):
+        self.conn.close()

rcdl/core/db_queries.py

@@ -0,0 +1,66 @@
+# core/db_queries.py
+
+"""
+Hold all the SQL commands strings
+"""
+
+CREATE_VIDEOS_TABLE = """
+CREATE TABLE IF NOT EXISTS videos (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    post_id TEXT,
+    creator_id TEXT,
+    service TEXT,
+    domain TEXT,
+    relative_path TEXT,
+    url TEXT,
+    part TEXT,
+    status TEXT DEFAULT 'not_downloaded',
+    fail_count INTEGER DEFAULT 0,
+    published TEXT,
+    title TEXT,
+    substring TEXT,
+    downloaded_at TEXT,
+    file_size REAL,
+    UNIQUE (service, url)
+)
+"""
+
+CREATE_SCHEMA_VERSION_TABLE = """
+CREATE TABLE IF NOT EXISTS schema_version (
+    version INTEGER NOT NULL
+)
+"""
+
+INSERT_VIDEO_UPSERT = """
+INSERT INTO videos (
+    post_id, creator_id, service, domain, relative_path, url, part,
+    status, fail_count, published, title, substring,
+    downloaded_at, file_size
+)
+VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+ON CONFLICT(service, url) DO UPDATE SET
+    status = excluded.status,
+    fail_count = excluded.fail_count,
+    relative_path = excluded.relative_path,
+    downloaded_at = excluded.downloaded_at,
+    file_size = excluded.file_size
+"""
+
+INSERT_IGNORE_VIDEO_UPSERT = """
+INSERT OR IGNORE INTO videos (
+    post_id, creator_id, service, domain, relative_path, url, part, 
+    status, fail_count, published, title, substring, 
+    downloaded_at, file_size
+)
+VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+"""
+
+CREATE_IDX_VIDEOS_STATUS = (
+    "CREATE INDEX IF NOT EXISTS idx_videos_status ON videos(status)"
+)
+CREATE_IDX_VIDEOS_CREATOR = (
+    "CREATE INDEX IF NOT EXISTS idx_videos_creator ON videos(creator_id)"
+)
+CREATE_IDX_VIDEOS_FAIL_COUNT = (
+    "CREATE INDEX IF NOT EXISTS idx_videos_fail_count ON videos(fail_count)"
+)