rcdl 3.0.0b18__py3-none-any.whl

rcdl/core/opti.py

@@ -0,0 +1,90 @@
+# core/opti.py
+
+"""
+Optimize media to reduce disk storage utilisation
+"""
+
+import os
+
+from rcdl.core.config import Config
+from rcdl.core.models import Status, Media
+from rcdl.core.db import DB
+from rcdl.core.downloader_subprocess import handbrake_optimized
+from rcdl.interface.ui import UI, NestedProgress
+from rcdl.utils import get_media_metadata, get_date_now
+
+
+def update_db(media: Media, user: str, result: int):
+    """Update DB if optimisation succesfful with new file_size, etc..."""
+    if result == 0:
+        path = os.path.join(Config.creator_folder(user), media.file_path)
+        _, file_size, checksum = get_media_metadata(path)
+        media.status = Status.OPTIMIZED
+        media.checksum = checksum
+        media.created_at = get_date_now()
+        media.file_size = file_size
+
+        with DB() as db:
+            db.update_media(media)
+
+
+def optimize():
+    """Optimize all medias in DB with DOWNLOADED
+    status that are not part of a fuse group"""
+    # get all video to opti
+    with DB() as db:
+        medias = db.query_media_by_status(Status.DOWNLOADED)
+        if Config.DEBUG:
+            medias.extend(db.query_media_by_status(Status.OPTIMIZED))
+
+    # progress
+    progress = NestedProgress(UI.console)
+    progress.start(
+        total=len(medias),
+        total_label="Optimizing videos",
+        current_label="Current video",
+    )
+
+    for media in medias:
+        # check media is not in a fuse group
+        with DB() as db:
+            fuse = db.query_fuses_by_id(media.post_id)
+        if fuse is not None:
+            progress.advance_total()
+            continue
+
+        # get post info
+        with DB() as db:
+            post = db.query_post_by_id(media.post_id)
+        if post is None:
+            UI.error(f"Could not match media {media.post_id} to a post by id")
+            progress.advance_total()
+            continue
+
+        result = handbrake_optimized(media, post.user, progress)
+
+        folder_path = Config.creator_folder(post.user)
+        video_path = os.path.join(folder_path, media.file_path)
+        output_path = video_path + ".opti.mp4"
+
+        if result == 0:
+            try:
+                os.replace(output_path, video_path)
+                update_db(media, post.user, result)
+            except FileNotFoundError as e:
+                UI.error(
+                    f"FileNotFoundError: Could not replace {video_path} "
+                    f"with {output_path} due to: {e}"
+                )
+            except PermissionError as e:
+                UI.error(
+                    f"PermissionError: Could not replace {video_path} "
+                    f"with {output_path} due to: {e}"
+                )
+            except OSError as e:
+                UI.error(
+                    f"OSError: Failed to replace {video_path} with {output_path} due to: {e}"
+                )
+            finally:
+                progress.advance_total()
+    progress.close()

rcdl/core/parser.py

@@ -0,0 +1,282 @@
+# core/parser.py
+
+"""Handle function to parse post and files"""
+
+import logging
+from pathvalidate import sanitize_filename
+
+from rcdl.interface.ui import UI
+from rcdl.core.models import Media, Creator, Post, CreatorStatus
+from rcdl.core.file_io import load_json, load_txt, write_txt
+from rcdl.core.config import Config
+
+
+COOMER_PAYSITES = ["onlyfans", "fansly", "candfans"]
+KEMONO_PAYSITES = [
+    "patreon",
+    "fanbox",
+    "fantia",
+    "boosty",
+    "gumroad",
+    "subscribestar",
+    "dlsite",
+]
+
+
+def get_domain(arg: str | dict | Media) -> str:
+    """From a service get the domain (coomer or kemono)
+    Input is either: service(str), post(dict), video(models.Video)
+    """
+
+    def _service(service: str) -> str:
+        if service in COOMER_PAYSITES:
+            return "coomer"
+        if service in KEMONO_PAYSITES:
+            return "kemono"
+        logging.error("Service %s not associated to any domain", service)
+        return ""
+
+    if isinstance(arg, dict):
+        return _service(arg["service"])
+    if isinstance(arg, Media):
+        return _service(arg.service)
+
+    return _service(arg)
+
+
+def get_title(post: Post) -> str:
+    """From a Post Model return the title"""
+    title = post.title
+    if title == "":
+        title = post.substring
+    if title == "":
+        title = post.id
+    return sanitize_filename(title)
+
+
+def get_title_json(post: dict) -> str:
+    """Extract title from a post(dict)"""
+    title = post["title"]
+    if title == "":
+        title = post["substring"]
+    if title == "":
+        title = post["id"]
+    return sanitize_filename(title)
+
+
+def get_date(post: dict) -> str:
+    """Extract date from a post(dict)"""
+    if "published" in post:
+        date = post["published"][0:10]
+    elif "added" in post:
+        date = post["added"][0:10]
+    else:
+        logging.error("Could not extract date from %s", post["id"])
+        date = "NA"
+    return date
+
+
+def get_part(post: dict, url: str) -> int:
+    """
+    For posts containing multiple video url. Each url is considered a part,
+    so all videos from the same posts will simply have a different part number
+    """
+    urls = extract_video_urls(post)
+    part = 0
+    if len(urls) == 1:
+        return 0
+
+    for u in urls:
+        if u == url:
+            return part
+        part += 1
+
+    logging.error(
+        "Could not extract part number for post id %s with url %s", post["id"], url
+    )
+    return -1
+
+
+def get_filename(post: dict, url: str) -> str:
+    """Get filename from pst dict and url"""
+    title = get_title_json(post)
+    date = get_date(post)
+    part = get_part(post, url)
+    file_title = f"{date}_{title}".replace("'", " ").replace('"', "")
+    filename = f"{file_title}_p{part}.mp4"
+    return filename
+
+
+def get_filename_fuse(post: Post) -> str:
+    """Get filename for fuse output from Post Model
+    Fuse output has 'X' as part number"""
+    title = get_title(post)
+    date = post.published[0:10]
+    part = "X"
+    file_title = f"{date}_{title}".replace("'", " ").replace('"', "")
+    filename = f"{file_title}_p{part}.mp4"
+    return filename
+
+
+def extract_video_urls(post: dict) -> list:
+    """Extract all videos urls from a dict post"""
+    video_extensions = (".mp4", ".webm", ".mov", ".avi", ".mkv", ".flv", ".wmv", ".m4v")
+    urls = set()
+
+    # Check main file
+    if post["file"]:
+        if post["file"]["path"]:
+            path = post["file"]["path"]
+            if path.endswith(video_extensions):
+                urls.add(f"{path}")
+
+    if post["attachments"]:
+        attachments = post["attachments"]
+        for attachment in attachments:
+            if attachment["path"]:
+                if attachment["path"].endswith(video_extensions):
+                    urls.add(f"{attachment['path']}")
+
+    return list(urls)
+
+
+def filter_posts_with_videos_from_list(data: list[dict]) -> list[dict]:
+    """Return posts with video url from a json with a list of posts"""
+
+    posts_with_videos = []
+    for post in data:
+        if len(extract_video_urls(post)) > 0:
+            posts_with_videos.append(post)
+    return posts_with_videos
+
+
+def filter_posts_with_videos_from_json(path: str) -> list:
+    """Return posts with video url from a json with a list of posts"""
+    posts = load_json(path)
+
+    posts_with_videos = []
+    for post in posts:
+        if len(extract_video_urls(post)) > 0:
+            posts_with_videos.append(post)
+    return posts_with_videos
+
+
+def valid_service(service: str) -> bool:
+    """Check if a service is valid (within list of DOMAIN services)"""
+    if service in COOMER_PAYSITES:
+        return True
+    if service in KEMONO_PAYSITES:
+        return True
+    return False
+
+
+def _default_creator(_id: str, service: str, domain: str):
+    return Creator(
+        id=_id,
+        service=service,
+        domain=domain,
+        name="",
+        indexed="",
+        updated="",
+        favorited=1,
+        status=CreatorStatus.NA,
+        max_date="",
+        max_posts=1,
+        max_size=1,
+        min_date="",
+    )
+
+
+def get_creator_from_line(line: str) -> Creator | None:
+    """
+    Convert a line into a Creator model
+    arg: line -> 'service/creator'
+    This is the format of creators.txt
+    """
+
+    parts = line.split("/")
+    if valid_service(parts[0].strip()):
+        return _default_creator(
+            parts[1].strip(), parts[0].strip(), get_domain(parts[0].strip())
+        )
+    if valid_service(parts[1].strip()):
+        return _default_creator(
+            parts[0].strip(), parts[1].strip(), get_domain(parts[1].strip())
+        )
+
+    UI.error(
+        f"Creator file not valid: {line} can not be interpreted."
+        f" Format is: 'service/creator_id'"
+    )
+    return None
+
+
+def get_creators() -> list[Creator]:
+    """
+    Load creators.txt and return a list of models.Creator
+    """
+    lines = load_txt(Config.CREATORS_FILE)
+    creators = []
+    for line in lines:
+        creator = get_creator_from_line(line)
+        if creator is None:
+            continue
+        creators.append(creator)
+    if len(creators) < 1:
+        UI.error(f"Could not find any creators. Check {Config.CREATORS_FILE}")
+    return creators
+
+
+def get_creators_from_posts(posts: list[dict]) -> list[Creator]:
+    """Extract a list of Creators model form a list of dict posts"""
+    creators = []
+    seen = set()
+
+    for post in posts:
+        key = (post["user"], post["service"], "coomer")
+        if key in seen:
+            continue
+
+        seen.add(key)
+        creators.append(_default_creator(post["user"], post["service"], "coomer"))
+    return creators
+
+
+def parse_creator_input(value: str) -> tuple[str | None, str]:
+    """Parse user input in cli to extract creator id & service"""
+    value = value.strip()
+
+    # url
+    if "://" in value:
+        parts = value.replace("https://", "").strip().split("/")
+        logging.info(
+            "From %s extracte service %s and creator %s", value, parts[1], parts[3]
+        )
+        return parts[1], parts[3]  # service, creator_id
+
+    # creators.txt format
+    if "/" in value:
+        c = get_creator_from_line(value)
+        if c is not None:
+            logging.info(
+                "From %s extracte service %s and creator %s",
+                value,
+                c.service,
+                c.id,
+            )
+            return c.service, c.id
+
+    logging.info("From %s extracted service None and creator %s", value, value)
+    return None, value
+
+
+def append_creator(creator: Creator):
+    """Append a creator to the creators.txt file
+    Creators.txt hold all creators used in refresh command"""
+    line = f"{creator.service}/{creator.id}"
+    lines = load_txt(Config.CREATORS_FILE)
+
+    if line in lines:
+        return
+    lines.append(line)
+    write_txt(Config.CREATORS_FILE, line, mode="a")

rcdl/gui/__main__.py

@@ -0,0 +1,5 @@
+# rcdl/gui/__main__.py
+
+from rcdl.gui.gui import run_gui
+
+run_gui()

rcdl/gui/db_viewer.py

@@ -0,0 +1,41 @@
+# gui/db_viewer.py
+
+import streamlit as st
+import sqlite3
+
+import pandas as pd
+
+from rcdl.core.config import Config
+
+TABLES = ["medias", "posts", "fuses"]
+
+
+def get_table_columns(table_name):
+    conn = sqlite3.connect(Config.DB_PATH)
+    cur = conn.cursor()
+    cur.execute(f"PRAGMA table_info({table_name})")
+    columns = [info[1] for info in cur.fetchall()]
+    conn.close()
+    return columns
+
+
+def get_table_data(table_name, sort_by=None, ascending=True):
+    conn = sqlite3.connect(Config.DB_PATH)
+    df = pd.read_sql_query(f"SELECT * FROM {table_name}", conn)
+    conn.close()
+    if sort_by and sort_by in df.columns:
+        df = df.sort_values(by=sort_by, ascending=ascending)
+    return df
+
+
+def run_db_viewer():
+    st.set_page_config(page_title="DB Viewer", layout="wide")
+    st.title("Database Viewer")
+
+    table_name = st.selectbox("Select Table", TABLES)
+
+    # Load data
+    df = get_table_data(table_name, sort_by=None, ascending=True)
+
+    st.write(f"Showing `{table_name}` table ({len(df)} rows)")
+    st.dataframe(df, width="stretch")

rcdl/gui/gui.py

@@ -0,0 +1,54 @@
+# gui/gui.py
+
+import streamlit as st
+
+from rcdl.gui.db_viewer import run_db_viewer
+from rcdl.gui.video_manager import video_manager
+
+st.markdown(
+    """
+    <style>
+    /* Remove top padding */
+    .block-container {
+        padding-top: 1rem !important;
+    }
+
+    /* Optional: remove Streamlit header */
+    header[data-testid="stHeader"] {
+        display: none;
+    }
+
+    /* Optional: remove footer */
+    footer {
+        display: none;
+    }
+    </style>
+    """,
+    unsafe_allow_html=True,
+)
+
+
+def run_gui():
+    """
+    Launches the Streamlit GUI.
+    This function can be called from a CLI command.
+    """
+    # Streamlit code
+    st.set_page_config(page_title="RCDL", layout="wide")
+
+    # Sidebar navigation
+    page = st.sidebar.radio("Go to", ["Home", "Manage Videos", "View DB"])
+
+    if page == "Home":
+        st.header("Home Page")
+        st.write("Develloped by - ritonun -")
+
+    elif page == "Manage Videos":
+        video_manager()
+
+    elif page == "View DB":
+        run_db_viewer()
+
+
+if __name__ == "__main__":
+    run_gui()

rcdl/gui/video_manager.py

@@ -0,0 +1,170 @@
+# gui/video_manager.py
+
+import os
+
+import streamlit as st
+
+from rcdl.core.config import Config
+from rcdl.core.models import Status, Media
+from rcdl.core.db import DB
+from rcdl.utils import format_seconds
+
+
+previous_statuses = {}
+
+
+def set_status(media: Media, status: Status):
+    key = media.post_id + media.url
+    previous_statuses[key] = media.status
+    media.status = status
+    with DB() as db:
+        db.update_media(media)
+    print(f"Set {media.post_id} to {status.value}")
+
+    for m in st.session_state.medias:
+        if m.post_id == media.post_id and m.url == media.url:
+            m.status = status
+            break
+
+
+def video_manager():
+    st.title("Video Manager")
+
+    # Filter & Sorting UI
+    with st.expander("Filters & Sorting", expanded=True):
+        col1, col2, col3 = st.columns(3)
+        with col1:
+            sort_by = st.selectbox(
+                "Sort By",
+                options=["file_size", "service", "duration", "file_path"],
+                index=0,
+            )
+        with col2:
+            ascending = st.radio(
+                "Order",
+                options=[True, False],
+                format_func=lambda x: "Ascending" if x else "Descending",
+                horizontal=True,
+            )
+        with col3:
+            creator_filter = st.text_input(
+                "Creator ID(user)", placeholder="Leave empty for all"
+            )
+        status_filter = st.multiselect(
+            "Status",
+            options=list(Status),
+            default=[Status.DOWNLOADED, Status.OPTIMIZED],
+        )
+
+        reload = st.button("Apply")
+
+    # load db
+    if reload or "medias" not in st.session_state:
+        with DB() as db:
+            medias = db.query_medias_by_status_sorted(
+                status_filter,
+                sort_by=sort_by,
+                ascending=ascending,
+            )
+
+            # check if in a fuse group
+
+            # creator filter
+            if creator_filter:
+                filtered = []
+                for m in medias:
+                    post = db.query_post_by_id(m.post_id)
+                    if post and post.user == creator_filter:
+                        filtered.append(m)
+                    # check i na fuse group
+                    fm = db.query_fuses_by_id(m.post_id)
+                    if fm is None:
+                        filtered.append(m)
+                medias = filtered
+
+            st.session_state.medias = medias
+            st.session_state.media_index = 0
+
+    medias = st.session_state.medias
+    if not medias:
+        st.info("No media found")
+        return
+
+    # session state
+    if "media_index" not in st.session_state:
+        st.session_state.media_index = 0
+
+    idx = st.session_state.media_index
+    media = medias[idx]
+
+    # media info
+    st.subheader(f"Media {idx + 1} / {len(medias)}")
+
+    with DB() as db:
+        post = db.query_post_by_id(media.post_id)
+        if post is None:
+            st.info("No matching post found")
+            return
+
+    col_video, col_info = st.columns([1, 2])
+    with col_info:
+        col1, col2 = st.columns(2)
+        with col1:
+            st.write("**Post ID:**", media.post_id)
+            st.write("**Service:**", media.service)
+            st.write("**User:**", post.user)
+            st.write("**Duration:**", format_seconds(media.duration))
+            st.write("**Sequence:**", media.sequence)
+            st.write("**Size:**", round(media.file_size / (1024 * 1024), 1), "MB")
+            st.write("**Status:**", media.status)
+            key = media.post_id + media.url
+            if key in previous_statuses:
+                st.write("**PREV STATUS:**", previous_statuses[key])
+            st.write("**Path:**", media.file_path)
+            st.write("**Created at**:", media.created_at[0:16])
+
+        with col2:
+            # controls
+            c1, c2, c3 = st.columns([1, 1, 2])
+            with c1:
+                if st.button("⏮ Prev", disabled=idx == 0):
+                    st.session_state.media_index -= 1
+                    st.rerun()
+                if st.button("⏭ Next", disabled=idx >= len(medias) - 1):
+                    st.session_state.media_index += 1
+                    st.rerun()
+            with c2:
+                if st.button("Remove"):
+                    set_status(media, Status.TO_BE_DELETED)
+                    st.rerun()
+                if st.button("Revert Status"):
+                    key = media.post_id + media.url
+                    if key in previous_statuses:
+                        set_status(media, previous_statuses[key])
+                    else:
+                        print("Not in previous status")
+                    st.rerun()
+            with c3:
+                chosen_status = st.selectbox(
+                    "Set Status",
+                    options=list(Status),
+                    index=list(Status).index(media.status)
+                    if media.status in list(Status)
+                    else 0,
+                )
+                if st.button("Apply Status"):
+                    set_status(media, chosen_status)
+                    st.rerun()
+
+    # video player
+    full_path = os.path.join(Config.creator_folder(post.user), media.file_path)
+    if os.path.exists(full_path):
+        with col_video:
+            with st.container():
+                if media.file_size > 199 * 1024 * 1024:  # 199MB
+                    with open(full_path, "rb") as f:
+                        st.video(f.read(), autoplay=True, loop=True)
+                else:
+                    st.video(full_path, autoplay=True, loop=True)
+    else:
+        st.error(f"Video file {full_path} not found on disk")