rcdl 2.2.2__py3-none-any.whl → 3.0.0b13__py3-none-any.whl

rcdl/core/parser.py

@@ -1,12 +1,14 @@
 # core/parser.py
 
+"""Handle function to parse post and files"""
+
 import logging
 from pathvalidate import sanitize_filename
 
-from .models import Video, VideoStatus, Creator
-from .file_io import load_json, load_txt, write_txt
-from .config import Config
 from rcdl.interface.ui import UI
+from rcdl.core.models import Media, Creator, Post, CreatorStatus
+from rcdl.core.file_io import load_json, load_txt, write_txt
+from rcdl.core.config import Config
 
 
 COOMER_PAYSITES = ["onlyfans", "fansly", "candfans"]
@@ -21,7 +23,7 @@ KEMONO_PAYSITES = [
 ]
 
 
-def get_domain(arg: str | dict | Video) -> str:
+def get_domain(arg: str | dict | Media) -> str:
     """From a service get the domain (coomer or kemono)
     Input is either: service(str), post(dict), video(models.Video)
     """
@@ -31,25 +33,34 @@ def get_domain(arg: str | dict | Video) -> str:
             return "coomer"
         if service in KEMONO_PAYSITES:
             return "kemono"
-        logging.error(f"Service {service} not associated to any domain")
+        logging.error("Service %s not associated to any domain", service)
         return ""
 
     if isinstance(arg, dict):
         return _service(arg["service"])
-    elif isinstance(arg, Video):
+    if isinstance(arg, Media):
         return _service(arg.service)
 
     return _service(arg)
 
 
-def get_title(post: dict) -> str:
+def get_title(post: Post) -> str:
+    """From a Post Model return the title"""
+    title = post.title
+    if title == "":
+        title = post.substring
+    if title == "":
+        title = post.id
+    return sanitize_filename(title)
+
+
+def get_title_json(post: dict) -> str:
     """Extract title from a post(dict)"""
     title = post["title"]
     if title == "":
-        if "content" in post:
-            title = post["content"]
-        elif "substring" in post:
-            title = post["substring"]
+        title = post["substring"]
+    if title == "":
+        title = post["id"]
     return sanitize_filename(title)
 
 
@@ -60,7 +71,7 @@ def get_date(post: dict) -> str:
     elif "added" in post:
         date = post["added"][0:10]
     else:
-        logging.error(f"Could not extract date from {post['id']}")
+        logging.error("Could not extract date from %s", post["id"])
         date = "NA"
     return date
 
@@ -81,13 +92,14 @@ def get_part(post: dict, url: str) -> int:
         part += 1
 
     logging.error(
-        f"Could not extract part number for post id {post['id']} with url {url}"
+        "Could not extract part number for post id %s with url %s", post["id"], url
     )
     return -1
 
 
 def get_filename(post: dict, url: str) -> str:
-    title = get_title(post)
+    """Get filename from pst dict and url"""
+    title = get_title_json(post)
     date = get_date(post)
     part = get_part(post, url)
     file_title = f"{date}_{title}".replace("'", " ").replace('"', "")
@@ -95,45 +107,19 @@ def get_filename(post: dict, url: str) -> str:
     return filename
 
 
-def convert_post_to_video(post: dict, url: str, discover=False) -> Video:
-    part = get_part(post, url)
+def get_filename_fuse(post: Post) -> str:
+    """Get filename for fuse output from Post Model
+    Fuse output has 'X' as part number"""
     title = get_title(post)
-    date = get_date(post)
-    filename = get_filename(post, url)
-
-    if discover:
-        filename = f"{post['user']}_{post['id']}.mp4"
-
-    return Video(
-        post_id=post["id"],
-        creator_id=post["user"],
-        service=post["service"],
-        relative_path=filename,
-        url=url,
-        domain=get_domain(post),
-        part=part,
-        published=date,
-        title=title,
-        status=VideoStatus.NOT_DOWNLOADED,
-        fail_count=0,
-    )
-
-
-def convert_posts_to_videos(posts: list[dict], discover: bool = False) -> list[Video]:
-    videos = []
-    for post in posts:
-        urls = extract_video_urls(post)
-        if not discover:
-            for url in urls:
-                videos.append(convert_post_to_video(post, url))
-        else:
-            if len(urls) == 0:
-                continue
-            videos.append(convert_post_to_video(post, urls[0], discover=discover))
-    return videos
+    date = post.published[0:10]
+    part = "X"
+    file_title = f"{date}_{title}".replace("'", " ").replace('"', "")
+    filename = f"{file_title}_p{part}.mp4"
+    return filename
 
 
 def extract_video_urls(post: dict) -> list:
+    """Extract all videos urls from a dict post"""
     video_extensions = (".mp4", ".webm", ".mov", ".avi", ".mkv", ".flv", ".wmv", ".m4v")
     urls = set()
 
@@ -176,6 +162,7 @@ def filter_posts_with_videos_from_json(path: str) -> list:
 
 
 def valid_service(service: str) -> bool:
+    """Check if a service is valid (within list of DOMAIN services)"""
     if service in COOMER_PAYSITES:
         return True
     if service in KEMONO_PAYSITES:
@@ -183,31 +170,44 @@ def valid_service(service: str) -> bool:
     return False
 
 
+def _default_creator(_id: str, service: str, domain: str):
+    return Creator(
+        id=_id,
+        service=service,
+        domain=domain,
+        name="",
+        indexed="",
+        updated="",
+        favorited=1,
+        status=CreatorStatus.NA,
+        max_date="",
+        max_posts=1,
+        max_size=1,
+        min_date="",
+    )
+
+
 def get_creator_from_line(line: str) -> Creator | None:
     """
     Convert a line into a Creator model
     arg: line -> 'service/creator'
     This is the format of creators.txt
     """
+
     parts = line.split("/")
     if valid_service(parts[0].strip()):
-        return Creator(
-            creator_id=parts[1].strip(),
-            service=parts[0].strip(),
-            domain=get_domain(parts[0].strip()),
-            status=None,
-        )
-    elif valid_service(parts[1].strip()):
-        return Creator(
-            creator_id=parts[0].strip(),
-            service=parts[1].strip(),
-            domain=get_domain(parts[1].strip()),
-            status=None,
+        return _default_creator(
+            parts[1].strip(), parts[0].strip(), get_domain(parts[0].strip())
         )
-    else:
-        UI.error(
-            f"Creator file not valid: {line} can not be interpreted. Format is: 'service/creator_id'"
+    if valid_service(parts[1].strip()):
+        return _default_creator(
+            parts[0].strip(), parts[1].strip(), get_domain(parts[1].strip())
         )
+
+    UI.error(
+        f"Creator file not valid: {line} can not be interpreted."
+        f" Format is: 'service/creator_id'"
+    )
     return None
 
 
@@ -228,7 +228,8 @@ def get_creators() -> list[Creator]:
 
 
 def get_creators_from_posts(posts: list[dict]) -> list[Creator]:
-    creators = list()
+    """Extract a list of Creators model form a list of dict posts"""
+    creators = []
     seen = set()
 
     for post in posts:
@@ -237,24 +238,20 @@ def get_creators_from_posts(posts: list[dict]) -> list[Creator]:
             continue
 
         seen.add(key)
-        creators.append(
-            Creator(
-                creator_id=post["user"],
-                service=post["service"],
-                domain="coomer",
-                status="to_be_treated",
-            )
-        )
+        creators.append(_default_creator(post["user"], post["service"], "coomer"))
     return creators
 
 
 def parse_creator_input(value: str) -> tuple[str | None, str]:
+    """Parse user input in cli to extract creator id & service"""
     value = value.strip()
 
     # url
     if "://" in value:
         parts = value.replace("https://", "").strip().split("/")
-        logging.info(f"From {value} extracte service {parts[1]} and creator {parts[3]}")
+        logging.info(
+            "From %s extracte service %s and creator %s", value, parts[1], parts[3]
+        )
         return parts[1], parts[3]  # service, creator_id
 
     # creators.txt format
@@ -262,16 +259,21 @@ def parse_creator_input(value: str) -> tuple[str | None, str]:
         c = get_creator_from_line(value)
         if c is not None:
             logging.info(
-                f"From {value} extracte service {c.service} and creator {c.creator_id}"
+                "From %s extracte service %s and creator %s",
+                value,
+                c.service,
+                c.id,
             )
-            return c.service, c.creator_id
+            return c.service, c.id
 
-    logging.info(f"From {value} extracte service None and creator {value}")
+    logging.info("From %s extracted service None and creator %s", value, value)
     return None, value
 
 
 def append_creator(creator: Creator):
-    line = f"{creator.service}/{creator.creator_id}"
+    """Append a creator to the creators.txt file
+    Creators.txt hold all creators used in refresh command"""
+    line = f"{creator.service}/{creator.id}"
     lines = load_txt(Config.CREATORS_FILE)
 
     if line in lines:

rcdl/gui/__main__.py

@@ -0,0 +1,5 @@
+# rcdl/gui/__main__.py
+
+from rcdl.gui.gui import run_gui
+
+run_gui()

rcdl/gui/db_viewer.py

@@ -0,0 +1,41 @@
+# gui/db_viewer.py
+
+import streamlit as st
+import sqlite3
+
+import pandas as pd
+
+from rcdl.core.config import Config
+
+TABLES = ["medias", "posts", "fuses"]
+
+
+def get_table_columns(table_name):
+    conn = sqlite3.connect(Config.DB_PATH)
+    cur = conn.cursor()
+    cur.execute(f"PRAGMA table_info({table_name})")
+    columns = [info[1] for info in cur.fetchall()]
+    conn.close()
+    return columns
+
+
+def get_table_data(table_name, sort_by=None, ascending=True):
+    conn = sqlite3.connect(Config.DB_PATH)
+    df = pd.read_sql_query(f"SELECT * FROM {table_name}", conn)
+    conn.close()
+    if sort_by and sort_by in df.columns:
+        df = df.sort_values(by=sort_by, ascending=ascending)
+    return df
+
+
+def run_db_viewer():
+    st.set_page_config(page_title="DB Viewer", layout="wide")
+    st.title("Database Viewer")
+
+    table_name = st.selectbox("Select Table", TABLES)
+
+    # Load data
+    df = get_table_data(table_name, sort_by=None, ascending=True)
+
+    st.write(f"Showing `{table_name}` table ({len(df)} rows)")
+    st.dataframe(df, width="stretch")

rcdl/gui/gui.py

@@ -0,0 +1,54 @@
+# gui/gui.py
+
+import streamlit as st
+
+from rcdl.gui.db_viewer import run_db_viewer
+from rcdl.gui.video_manager import video_manager
+
+st.markdown(
+    """
+    <style>
+    /* Remove top padding */
+    .block-container {
+        padding-top: 1rem !important;
+    }
+
+    /* Optional: remove Streamlit header */
+    header[data-testid="stHeader"] {
+        display: none;
+    }
+
+    /* Optional: remove footer */
+    footer {
+        display: none;
+    }
+    </style>
+    """,
+    unsafe_allow_html=True,
+)
+
+
+def run_gui():
+    """
+    Launches the Streamlit GUI.
+    This function can be called from a CLI command.
+    """
+    # Streamlit code
+    st.set_page_config(page_title="RCDL", layout="wide")
+
+    # Sidebar navigation
+    page = st.sidebar.radio("Go to", ["Home", "Manage Videos", "View DB"])
+
+    if page == "Home":
+        st.header("Home Page")
+        st.write("Develloped by - ritonun -")
+
+    elif page == "Manage Videos":
+        video_manager()
+
+    elif page == "View DB":
+        run_db_viewer()
+
+
+if __name__ == "__main__":
+    run_gui()

rcdl/gui/video_manager.py

@@ -0,0 +1,170 @@
+# gui/video_manager.py
+
+import os
+
+import streamlit as st
+
+from rcdl.core.config import Config
+from rcdl.core.models import Status, Media
+from rcdl.core.db import DB
+from rcdl.utils import format_seconds
+
+
+previous_statuses = {}
+
+
+def set_status(media: Media, status: Status):
+    key = media.post_id + media.url
+    previous_statuses[key] = media.status
+    media.status = status
+    with DB() as db:
+        db.update_media(media)
+    print(f"Set {media.post_id} to {status.value}")
+
+    for m in st.session_state.medias:
+        if m.post_id == media.post_id and m.url == media.url:
+            m.status = status
+            break
+
+
+def video_manager():
+    st.title("Video Manager")
+
+    # Filter & Sorting UI
+    with st.expander("Filters & Sorting", expanded=True):
+        col1, col2, col3 = st.columns(3)
+        with col1:
+            sort_by = st.selectbox(
+                "Sort By",
+                options=["file_size", "service", "duration", "file_path"],
+                index=0,
+            )
+        with col2:
+            ascending = st.radio(
+                "Order",
+                options=[True, False],
+                format_func=lambda x: "Ascending" if x else "Descending",
+                horizontal=True,
+            )
+        with col3:
+            creator_filter = st.text_input(
+                "Creator ID(user)", placeholder="Leave empty for all"
+            )
+        status_filter = st.multiselect(
+            "Status",
+            options=list(Status),
+            default=[Status.DOWNLOADED, Status.OPTIMIZED],
+        )
+
+        reload = st.button("Apply")
+
+    # load db
+    if reload or "medias" not in st.session_state:
+        with DB() as db:
+            medias = db.query_medias_by_status_sorted(
+                status_filter,
+                sort_by=sort_by,
+                ascending=ascending,
+            )
+
+            # check if in a fuse group
+
+            # creator filter
+            if creator_filter:
+                filtered = []
+                for m in medias:
+                    post = db.query_post_by_id(m.post_id)
+                    if post and post.user == creator_filter:
+                        filtered.append(m)
+                    # check i na fuse group
+                    fm = db.query_fuses_by_id(m.post_id)
+                    if fm is None:
+                        filtered.append(m)
+                medias = filtered
+
+            st.session_state.medias = medias
+            st.session_state.media_index = 0
+
+    medias = st.session_state.medias
+    if not medias:
+        st.info("No media found")
+        return
+
+    # session state
+    if "media_index" not in st.session_state:
+        st.session_state.media_index = 0
+
+    idx = st.session_state.media_index
+    media = medias[idx]
+
+    # media info
+    st.subheader(f"Media {idx + 1} / {len(medias)}")
+
+    with DB() as db:
+        post = db.query_post_by_id(media.post_id)
+        if post is None:
+            st.info("No matching post found")
+            return
+
+    col_video, col_info = st.columns([1, 2])
+    with col_info:
+        col1, col2 = st.columns(2)
+        with col1:
+            st.write("**Post ID:**", media.post_id)
+            st.write("**Service:**", media.service)
+            st.write("**User:**", post.user)
+            st.write("**Duration:**", format_seconds(media.duration))
+            st.write("**Sequence:**", media.sequence)
+            st.write("**Size:**", round(media.file_size / (1024 * 1024), 1), "MB")
+            st.write("**Status:**", media.status)
+            key = media.post_id + media.url
+            if key in previous_statuses:
+                st.write("**PREV STATUS:**", previous_statuses[key])
+            st.write("**Path:**", media.file_path)
+            st.write("**Created at**:", media.created_at[0:16])
+
+        with col2:
+            # controls
+            c1, c2, c3 = st.columns([1, 1, 2])
+            with c1:
+                if st.button("⏮ Prev", disabled=idx == 0):
+                    st.session_state.media_index -= 1
+                    st.rerun()
+                if st.button("⏭ Next", disabled=idx >= len(medias) - 1):
+                    st.session_state.media_index += 1
+                    st.rerun()
+            with c2:
+                if st.button("Remove"):
+                    set_status(media, Status.TO_BE_DELETED)
+                    st.rerun()
+                if st.button("Revert Status"):
+                    key = media.post_id + media.url
+                    if key in previous_statuses:
+                        set_status(media, previous_statuses[key])
+                    else:
+                        print("Not in previous status")
+                    st.rerun()
+            with c3:
+                chosen_status = st.selectbox(
+                    "Set Status",
+                    options=list(Status),
+                    index=list(Status).index(media.status)
+                    if media.status in list(Status)
+                    else 0,
+                )
+                if st.button("Apply Status"):
+                    set_status(media, chosen_status)
+                    st.rerun()
+
+    # video player
+    full_path = os.path.join(Config.creator_folder(post.user), media.file_path)
+    if os.path.exists(full_path):
+        with col_video:
+            with st.container():
+                if media.file_size > 199 * 1024 * 1024:  # 199MB
+                    with open(full_path, "rb") as f:
+                        st.video(f.read(), autoplay=True, loop=True)
+                else:
+                    st.video(full_path, autoplay=True, loop=True)
+    else:
+        st.error(f"Video file {full_path} not found on disk")