rcdl 2.2.2__py3-none-any.whl → 3.0.0b13__py3-none-any.whl

rcdl/core/db.py

@@ -1,235 +1,283 @@
 # core/db.py
 
-"""Handle SQL DB and DB Parsing"""
+"""
+Handle SQL Database
+"""
 
 import sqlite3
-import logging
-from typing import Iterable
 
+from rcdl.core import adapters
+from rcdl.core import db_queries as queries
+from rcdl.core.config import Config
+from rcdl.core.models import Post, Media, Status, FusedMedia, FusedStatus
+from rcdl.utils import get_date_now
 
-import rcdl.core.db_queries as queries
-from .config import Config
-from .models import Video, VideoStatus
+from rcdl.interface.ui import UI
 
 
 class DB:
+    """Handle all sqlite database command"""
+
     def __init__(self):
         self.conn = sqlite3.connect(Config.DB_PATH)
         self.conn.row_factory = sqlite3.Row
-        self.conn.execute("PRAGMA foreign_keys = ON")  # check input into db
-        self.conn.execute("PRAGMA journal_mode = WAL")  #
-        self.conn.execute("PRAGMA synchronous = NORMAL")  # faster write speed
 
     def __enter__(self):
+        """necessary to use with openDB()"""
         return self
 
     def __exit__(self, exc_type, exc_value, traceback):
+        """necessary to use with openDB()"""
         self.close()
 
-    def init_table(self):
-        # init table for videos to DL
-        self.conn.execute(queries.CREATE_VIDEOS_TABLE)
-        self.conn.execute(queries.CREATE_IDX_VIDEOS_STATUS)
-        self.conn.execute(queries.CREATE_IDX_VIDEOS_CREATOR)
-        self.conn.execute(queries.CREATE_IDX_VIDEOS_FAIL_COUNT)
+    def close(self):
+        """Properly close database"""
+        self.conn.close()
 
-        # init table for version schem, easy migration if necessary
-        self.conn.execute(queries.CREATE_SCHEMA_VERSION_TABLE)
+    def init_database(self):
+        """Create tables (posts, fuses, medias) if they dont exist"""
+        self.conn.execute(queries.CREATE_POSTS_TABLE)
+        self.conn.execute(queries.CREATE_MEDIAS_TABLE)
+        self.conn.execute(queries.CREATE_FUSE_TABLE)
 
-        # Initialize version if empty
-        cur = self.conn.cursor()
-        cur.execute("SELECT COUNT(*) as cnt FROM schema_version")
+        self.conn.commit()
+
+    def get_nb_per_status(self):
+        """Return an info dict per tables with number of entry per status
+        info['tables1']['status1'] = X
+        ...
+        """
+        info = {}
+        info["medias"] = {}
+        info["fuses"] = {}
+        info["posts"] = 0
+        for status in Status:
+            info["medias"][status] = len(self.query_media_by_status(status))
+        for status in FusedStatus:
+            info["fuses"][status] = len(self.query_fuses_by_status(status))
+
+        cur = self.conn.execute(("SELECT COUNT(*) AS count FROM posts"))
         row = cur.fetchone()
-        if row["cnt"] == 0:
-            cur.execute("INSERT INTO schema_version (version) VALUES (?)", (1,))
+        info["posts"] = row["count"] if row else 0
+        return info
 
-        self.conn.commit()
+    def query_post_by_id(self, _id: str) -> Post | None:
+        """Get a post from his post id"""
+        row = self.conn.execute(queries.QUERY_POST_ID, (_id,)).fetchone()
+
+        UI.debug(f"{queries.QUERY_POST_ID} {_id} returned {row}")
 
-    def get_schema_version(self) -> int:
-        # get current db version -> for future update to db
+        if row is None:
+            return None
+
+        return adapters.row_to_post(row)
+
+    def query_post_by_user(self, user: str) -> list[Post]:
+        """Get all posts of a user"""
         cur = self.conn.cursor()
-        cur.execute("SELECT version FROM schema_version")
-        row = cur.fetchone()
-        return row["version"] if row else 0
+        cur.execute(queries.QUERY_POST_USER, (user,))
+        rows = cur.fetchall()
 
-    def set_schema_version(self, version: int):
-        # set current db version
-        self.conn.execute("UPDATE schema_version SET version = ?", (version,))
-        self.conn.commit()
+        UI.debug(f"{queries.QUERY_POST_USER} {user} returned {len(rows)} results")
 
-    def _row_to_video(self, row: sqlite3.Row) -> Video:
-        # helper function to retunr Video model from a sql result
-        return Video(
-            post_id=row["post_id"],
-            creator_id=row["creator_id"],
-            service=row["service"],
-            domain=row["domain"],
-            relative_path=row["relative_path"],
-            url=row["url"],
-            part=row["part"],
-            status=VideoStatus(row["status"]),
-            fail_count=row["fail_count"],
-            published=row["published"],
-            title=row["title"],
-            substring=row["substring"],
-            downloaded_at=row["downloaded_at"],
-            file_size=row["file_size"],
+        return adapters.rows_to_posts(rows)
+
+    def query_media_by_status(self, status: Status) -> list[Media]:
+        """Get all medias with specified status"""
+        rows = self.conn.execute(queries.QUERY_MEDIA_STATUS, (status.value,)).fetchall()
+        UI.debug(
+            f"{queries.QUERY_MEDIA_STATUS} {status.value} returned {len(rows)} result"
         )
 
-    def query_videos(
+        return adapters.rows_to_medias(rows)
+
+    def query_medias_by_status_sorted(
         self,
-        *,
-        status: VideoStatus | Iterable[VideoStatus] | None = None,
-        creator_id: str | None = None,
-        post_id: str | None = None,
-        max_fail_count: int | None = None,
-        min_part_number: int | None = None,
-        pending: bool = False,
-    ) -> list[Video]:
-        """
-        Query the DB.
-        Parametes are 'AND' so if video status and creator_id are sepcified,
-        it will look for a match between the two
+        status: Status | list[Status],
+        sort_by: str | None = None,
+        ascending: bool = True,
+    ) -> list[Media]:
+        """Get all medias with specified status (one or multiple)
+        Return them sorted by column and asc or desc"""
+
+        # validate sort column
+        valid_columns = {
+            "id",
+            "post_id",
+            "service",
+            "url",
+            "duration",
+            "sequence",
+            "status",
+            "checksum",
+            "file_path",
+            "created_at",
+            "updated_at",
+            "file_size",
+            "fail_count",
+        }
+        order_clause = ""
+        if sort_by:
+            if sort_by not in valid_columns:
+                UI.error(f"Invalid sort column: {sort_by}")
+            order_clause = f"ORDER BY {sort_by} {'ASC' if ascending else 'DESC'}"
 
-        pending:
-        No parameters are taken into account if pending True.
-        Look for videos with: NOT DOWNLOADED status OR (FAILED & fail_count < Config.max_fail_count)
-        """
-        sql = "SELECT * FROM videos"
-        conditions = []
-        params = []
-
-        if pending:
-            sql += " WHERE status = ? OR (status = ? AND fail_count < ?)"
-            params.extend(
-                [
-                    VideoStatus.NOT_DOWNLOADED.value,
-                    VideoStatus.FAILED.value,
-                    max_fail_count or Config.MAX_FAIL_COUNT,
-                ]
+        # status filter
+        if isinstance(status, Status):
+            status = [status]
+
+        status_values = [s.value if isinstance(s, Status) else s for s in status]
+        placeholders = ", ".join("?" for _ in status_values)
+
+        sql = f"SELECT * FROM medias WHERE status IN ({placeholders}) {order_clause}"
+        rows = self.conn.execute(sql, status_values).fetchall()
+
+        UI.debug(
+            f"Queried medias with status={status_values}, sorted by {sort_by}, ascending={ascending}, {len(rows)} results"
+        )
+
+        return adapters.rows_to_medias(rows)
+
+    def query_media_by_post_id(self, _id: str) -> list[Media]:
+        """Get all medias from the same post by post id"""
+        rows = self.conn.execute(queries.QUERY_MEDIA_ID, (_id,)).fetchall()
+        UI.debug(f"{queries.QUERY_MEDIA_ID} {_id} returned {len(rows)} result")
+        return adapters.rows_to_medias(rows)
+
+    def query_fuses_by_status(self, status: FusedStatus) -> list[FusedMedia]:
+        """Get all fused_media with specified status"""
+        rows = self.conn.execute(queries.QUERY_FUSES_STATUS, (status.value,)).fetchall()
+        UI.debug(
+            f"{queries.QUERY_FUSES_STATUS} {status.value} returned {len(rows)} result"
+        )
+
+        return adapters.rows_to_fuses(rows)
+
+    def query_fuses_by_id(self, _id: str) -> FusedMedia | None:
+        """Get a fuse group by its unique post id"""
+        row = self.conn.execute(queries.QUERY_FUSES_ID, (_id,)).fetchone()
+        UI.debug(f"{queries.QUERY_FUSES_ID} {_id} returned {row} result")
+        return adapters.row_to_fused_media(row)
+
+    def insert_posts(self, posts: list[Post] | Post):
+        """Add post to DB if it does not already exist (UNIQUE post_id)"""
+        if isinstance(posts, Post):
+            posts = [posts]
+
+        values = []
+        for post in posts:
+            values.append(
+                (
+                    post.id,
+                    post.user,
+                    post.service,
+                    post.domain,
+                    post.published,
+                    post.json_hash,
+                    post.raw_json,
+                    post.fetched_at,
+                )
             )
-        else:
-            if status is not None:
-                if isinstance(status, VideoStatus):
-                    conditions.append("status = ?")
-                    params.append(status.value)
-                else:
-                    statuses = list(status)
-                    placeholders = ",".join("?" for _ in statuses)
-                    conditions.append(f"status IN ({placeholders})")
-                    params.extend(s.value for s in statuses)
-
-            if creator_id is not None:
-                conditions.append("creator_id = ?")
-                params.append(creator_id)
-
-            if post_id is not None:
-                conditions.append("post_id = ?")
-                params.append(post_id)
-
-            if max_fail_count is not None:
-                conditions.append("fail_count < ?")
-                params.append(max_fail_count)
-
-            if min_part_number is not None:
-                conditions.append("part > ?")
-                params.append(min_part_number)
-
-            if conditions:
-                sql += " WHERE " + " AND ".join(conditions)
-
-        logging.debug(f"SQL CMD: {sql} with params: {params}")
-        cur = self.conn.cursor()
-        cur.execute(sql, params)
-        rows = cur.fetchall()
-        if Config.DEBUG:
-            logging.debug(f"DB query returned {len(rows)} result")
 
-        return [self._row_to_video(r) for r in rows]
+        with self.conn:
+            self.conn.executemany(queries.INSERT_POST, values)
 
-    def get_db_videos_info(self):
-        """Return number of videos per status
-        return info: dict {
-            "not_downloaded": int,
-            "failed": int,
-            etc...
-        }
-        """
-        info = {}
-        for status in VideoStatus:
-            vids = self.query_videos(status=status)
-            info[status.value] = len(vids)
-        return info
+        inserted = self.conn.total_changes
+        UI.debug(f"Inserted {inserted} new posts out of {len(posts)} total posts")
 
-    def set_status(
-        self, video: Video, status: VideoStatus, *, fail_count: int | None = None
-    ):
-        """Set video status to specified status"""
-        video.status = status
-        if fail_count is not None:
-            video.fail_count = fail_count
-        self._upsert_video(video)
+    def insert_medias(self, medias: list[Media] | Media):
+        """Insert media into the db if it does not already exist (UNIQUE post_id, url)"""
+        if isinstance(medias, Media):
+            medias = [medias]
 
-    def insert_videos(self, videos: list[Video]):
-        """
-        Insert a video if not already present. Else ignore.
-        Does not modify any values
-        """
-        if not videos:
-            return
+        values = []
+        for media in medias:
+            values.append(
+                (
+                    media.post_id,
+                    media.service,
+                    media.url,
+                    media.duration,
+                    media.sequence,
+                    media.status.value,
+                    media.checksum,
+                    media.file_path,
+                    media.created_at,
+                    get_date_now(),
+                    media.file_size,
+                    media.fail_count,
+                )
+            )
 
-        rows = []
-        for video in videos:
-            rows.append(
+        with self.conn:
+            self.conn.executemany(queries.INSERT_MEDIA, values)
+
+        inserted = self.conn.total_changes
+        UI.debug(f"Inserted {inserted} new media out of {len(medias)} total medias")
+
+    def update_media(self, media: Media):
+        """Update media entry in the db. Found it by post_id & url, and update:
+        - duration, file_size, checksum, status, create_at, updated_at, fail_count"""
+        params = (
+            media.duration,
+            media.file_size,
+            media.checksum,
+            media.status.value,
+            media.created_at,
+            get_date_now(),
+            media.fail_count,
+            media.post_id,
+            media.url,
+        )
+        with self.conn:
+            self.conn.execute(queries.UPDATE_MEDIA, params)
+            UI.debug(f"Updated media {media.post_id} / {media.url}")
+
+    def insert_fused_media(self, fuses: list[FusedMedia] | FusedMedia):
+        """Insert fused_media into the db if it does not already exist (UNIQUE post_id)"""
+        if isinstance(fuses, FusedMedia):
+            fuses = [fuses]
+
+        values = []
+        for fuse in fuses:
+            values.append(
                 (
-                    video.post_id,
-                    video.creator_id,
-                    video.service,
-                    video.domain,
-                    video.relative_path,
-                    video.url,
-                    video.part,
-                    VideoStatus.NOT_DOWNLOADED.value,
-                    0,
-                    video.published,
-                    video.title,
-                    video.substring,
-                    None,
-                    None,
+                    fuse.id,
+                    fuse.duration,
+                    fuse.total_parts,
+                    fuse.status.value,
+                    fuse.checksum,
+                    fuse.file_path,
+                    fuse.created_at,
+                    get_date_now(),
+                    fuse.file_size,
+                    fuse.fail_count,
                 )
             )
 
-        self.conn.executemany(queries.INSERT_IGNORE_VIDEO_UPSERT, rows)
-        self.conn.commit()
+        with self.conn:
+            self.conn.executemany(queries.INSERT_FUSED_MEDIA, values)
 
-    def _upsert_video(self, video: Video):
-        """Upsert a video.
-        If video already in DB, update specifics fields:
-             status, fail_count, relative path, file_size, downloaded_at
-        """
-        if video.status is None:
-            video.status = VideoStatus.NOT_DOWNLOADED
-
-        self.conn.execute(
-            queries.INSERT_VIDEO_UPSERT,
-            (
-                video.post_id,
-                video.creator_id,
-                video.service,
-                video.domain,
-                video.relative_path,
-                video.url,
-                video.part,
-                video.status.value,
-                video.fail_count,
-                video.published,
-                video.title,
-                video.substring,
-                video.downloaded_at,
-                video.file_size,
-            ),
+        inserted = self.conn.total_changes
+        UI.debug(
+            f"Inserted {inserted} new fused_media out of {len(fuses)} total fused_media"
         )
-        self.conn.commit()
 
-    def close(self):
-        self.conn.close()
+    def update_fuse(self, fuse: FusedMedia):
+        """Update fuse group: duration, status, checksum,
+        created_at, updated_at, file_size, fail_count
+
+        """
+        params = (
+            fuse.duration,
+            fuse.status.value,
+            fuse.checksum,
+            fuse.created_at,
+            get_date_now(),
+            fuse.file_size,
+            fuse.fail_count,
+            fuse.id,
+        )
+        with self.conn:
+            self.conn.execute(queries.UPDATE_FUSE, params)
+            UI.debug(f"Updated fuse {fuse.id} / {fuse.file_path}")

rcdl/core/db_queries.py

@@ -1,66 +1,97 @@
 # core/db_queries.py
-
 """
-Hold all the SQL commands strings
+Hold SQL STRING
 """
 
-CREATE_VIDEOS_TABLE = """
-CREATE TABLE IF NOT EXISTS videos (
+CREATE_MEDIAS_TABLE = """
+CREATE TABLE IF NOT EXISTS medias (
     id INTEGER PRIMARY KEY AUTOINCREMENT,
     post_id TEXT,
-    creator_id TEXT,
     service TEXT,
-    domain TEXT,
-    relative_path TEXT,
     url TEXT,
-    part TEXT,
-    status TEXT DEFAULT 'not_downloaded',
-    fail_count INTEGER DEFAULT 0,
-    published TEXT,
-    title TEXT,
-    substring TEXT,
-    downloaded_at TEXT,
-    file_size REAL,
-    UNIQUE (service, url)
+    duration REAL,
+    sequence INTEGER,
+    status TEXT,
+    checksum TEXT,
+    file_path TEXT,
+    created_at DATETIME,
+    updated_at DATETIME,
+    file_size INTEGER,
+    fail_count INTEGER,
+    UNIQUE(post_id, url)
 )
 """
 
-CREATE_SCHEMA_VERSION_TABLE = """
-CREATE TABLE IF NOT EXISTS schema_version (
-    version INTEGER NOT NULL
+CREATE_POSTS_TABLE = """
+CREATE TABLE IF NOT EXISTS posts (
+    id TEXT PRIMARY KEY,
+    user TEXT,
+    service TEXT,
+    domain TEXT,
+    published DATETIME,
+    json_hash TEXT,
+    raw_json JSON,
+    fetched_at DATETIME
 )
 """
 
-INSERT_VIDEO_UPSERT = """
-INSERT INTO videos (
-    post_id, creator_id, service, domain, relative_path, url, part,
-    status, fail_count, published, title, substring,
-    downloaded_at, file_size
+CREATE_FUSE_TABLE = """
+CREATE TABLE IF NOT EXISTS fuses (
+    id TEXT PRIMARY KEY,
+    duration INTEGER,
+    total_parts INTEGER,
+    status TEXT,
+    checksum TEXT,
+    file_path TEXT,
+    created_at DATETIME,
+    updated_at DATETIME,
+    file_size INTEGER,
+    fail_count INTEGER
 )
-VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-ON CONFLICT(service, url) DO UPDATE SET
-    status = excluded.status,
-    fail_count = excluded.fail_count,
-    relative_path = excluded.relative_path,
-    downloaded_at = excluded.downloaded_at,
-    file_size = excluded.file_size
 """
 
-INSERT_IGNORE_VIDEO_UPSERT = """
-INSERT OR IGNORE INTO videos (
-    post_id, creator_id, service, domain, relative_path, url, part, 
-    status, fail_count, published, title, substring, 
-    downloaded_at, file_size
+INSERT_POST = """
+INSERT OR IGNORE INTO posts (
+    id, user, service, domain, published,
+    json_hash, raw_json, fetched_at
 )
-VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+VALUES (?, ?, ?, ?, ?, ?, ?, ?)
 """
 
-CREATE_IDX_VIDEOS_STATUS = (
-    "CREATE INDEX IF NOT EXISTS idx_videos_status ON videos(status)"
+INSERT_FUSED_MEDIA = """
+INSERT OR IGNORE INTO fuses (
+    id, duration, total_parts, status, checksum,
+    file_path, created_at, updated_at, file_size, fail_count
 )
-CREATE_IDX_VIDEOS_CREATOR = (
-    "CREATE INDEX IF NOT EXISTS idx_videos_creator ON videos(creator_id)"
-)
-CREATE_IDX_VIDEOS_FAIL_COUNT = (
-    "CREATE INDEX IF NOT EXISTS idx_videos_fail_count ON videos(fail_count)"
+VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+"""
+
+INSERT_MEDIA = """
+INSERT OR IGNORE INTO medias (
+    post_id, service, url, duration, sequence, status,
+    checksum, file_path, created_at, updated_at, file_size, fail_count
 )
+VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+"""
+
+UPDATE_MEDIA = """
+UPDATE medias
+SET duration = ?, file_size = ?, checksum = ?, status = ?,
+    created_at = ?, updated_at = ?, fail_count = ?
+WHERE post_id = ? AND url = ?
+"""
+
+UPDATE_FUSE = """
+UPDATE fuses
+SET duration = ?, status = ?, checksum = ?,
+    created_at = ?, updated_at = ?, file_size = ?,
+    fail_count = ?
+WHERE id = ?
+"""
+
+QUERY_POST_ID = "SELECT * FROM posts WHERE id = ?"
+QUERY_POST_USER = "SELECT * FROM posts WHERE user = ?"
+QUERY_MEDIA_STATUS = "SELECT * FROM medias WHERE status = ?"
+QUERY_MEDIA_ID = "SELECT * FROM medias WHERE post_id = ?"
+QUERY_FUSES_STATUS = "SELECT * FROM fuses WHERE status = ?"
+QUERY_FUSES_ID = "SELECT * FROM fuses WHERE id = ?"