rcdl 2.2.2__py3-none-any.whl → 3.0.0b23__py3-none-any.whl

rcdl/core/fuse.py

@@ -1,118 +1,127 @@
 # core/fuse.py
 
+"""Handle merging videos from a same post"""
+
 import os
+import subprocess
 
-from rcdl.interface.ui import UI
-from rcdl.core.db import DB
-from rcdl.core.models import VideoStatus
 from rcdl.core.config import Config
-from rcdl.core.downloader_subprocess import ffmpeg_concat
-
+from rcdl.core.db import DB
+from rcdl.core.models import FusedStatus, Status, FusedMedia, Media, Post
+from rcdl.interface.ui import UI, NestedProgress
+import rcdl.core.downloader_subprocess as dls
+from rcdl.utils import get_media_metadata, get_date_now
+
+
+def update_db(fuse: FusedMedia, medias: list[Media], user: str, result):
+    """Update DB depending on subprocess result (SUCESS/FAILURE)"""
+    if result == 0:
+        path = os.path.join(Config.creator_folder(user), fuse.file_path)
+        duration, file_size, checksum = get_media_metadata(path)
+        fuse.duration = duration
+        fuse.status = FusedStatus.FUSED
+        fuse.checksum = checksum
+        fuse.created_at = get_date_now()
+        fuse.file_size = file_size
+        for media in medias:
+            media.status = Status.FUSED
+    else:
+        fuse.fail_count += 1
+    with DB() as db:
+        db.update_fuse(fuse)
+        for media in medias:
+            db.update_media(media)
+
+
+def get_medias_and_post(
+    post_id: str, total_parts: int
+) -> tuple[None, None] | tuple[list[Media], Post]:
+    """Get medias and post related to a fuse group.
+    Return a list[Media] and a Post
+    Handle Errors, return None, None"""
+    # get associated post
+    with DB() as db:
+        post = db.query_post_by_id(post_id)
+    if post is None:
+        UI.error(f"Could not match fuses post id {post_id} to a post in post tables")
+        return None, None
 
-def fuse_videos():
-    """Fuse videos"""
+    # get all videos of a post
+    with DB() as db:
+        medias = db.query_media_by_post_id(post_id)
 
-    allowed_status = [
-        VideoStatus.DOWNLOADED,
-        VideoStatus.CONCAT_WIP,
-        VideoStatus.CONCAT_FAILED,
-    ]
+    # check number of media in db match total part expected in fused media
+    if len(medias) != total_parts:
+        UI.error(f"Found {len(medias)} videos part. Expected {total_parts}")
+        return None, None
 
+    # check all video are downloaded
+    allowed_status = [Status.DOWNLOADED, Status.OPTIMIZED]
     if Config.DEBUG:
-        allowed_status.append(VideoStatus.DOWNLOADED)
-
-    # load db videos
+        allowed_status.append(Status.FUSED)
+    ok = True
+    for media in medias:
+        if media.status not in allowed_status:
+            ok = False
+            break
+    if not ok:
+        return None, None
+
+    # sort medias list
+    sorted_medias = sorted(medias, key=lambda m: m.sequence)
+    return sorted_medias, post
+
+
+def fuse_medias():
+    """Fuse all media part of a fuse group with status PENDING in DB fuses"""
+    # get all fused media
     with DB() as db:
-        videos = db.query_videos(status=allowed_status, min_part_number=1)
-
-    # get unique posts id
-    posts_ids = set()
-    for video in videos:
-        posts_ids.add(video.post_id)
-
-    with UI.progress_total_concat() as progress:
-        task = progress.add_task("Total concat", total=len(posts_ids))
-
-        for post_id in posts_ids:
-            UI.info(f"Looking at post_id: {post_id}")
-
-            # get all videos with same post_id
-            with DB() as db:
-                videos = db.query_videos(post_id=post_id)
-            if not videos:
-                UI.error("Query SQL Failed.")
-                progress.update(task, advance=1)
-                continue
-
-            # check each videos of the same post is fully downloaded
-            ok = True
-            for video in videos:
-                if video.status not in allowed_status:
-                    ok = False
-                    break
-            if not ok:
-                progress.update(task, advance=1)
-                continue
-
-            # sort by part number
-            videos.sort(key=lambda v: int(v.part))
-
-            ok = True
-            for video in videos:
-                # make sure video exist
-                path = os.path.join(
-                    Config.creator_folder(video.creator_id), video.relative_path
-                )
-                if not os.path.exists(path):
-                    UI.error(f"Video @ {path} does not exists")
-                    ok = False
-
-                # if status is concat WIP, these should not be possible
-                if video.status == VideoStatus.CONCAT_WIP:
-                    UI.warning(
-                        f"Video '{video.relative_path}' has status 'CONCAT_WIP'. This is a bug and should not be possible."
-                    )
-
-            # update videos status in db to CONCAT_WIP
-            # in case of problems in the scripts, we will know
-            with DB() as db:
-                for video in videos:
-                    db.set_status(video, VideoStatus.CONCAT_WIP)
-
-            result = 1
-            try:
-                result = ffmpeg_concat(videos)
-            except Exception as e:
-                UI.error(f"Failed concat due to: {e}")
-
-            # concat failed
-            if not result == 0:
-                with DB() as db:
-                    for video in videos:
-                        db.set_status(video, VideoStatus.CONCAT_FAILED)
-                continue
-
-            # concat succeeded
-            with DB() as db:
-                for video in videos:
-                    db.set_status(video, VideoStatus.CONCAT_DONE)
-
-            # update progress bar
-            progress.update(
-                task,
-                advance=1,
-                description=f"Concated videos for post id {post_id}",
+        fuses = db.query_fuses_by_status(FusedStatus.PENDING)
+    if Config.DEBUG:
+        with DB() as db:
+            ok_fuses = db.query_fuses_by_status(FusedStatus.FUSED)
+        fuses.extend(ok_fuses)
+
+    progress = NestedProgress(UI.console)
+    progress.start(
+        total=len(fuses), total_label="Fusing videos", current_label="Current fuse"
+    )
+
+    for fm in fuses:
+        medias, post = get_medias_and_post(fm.id, fm.total_parts)
+        if medias is None or post is None:
+            progress.advance_total()
+            continue
+
+        # concat medias
+        result = 1
+        try:
+            result = dls.ffmpeg_concat(medias, post, progress)
+        except (OSError, subprocess.SubprocessError, ValueError) as e:
+            UI.error(f"Failed to concat video (id:{post.id}) due to: {e}")
+
+        # update db
+        update_db(fm, medias, post.user, result)
+
+        progress.advance_total()
+
+        # remove part file
+        for media in medias:
+            media_full_path = os.path.join(
+                Config.creator_folder(post.user), media.file_path
             )
+            try:
+                if Config.DEBUG:
+                    UI.info(f"Skipped '{media_full_path}' removal")
+                    continue
+                os.remove(media_full_path)
+                UI.info(f"Removed file '{media_full_path}'")
+            except (FileNotFoundError, PermissionError) as e:
+                UI.error(
+                    f"FileNotFound/PermissionError: Failed to "
+                    f"remove media '{media_full_path}' due to: {e}"
+                )
+            except OSError as e:
+                UI.error(f"Failed to remove media '{media_full_path}' due to: {e}")
 
-            # remove part video if concat OK
-            with DB() as db:
-                for video in videos:
-                    path = os.path.join(
-                        Config.creator_folder(video.creator_id), video.relative_path
-                    )
-                    try:
-                        os.remove(path)
-                        UI.info(f"Removed {path}")
-                        db.set_status(video, VideoStatus.REMOVED)
-                    except Exception as e:
-                        UI.error(f"Failed to remove {path} due to error: {e}")
+    progress.close()

rcdl/core/models.py

@@ -1,56 +1,105 @@
 # core/models.py
 
+"""Hold all dataclass models and enums"""
+
 from dataclasses import dataclass
-from typing import Optional
 from enum import Enum
 
 
-class VideoStatus(Enum):
-    NOT_DOWNLOADED = "not_downloaded"
-    DOWNLOADED = "downloaded"
-    FAILED = "failed"
-    SKIPPED = "skipped"
-    IGNORED = "ignored"
-    REMOVED = "removed"
-    CONCAT_WIP = "concat_wip"  # concat in progress
-    CONCAT_DONE = "concat_done"
-    CONCAT_FAILED = "concat_failed"
+class Status(Enum):
+    """Status for media"""
+
+    PENDING = "pending"  # to be downloaded
+    DOWNLOADED = "downloaded"  # video has been downloaded
+    FUSED = "fused"  # video has been fused, and impliitly removed
+    TO_BE_DELETED = "to_be_delete"  # video has been marked for delete
+    DELETED = "deleted"  # video has been deleted
+    OPTIMIZED = "optimized"  # video has been optimized (reduce file size)
+
 
+class FusedStatus(Enum):
+    """Status for fused group"""
 
-class DiscoverStatus(Enum):
-    TO_BE_TREATED = "to_be_treated"
-    DOWNLOADED = "downloaded"
-    BLACKLISTED = "blacklisted"
-    WHITELSITED = "whitelisted"
-    DOWNLOAD_MORE = "download_more"
+    PENDING = "pending"
+    FUSED = "fused"
+
+
+class CreatorStatus(Enum):
+    FAVORITED = "FAVORITED"
+    NA = "NA"
 
 
 @dataclass
-class Creator:
-    creator_id: str
+class Post:
+    """Post model that shadow post dict response of request
+    Partially used in posts db (check db_queries.py)
+    """
+
+    id: str
+    user: str
     service: str
     domain: str
-    status: Optional[str]
+    title: str
+    substring: str
+    published: str
+    file: dict
+    attachments: list
+    json_hash: str
+    raw_json: str
+    fetched_at: str
 
 
 @dataclass
-class Video:
-    # important fields
+class Media:
+    """Media model: use in medias DB"""
+
     post_id: str
-    creator_id: str
     service: str
-    domain: str
-    relative_path: str
     url: str
-    part: int = 0
+    duration: float
+    sequence: int
+    status: Status
+    checksum: str
+    file_path: str
+    created_at: str
+    updated_at: str
+    file_size: int
+    fail_count: int = 0
 
-    # metadata
-    published: Optional[str] = None
-    title: Optional[str] = None
-    substring: Optional[str] = None
-    downloaded_at: Optional[str] = None
-    file_size: Optional[float] = None
 
-    # status in cdl
-    status: Optional[VideoStatus] = None
+@dataclass
+class FusedMedia:
+    """Fuses group models.
+    Used in fuses db."""
+
+    id: str
+    duration: int
+    total_parts: int
+    status: FusedStatus
+    checksum: str
+    file_path: str
+    created_at: str
+    updated_at: str
+    file_size: int
     fail_count: int = 0
+
+
+@dataclass
+class Creator:
+    """Creator model"""
+
+    id: str
+    name: str
+    service: str
+    domain: str
+    indexed: str
+    updated: str
+    favorited: int
+
+    status: CreatorStatus
+
+    # param
+    max_size: int
+    max_posts: int
+    min_date: str
+    max_date: str

rcdl/core/opti.py

@@ -0,0 +1,90 @@
+# core/opti.py
+
+"""
+Optimize media to reduce disk storage utilisation
+"""
+
+import os
+
+from rcdl.core.config import Config
+from rcdl.core.models import Status, Media
+from rcdl.core.db import DB
+from rcdl.core.downloader_subprocess import handbrake_optimized
+from rcdl.interface.ui import UI, NestedProgress
+from rcdl.utils import get_media_metadata, get_date_now
+
+
+def update_db(media: Media, user: str, result: int):
+    """Update DB if optimisation succesfful with new file_size, etc..."""
+    if result == 0:
+        path = os.path.join(Config.creator_folder(user), media.file_path)
+        _, file_size, checksum = get_media_metadata(path)
+        media.status = Status.OPTIMIZED
+        media.checksum = checksum
+        media.created_at = get_date_now()
+        media.file_size = file_size
+
+        with DB() as db:
+            db.update_media(media)
+
+
+def optimize():
+    """Optimize all medias in DB with DOWNLOADED
+    status that are not part of a fuse group"""
+    # get all video to opti
+    with DB() as db:
+        medias = db.query_media_by_status(Status.DOWNLOADED)
+        if Config.DEBUG:
+            medias.extend(db.query_media_by_status(Status.OPTIMIZED))
+
+    # progress
+    progress = NestedProgress(UI.console)
+    progress.start(
+        total=len(medias),
+        total_label="Optimizing videos",
+        current_label="Current video",
+    )
+
+    for media in medias:
+        # check media is not in a fuse group
+        with DB() as db:
+            fuse = db.query_fuses_by_id(media.post_id)
+        if fuse is not None:
+            progress.advance_total()
+            continue
+
+        # get post info
+        with DB() as db:
+            post = db.query_post_by_id(media.post_id)
+        if post is None:
+            UI.error(f"Could not match media {media.post_id} to a post by id")
+            progress.advance_total()
+            continue
+
+        result = handbrake_optimized(media, post.user, progress)
+
+        folder_path = Config.creator_folder(post.user)
+        video_path = os.path.join(folder_path, media.file_path)
+        output_path = video_path + ".opti.mp4"
+
+        if result == 0:
+            try:
+                os.replace(output_path, video_path)
+                update_db(media, post.user, result)
+            except FileNotFoundError as e:
+                UI.error(
+                    f"FileNotFoundError: Could not replace {video_path} "
+                    f"with {output_path} due to: {e}"
+                )
+            except PermissionError as e:
+                UI.error(
+                    f"PermissionError: Could not replace {video_path} "
+                    f"with {output_path} due to: {e}"
+                )
+            except OSError as e:
+                UI.error(
+                    f"OSError: Failed to replace {video_path} with {output_path} due to: {e}"
+                )
+            finally:
+                progress.advance_total()
+    progress.close()