sticker-convert 2.8.0__py3-none-any.whl → 2.8.2__py3-none-any.whl

sticker_convert/job.py

@@ -5,12 +5,10 @@ import os
 import shutil
 import traceback
 from datetime import datetime
-from multiprocessing import Process, Value
-from multiprocessing.managers import ListProxy, SyncManager
+from multiprocessing import Manager, Process, Value
 from pathlib import Path
-from queue import Queue
 from threading import Thread
-from typing import TYPE_CHECKING, Any, Callable, Dict, Generator, List, Optional, Tuple
+from typing import Any, Callable, Dict, List, Optional, Tuple
 from urllib.parse import urlparse
 
 from sticker_convert.converter import StickerConvert
@@ -23,18 +21,11 @@ from sticker_convert.uploaders.compress_wastickers import CompressWastickers
 from sticker_convert.uploaders.upload_signal import UploadSignal
 from sticker_convert.uploaders.upload_telegram import UploadTelegram
 from sticker_convert.uploaders.xcode_imessage import XcodeImessage
-from sticker_convert.utils.callback import CallbackReturn, CbQueueItemType
+from sticker_convert.utils.callback import CallbackReturn, CbQueueType, ResultsListType, WorkQueueType
 from sticker_convert.utils.files.json_resources_loader import OUTPUT_JSON
 from sticker_convert.utils.files.metadata_handler import MetadataHandler
 from sticker_convert.utils.media.codec_info import CodecInfo
 
-WorkListItemType = Optional[Tuple[Callable[..., Any], Tuple[Any, ...]]]
-if TYPE_CHECKING:
-    # mypy complains about this
-    WorkListType = ListProxy[WorkListItemType]  # type: ignore
-else:
-    WorkListType = List[WorkListItemType]
-
 
 class Executor:
     def __init__(
@@ -51,33 +42,29 @@ class Executor:
         self.cb_ask_bool = cb_ask_bool
         self.cb_ask_str = cb_ask_str
 
-        self.manager = SyncManager()
-        self.manager.start()
-        # Using list instead of queue for work_list as it can cause random deadlocks
-        # Especially when using scale_filter=nearest
-        self.work_list: WorkListType = self.manager.list()
-        self.results_queue: Queue[Any] = self.manager.Queue()
-        self.cb_queue: Queue[CbQueueItemType] = self.manager.Queue()
-        self.cb_return = CallbackReturn()
+        self.manager = Manager()
+        self.work_queue: WorkQueueType = self.manager.Queue()
+        self.cb_queue: CbQueueType = self.manager.Queue()
+        self.results_list: ResultsListType = self.manager.list()
+        self.cb_return = CallbackReturn(self.manager)
         self.processes: List[Process] = []
 
         self.is_cancel_job = Value("i", 0)
 
-        self.cb_thread_instance = Thread(
-            target=self.cb_thread,
-            args=(
-                self.cb_queue,
-                self.cb_return,
-            ),
-        )
-        self.cb_thread_instance.start()
+        self.cb_thread_instance: Optional[Thread] = None
 
     def cb_thread(
         self,
-        cb_queue: Queue[CbQueueItemType],
-        cb_return: CallbackReturn,
+        cb_queue: CbQueueType,
+        processes: int,
     ) -> None:
+        processes_done = 0
         for i in iter(cb_queue.get, None):
+            if i == "__PROCESS_DONE__":
+                processes_done += 1
+                if processes_done == processes:
+                    cb_queue.put(None)
+                continue
             if isinstance(i, tuple):
                 action = i[0]
                 if len(i) >= 2:
@@ -92,42 +79,44 @@ class Executor:
                 action = i
                 args = tuple()
                 kwargs = {}
-            if action == "msg":
-                self.cb_msg(*args, **kwargs)
-            elif action == "bar":
-                self.cb_bar(*args, **kwargs)
-            elif action == "update_bar":
-                self.cb_bar(update_bar=1)
-            elif action == "msg_block":
-                cb_return.set_response(self.cb_msg_block(*args, **kwargs))
-            elif action == "ask_bool":
-                cb_return.set_response(self.cb_ask_bool(*args, **kwargs))
-            elif action == "ask_str":
-                cb_return.set_response(self.cb_ask_str(*args, **kwargs))
-            else:
-                self.cb_msg(action)
+            self.cb(action, args, kwargs)
+
+    def cb(
+        self,
+        action: Optional[str],
+        args: Optional[Tuple[str, ...]] = None,
+        kwargs: Optional[Dict[str, Any]] = None,
+    ) -> None:
+        if args is None:
+            args = tuple()
+        if kwargs is None:
+            kwargs = {}
+        if action == "msg":
+            self.cb_msg(*args, **kwargs)
+        elif action == "bar":
+            self.cb_bar(*args, **kwargs)
+        elif action == "update_bar":
+            self.cb_bar(update_bar=1)
+        elif action == "msg_block":
+            self.cb_return.set_response(self.cb_msg_block(*args, **kwargs))
+        elif action == "ask_bool":
+            self.cb_return.set_response(self.cb_ask_bool(*args, **kwargs))
+        elif action == "ask_str":
+            self.cb_return.set_response(self.cb_ask_str(*args, **kwargs))
+        else:
+            self.cb_msg(action)
 
     @staticmethod
     def worker(
-        work_list: WorkListType,
-        results_queue: Queue[Any],
-        cb_queue: Queue[CbQueueItemType],
+        work_queue: WorkQueueType,
+        results_list: ResultsListType,
+        cb_queue: CbQueueType,
         cb_return: CallbackReturn,
     ) -> None:
-        while True:
-            try:
-                work = work_list.pop(0)
-            except IndexError:
-                break
-
-            if work is None:
-                break
-            else:
-                work_func, work_args = work
-
+        for work_func, work_args in iter(work_queue.get, None):
             try:
                 results = work_func(*work_args, cb_queue, cb_return)
-                results_queue.put(results)
+                results_list.append(results)
             except Exception:
                 arg_dump: List[Any] = []
                 for i in work_args:
@@ -142,15 +131,25 @@ class Executor:
                 e += "#####################"
                 cb_queue.put(e)
 
-        work_list.append(None)
+        work_queue.put(None)
+        cb_queue.put("__PROCESS_DONE__")
 
     def start_workers(self, processes: int = 1) -> None:
+        self.cb_thread_instance = Thread(
+            target=self.cb_thread,
+            args=(self.cb_queue, processes),
+        )
+        self.cb_thread_instance.start()
+
+        self.results_list[:] = []
+        while not self.work_queue.empty():
+            self.work_queue.get()
         for _ in range(processes):
             process = Process(
                 target=Executor.worker,
                 args=(
-                    self.work_list,
-                    self.results_queue,
+                    self.work_queue,
+                    self.results_list,
                     self.cb_queue,
                     self.cb_return,
                 ),
@@ -163,18 +162,16 @@ class Executor:
     def add_work(
         self, work_func: Callable[..., Any], work_args: Tuple[Any, ...]
     ) -> None:
-        self.work_list.append((work_func, work_args))
+        self.work_queue.put((work_func, work_args))
 
     def join_workers(self) -> None:
-        self.work_list.append(None)
+        self.work_queue.put(None)
         try:
             for process in self.processes:
                 process.join()
         except KeyboardInterrupt:
             pass
 
-        self.results_queue.put(None)
-        self.work_list[:] = []
         self.processes.clear()
 
     def kill_workers(self, *_: Any, **__: Any) -> None:
@@ -183,25 +180,14 @@ class Executor:
         for process in self.processes:
             process.terminate()
 
-        self.cleanup(killed=True)
+        self.cb_msg("Job cancelled.")
+        self.cleanup()
 
-    def cleanup(self, killed: bool = False) -> None:
-        if killed:
-            self.cb_queue.put("Job cancelled.")
-        self.cb_queue.put(("bar", None, {"set_progress_mode": "clear"}))
+    def cleanup(self) -> None:
+        self.cb_bar("clear")
         self.cb_queue.put(None)
-        self.cb_thread_instance.join()
-
-    def get_result(self) -> Generator[Any, None, None]:
-        yield from iter(self.results_queue.get, None)
-
-    def cb(
-        self,
-        action: Optional[str],
-        args: Optional[Tuple[str, ...]] = None,
-        kwargs: Optional[Dict[str, Any]] = None,
-    ) -> None:
-        self.cb_queue.put((action, args, kwargs))
+        if self.cb_thread_instance:
+            self.cb_thread_instance.join()
 
 
 class Job:
@@ -547,17 +533,18 @@ class Job:
             self.executor.cb("Nothing to download")
             return True
 
+        self.executor.start_workers(processes=1)
+
         for downloader in downloaders:
             self.executor.add_work(
                 work_func=downloader,
                 work_args=(self.opt_input.url, self.opt_input.dir, self.opt_cred),
             )
 
-        self.executor.start_workers(processes=1)
         self.executor.join_workers()
 
         # Return False if any of the job returns failure
-        for result in self.executor.get_result():
+        for result in self.executor.results_list:
             if result is False:
                 return False
 
@@ -613,12 +600,17 @@ class Job:
                 in_fs.append(i)
 
         in_fs_count = len(in_fs)
+        if in_fs_count == 0:
+            self.executor.cb("No files to compress")
+            return True
 
         self.executor.cb(msg)
         self.executor.cb(
             "bar", kwargs={"set_progress_mode": "determinate", "steps": in_fs_count}
         )
 
+        self.executor.start_workers(processes=min(self.opt_comp.processes, in_fs_count))
+
         for i in in_fs:
             in_f = input_dir / i.name
             out_f = output_dir / Path(i).stem
@@ -627,11 +619,10 @@ class Job:
                 work_func=StickerConvert.convert, work_args=(in_f, out_f, self.opt_comp)
             )
 
-        self.executor.start_workers(processes=min(self.opt_comp.processes, in_fs_count))
         self.executor.join_workers()
 
         # Return False if any of the job returns failure
-        for result in self.executor.get_result():
+        for result in self.executor.results_list:
             if result[0] is False:
                 return False
 
@@ -661,16 +652,17 @@ class Job:
         if self.opt_output.option == "imessage":
             exporters.append(XcodeImessage.start)
 
+        self.executor.start_workers(processes=1)
+
         for exporter in exporters:
             self.executor.add_work(
                 work_func=exporter,
                 work_args=(self.opt_output, self.opt_comp, self.opt_cred),
             )
 
-        self.executor.start_workers(processes=1)
         self.executor.join_workers()
 
-        for result in self.executor.get_result():
+        for result in self.executor.results_list:
             self.out_urls.extend(result)
 
         if self.out_urls:

sticker_convert/job_option.py

@@ -69,6 +69,8 @@ class CompOption(BaseOption):
 
     bg_color: Optional[str] = None
 
+    padding_percent: int = 0
+
     steps: int = 1
     fake_vid: Optional[bool] = None
     quantize_method: Optional[str] = None
@@ -104,6 +106,7 @@ class CompOption(BaseOption):
             "duration": {"min": self.duration_min, "max": self.duration_max},
             "steps": self.steps,
             "bg_color": self.bg_color,
+            "padding_percent": self.padding_percent,
             "fake_vid": self.fake_vid,
             "quantize_method": self.quantize_method,
             "scale_filter": self.scale_filter,

sticker_convert/resources/compression.json

@@ -38,6 +38,7 @@
             "min": 0,
             "max": 10000
         },
+        "padding_percent": 0,
         "bg_color": "",
         "steps": 16,
         "fake_vid": false,
@@ -84,6 +85,7 @@
             "min": 0,
             "max": 3000
         },
+        "padding_percent": 0,
         "bg_color": "",
         "steps": 16,
         "fake_vid": false,
@@ -130,6 +132,7 @@
             "min": 0,
             "max": 3000
         },
+        "padding_percent": 0,
         "bg_color": "",
         "steps": 16,
         "fake_vid": false,
@@ -176,6 +179,7 @@
             "min": 0,
             "max": 3000
         },
+        "padding_percent": 0,
         "bg_color": "",
         "steps": 16,
         "fake_vid": false,
@@ -222,6 +226,7 @@
             "min": 8,
             "max": 10000
         },
+        "padding_percent": 0,
         "bg_color": "",
         "steps": 16,
         "fake_vid": true,
@@ -268,6 +273,7 @@
             "min": 83,
             "max": 4000
         },
+        "padding_percent": 0,
         "bg_color": "",
         "steps": 16,
         "fake_vid": false,
@@ -314,6 +320,7 @@
             "min": 0,
             "max": 5000
         },
+        "padding_percent": 0,
         "bg_color": "",
         "steps": 16,
         "fake_vid": false,
@@ -360,6 +367,7 @@
             "min": 0,
             "max": 2000
         },
+        "padding_percent": 0,
         "bg_color": "",
         "steps": 16,
         "fake_vid": false,
@@ -406,6 +414,7 @@
             "min": 0,
             "max": 2000
         },
+        "padding_percent": 0,
         "bg_color": "",
         "steps": 16,
         "fake_vid": false,
@@ -452,6 +461,7 @@
             "min": 0,
             "max": 2000
         },
+        "padding_percent": 0,
         "bg_color": "",
         "steps": 16,
         "fake_vid": false,
@@ -498,6 +508,7 @@
             "min": 0,
             "max": 10000
         },
+        "padding_percent": 0,
         "bg_color": "",
         "steps": 16,
         "fake_vid": false,

sticker_convert/resources/help.json

@@ -39,6 +39,7 @@
         "duration": "Change playback speed if outside of duration limit.\nDuration set in miliseconds.\n0 will disable limit.",
         "duration_min": "Set minimum output duration in miliseconds.",
         "duration_max": "Set maximum output duration in miliseconds.",
+        "padding_percent": "Set percentage of space used as padding.",
         "bg_color": "Set custom background color.\nExample: 00ff00 for green.\nIf this is not set, background color would be auto set to black if image is bright, or white if image is dark.\nNote: The color should not be visible if output format supports transparency.",
         "size": "Set maximum file size in bytes for video and image.",
         "vid_size_max": "Set maximum file size limit for animated stickers.",

sticker_convert/uploaders/compress_wastickers.py

@@ -3,13 +3,12 @@ import copy
 import shutil
 import zipfile
 from pathlib import Path
-from queue import Queue
-from typing import Any, List, Union
+from typing import Any, List
 
 from sticker_convert.converter import StickerConvert
 from sticker_convert.job_option import CompOption, CredOption, OutputOption
 from sticker_convert.uploaders.upload_base import UploadBase
-from sticker_convert.utils.callback import Callback, CallbackReturn, CbQueueItemType
+from sticker_convert.utils.callback import CallbackProtocol, CallbackReturn
 from sticker_convert.utils.files.cache_store import CacheStore
 from sticker_convert.utils.files.metadata_handler import MetadataHandler
 from sticker_convert.utils.files.sanitize_filename import sanitize_filename
@@ -150,7 +149,7 @@ class CompressWastickers(UploadBase):
         opt_output: OutputOption,
         opt_comp: CompOption,
         opt_cred: CredOption,
-        cb: "Union[Queue[CbQueueItemType], Callback]",
+        cb: CallbackProtocol,
         cb_return: CallbackReturn,
     ) -> List[str]:
         exporter = CompressWastickers(opt_output, opt_comp, opt_cred, cb, cb_return)

sticker_convert/uploaders/upload_base.py

@@ -1,9 +1,7 @@
 #!/usr/bin/env python3
-from queue import Queue
-from typing import Union
 
 from sticker_convert.job_option import CompOption, CredOption, OutputOption
-from sticker_convert.utils.callback import Callback, CallbackReturn, CbQueueItemType
+from sticker_convert.utils.callback import CallbackProtocol, CallbackReturn
 
 
 class UploadBase:
@@ -12,13 +10,9 @@ class UploadBase:
         opt_output: OutputOption,
         opt_comp: CompOption,
         opt_cred: CredOption,
-        cb: "Union[Queue[CbQueueItemType], Callback]",
+        cb: CallbackProtocol,
         cb_return: CallbackReturn,
     ) -> None:
-        if not cb:
-            cb = Callback(silent=True)
-            cb_return = CallbackReturn()
-
         self.opt_output = opt_output
         self.opt_comp = opt_comp
         self.opt_cred = opt_cred

sticker_convert/uploaders/upload_signal.py

@@ -1,8 +1,7 @@
 #!/usr/bin/env python3
 import copy
 from pathlib import Path
-from queue import Queue
-from typing import Any, Dict, List, Union
+from typing import Any, Dict, List
 
 import anyio
 from signalstickers_client import StickersClient  # type: ignore
@@ -12,7 +11,7 @@ from signalstickers_client.models import LocalStickerPack, Sticker  # type: igno
 from sticker_convert.converter import StickerConvert
 from sticker_convert.job_option import CompOption, CredOption, OutputOption
 from sticker_convert.uploaders.upload_base import UploadBase
-from sticker_convert.utils.callback import Callback, CallbackReturn, CbQueueItemType
+from sticker_convert.utils.callback import CallbackProtocol, CallbackReturn
 from sticker_convert.utils.files.metadata_handler import MetadataHandler
 from sticker_convert.utils.media.codec_info import CodecInfo
 from sticker_convert.utils.media.format_verify import FormatVerify
@@ -167,7 +166,7 @@ class UploadSignal(UploadBase):
         opt_output: OutputOption,
         opt_comp: CompOption,
         opt_cred: CredOption,
-        cb: "Union[Queue[CbQueueItemType], Callback]",
+        cb: CallbackProtocol,
         cb_return: CallbackReturn,
     ) -> List[str]:
         exporter = UploadSignal(opt_output, opt_comp, opt_cred, cb, cb_return)

sticker_convert/uploaders/upload_telegram.py

@@ -2,7 +2,6 @@
 import copy
 import re
 from pathlib import Path
-from queue import Queue
 from typing import Any, Dict, List, Optional, Tuple, Union, cast
 
 import anyio
@@ -13,7 +12,7 @@ from telegram.ext import AIORateLimiter, ApplicationBuilder
 from sticker_convert.converter import StickerConvert
 from sticker_convert.job_option import CompOption, CredOption, OutputOption
 from sticker_convert.uploaders.upload_base import UploadBase
-from sticker_convert.utils.callback import Callback, CallbackReturn, CbQueueItemType
+from sticker_convert.utils.callback import CallbackProtocol, CallbackReturn
 from sticker_convert.utils.files.metadata_handler import MetadataHandler
 from sticker_convert.utils.media.format_verify import FormatVerify
 
@@ -79,7 +78,7 @@ class UploadTelegram(UploadBase):
         application = (  # type: ignore
             ApplicationBuilder()
             .token(self.opt_cred.telegram_token.strip())
-            .rate_limiter(AIORateLimiter())
+            .rate_limiter(AIORateLimiter(max_retries=3))
             .connect_timeout(timeout)
             .pool_timeout(timeout)
             .read_timeout(timeout)
@@ -350,7 +349,7 @@ class UploadTelegram(UploadBase):
         opt_output: OutputOption,
         opt_comp: CompOption,
         opt_cred: CredOption,
-        cb: "Union[Queue[CbQueueItemType], Callback]",
+        cb: CallbackProtocol,
         cb_return: CallbackReturn,
     ) -> List[str]:
         exporter = UploadTelegram(

sticker_convert/uploaders/xcode_imessage.py

@@ -6,14 +6,13 @@ import plistlib
 import shutil
 import zipfile
 from pathlib import Path
-from queue import Queue
-from typing import Any, Dict, List, Union
+from typing import Any, Dict, List
 
 from sticker_convert.converter import StickerConvert
 from sticker_convert.definitions import ROOT_DIR
 from sticker_convert.job_option import CompOption, CredOption, OutputOption
 from sticker_convert.uploaders.upload_base import UploadBase
-from sticker_convert.utils.callback import Callback, CallbackReturn, CbQueueItemType
+from sticker_convert.utils.callback import CallbackProtocol, CallbackReturn
 from sticker_convert.utils.files.metadata_handler import XCODE_IMESSAGE_ICONSET, MetadataHandler
 from sticker_convert.utils.files.sanitize_filename import sanitize_filename
 from sticker_convert.utils.media.codec_info import CodecInfo
@@ -279,7 +278,7 @@ class XcodeImessage(UploadBase):
         opt_output: OutputOption,
         opt_comp: CompOption,
         opt_cred: CredOption,
-        cb: "Union[Queue[CbQueueItemType], Callback]",
+        cb: CallbackProtocol,
         cb_return: CallbackReturn,
     ) -> List[str]:
         exporter = XcodeImessage(opt_output, opt_comp, opt_cred, cb, cb_return)

sticker_convert/utils/callback.py

@@ -1,39 +1,57 @@
 #!/usr/bin/env python3
-from multiprocessing import Event, Queue
-from typing import Any, Callable, Dict, Optional, Tuple, Union
+from multiprocessing import Event, Manager
+from multiprocessing.managers import ListProxy, SyncManager
+from queue import Queue
+from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Protocol, Tuple, Union
 
 from tqdm import tqdm
 
 CbQueueTupleType = Tuple[
     Optional[str], Optional[Tuple[Any, ...]], Optional[Dict[str, Any]]
 ]
-CbQueueItemType = Union[
-    CbQueueTupleType,
-    str,
-    None,
-]
+CbQueueItemType = Union[CbQueueTupleType, str, None]
+WorkQueueItemType = Optional[Tuple[Callable[..., Any], Tuple[Any, ...]]]
+ResponseItemType = Union[bool, str, None]
+
+if TYPE_CHECKING:
+    # mypy complains about this
+    ResultsListType = ListProxy[Any]  # type: ignore
+    ResponseListType = ListProxy[ResponseItemType]  # type: ignore
+    CbQueueType = Queue[CbQueueItemType]  # type: ignore
+    WorkQueueType = Queue[WorkQueueItemType]  # type: ignore
+else:
+    ResultsListType = List[Any]
+    ResponseListType = List[ResponseItemType]
+    CbQueueType = Queue
+    WorkQueueType = Queue
 
 
 class CallbackReturn:
-    def __init__(self) -> None:
+    def __init__(self, manager: Optional[SyncManager] = None) -> None:
         self.response_event = Event()
-        self.response_queue: Queue[Union[bool, str, None]] = Queue()
+        if manager is None:
+            manager = Manager()
+        self.response_queue: ResponseListType = manager.list()
 
-    def set_response(self, response: Union[bool, str, None]) -> None:
-        self.response_queue.put(response)
+    def set_response(self, response: ResponseItemType) -> None:
+        self.response_queue.append(response)
         self.response_event.set()
 
-    def get_response(self) -> Union[bool, str, None]:
+    def get_response(self) -> ResponseItemType:
         self.response_event.wait()
 
-        response = self.response_queue.get()
+        response = self.response_queue.pop()
 
         self.response_event.clear()
 
         return response
 
 
-class Callback:
+class CallbackProtocol(Protocol):
+    def put(self, i: Union[CbQueueItemType, str]) -> Any: ...
+
+
+class Callback(CallbackProtocol):
     def __init__(
         self,
         msg: Optional[Callable[..., None]] = None,
@@ -155,13 +173,7 @@ class Callback:
 
         return response
 
-    def put(
-        self,
-        i: Union[
-            CbQueueItemType,
-            str,
-        ],
-    ) -> Union[str, bool, None]:
+    def put(self, i: Union[CbQueueItemType, str]) -> Union[str, bool, None]:
         if isinstance(i, tuple):
             action = i[0]
             if len(i) >= 2:

sticker_convert/utils/media/codec_info.py

@@ -258,11 +258,7 @@ class CodecInfo:
                     fps = frames_apparent / total_duration * 1000
                 return fps, frames, total_duration
 
-        return (
-            0.0,
-            1,
-            0,
-        )
+        return 0.0, 1, 0
 
     @staticmethod
     def _get_file_fps_frames_duration_webp(
@@ -294,8 +290,8 @@ class CodecInfo:
                     total_duration += frame_duration
                     frames += 1
 
-        if frames == 0:
-            return 0.0, 0, 0
+        if frames <= 1:
+            return 0.0, 1, 0
 
         if len(durations) == 1:
             fps = frames / total_duration * 1000

sticker_convert/version.py

@@ -1,3 +1,3 @@
 #!/usr/bin/env python3
 
-__version__ = "2.8.0"
+__version__ = "2.8.2"