sticker-convert 2.7.2__py3-none-any.whl → 2.7.4__py3-none-any.whl

sticker_convert/job.py

@@ -7,15 +7,14 @@ import shutil
 import traceback
 from datetime import datetime
 from multiprocessing import Process, Value
-from multiprocessing.managers import SyncManager
+from multiprocessing.managers import ListProxy, SyncManager
 from pathlib import Path
 from queue import Queue
 from threading import Thread
-from typing import Any, Callable, Generator, Optional, Union
+from typing import TYPE_CHECKING, Any, Callable, Dict, Generator, Iterator, List, Optional, Tuple
 from urllib.parse import urlparse
 
 from sticker_convert.converter import StickerConvert
-from sticker_convert.definitions import ROOT_DIR
 from sticker_convert.downloaders.download_kakao import DownloadKakao
 from sticker_convert.downloaders.download_line import DownloadLine
 from sticker_convert.downloaders.download_signal import DownloadSignal
@@ -25,11 +24,19 @@ from sticker_convert.uploaders.compress_wastickers import CompressWastickers
 from sticker_convert.uploaders.upload_signal import UploadSignal
 from sticker_convert.uploaders.upload_telegram import UploadTelegram
 from sticker_convert.uploaders.xcode_imessage import XcodeImessage
-from sticker_convert.utils.callback import CallbackReturn
-from sticker_convert.utils.files.json_manager import JsonManager
+from sticker_convert.utils.callback import CallbackReturn, CbQueueItemType
+from sticker_convert.utils.files.json_resources_loader import OUTPUT_JSON
 from sticker_convert.utils.files.metadata_handler import MetadataHandler
 from sticker_convert.utils.media.codec_info import CodecInfo
 
+CbQueueType = Queue[CbQueueItemType]
+WorkListItemType = Optional[Tuple[Callable[..., Any], Tuple[Any, ...]]]
+if TYPE_CHECKING:
+    # mypy complains about this
+    WorkListType = ListProxy[WorkListItemType]  # type: ignore
+else:
+    WorkListType = List[WorkListItemType]
+
 
 class Executor:
     def __init__(
@@ -39,7 +46,7 @@ class Executor:
         cb_bar: Callable[..., None],
         cb_ask_bool: Callable[..., bool],
         cb_ask_str: Callable[..., str],
-    ):
+    ) -> None:
         self.cb_msg = cb_msg
         self.cb_msg_block = cb_msg_block
         self.cb_bar = cb_bar
@@ -48,21 +55,13 @@ class Executor:
 
         self.manager = SyncManager()
         self.manager.start()
-        self.work_queue: Queue[
-            Optional[tuple[Callable[..., Any], tuple[Any, ...]]]
-        ] = self.manager.Queue()
+        # Using list instead of queue for work_list as it can cause random deadlocks
+        # Especially when using scale_filter=nearest
+        self.work_list: WorkListType = self.manager.list()
         self.results_queue: Queue[Any] = self.manager.Queue()
-        self.cb_queue: Queue[
-            Union[
-                tuple[
-                    Optional[str], Optional[tuple[Any, ...]], Optional[dict[str, str]]
-                ],
-                str,
-                None,
-            ]
-        ] = self.manager.Queue()
+        self.cb_queue: CbQueueType = self.manager.Queue()
         self.cb_return = CallbackReturn()
-        self.processes: list[Process] = []
+        self.processes: List[Process] = []
 
         self.is_cancel_job = Value("i", 0)
 
@@ -77,31 +76,24 @@ class Executor:
 
     def cb_thread(
         self,
-        cb_queue: Queue[
-            Union[
-                tuple[
-                    Optional[str], Optional[tuple[Any, ...]], Optional[dict[str, str]]
-                ],
-                str,
-            ]
-        ],
+        cb_queue: CbQueueType,
         cb_return: CallbackReturn,
-    ):
+    ) -> None:
         for i in iter(cb_queue.get, None):
             if isinstance(i, tuple):
                 action = i[0]
                 if len(i) >= 2:
-                    args: tuple[str, ...] = i[1] if i[1] else tuple()
+                    args: Tuple[str, ...] = i[1] if i[1] else tuple()
                 else:
                     args = tuple()
                 if len(i) >= 3:
-                    kwargs: dict[str, str] = i[2] if i[2] else dict()
+                    kwargs: Dict[str, str] = i[2] if i[2] else {}
                 else:
-                    kwargs = dict()
+                    kwargs = {}
             else:
                 action = i
                 args = tuple()
-                kwargs = dict()
+                kwargs = {}
             if action == "msg":
                 self.cb_msg(*args, **kwargs)
             elif action == "bar":
@@ -119,24 +111,27 @@ class Executor:
 
     @staticmethod
     def worker(
-        work_queue: Queue[Optional[tuple[Callable[..., Any], tuple[Any, ...]]]],
+        work_list: WorkListType,
         results_queue: Queue[Any],
-        cb_queue: Queue[
-            Union[
-                tuple[
-                    Optional[str], Optional[tuple[Any, ...]], Optional[dict[str, str]]
-                ],
-                str,
-            ]
-        ],
+        cb_queue: CbQueueType,
         cb_return: CallbackReturn,
-    ):
-        for work_func, work_args in iter(work_queue.get, None):
+    ) -> None:
+        while True:
+            try:
+                work = work_list.pop(0)
+            except IndexError:
+                break
+
+            if work is None:
+                break
+            else:
+                work_func, work_args = work
+
             try:
                 results = work_func(*work_args, cb_queue, cb_return)
                 results_queue.put(results)
             except Exception:
-                arg_dump: list[Any] = []
+                arg_dump: List[Any] = []
                 for i in work_args:
                     if isinstance(i, CredOption):
                         arg_dump.append("CredOption(REDACTED)")
@@ -148,18 +143,14 @@ class Executor:
                 e += traceback.format_exc()
                 e += "#####################"
                 cb_queue.put(e)
-        work_queue.put(None)
-
-    def start_workers(self, processes: int = 1):
-        # Would contain None from previous run
-        while not self.work_queue.empty():
-            self.work_queue.get()
+        work_list.append(None)
 
+    def start_workers(self, processes: int = 1) -> None:
         for _ in range(processes):
             process = Process(
                 target=Executor.worker,
                 args=(
-                    self.work_queue,
+                    self.work_list,
                     self.results_queue,
                     self.cb_queue,
                     self.cb_return,
@@ -170,11 +161,13 @@ class Executor:
             process.start()
             self.processes.append(process)
 
-    def add_work(self, work_func: Callable[..., Any], work_args: tuple[Any, ...]):
-        self.work_queue.put((work_func, work_args))
+    def add_work(
+        self, work_func: Callable[..., Any], work_args: Tuple[Any, ...]
+    ) -> None:
+        self.work_list.append((work_func, work_args))
 
-    def join_workers(self):
-        self.work_queue.put(None)
+    def join_workers(self) -> None:
+        self.work_list.append(None)
         try:
             for process in self.processes:
                 process.join()
@@ -182,13 +175,12 @@ class Executor:
             pass
 
         self.results_queue.put(None)
+        self.work_list = self.manager.list()
+        self.processes.clear()
 
-        self.process = []
-
-    def kill_workers(self, *args: Any, **kwargs: Any):
+    def kill_workers(self, *_: Any, **__: Any) -> None:
         self.is_cancel_job.value = 1  # type: ignore
-        while not self.work_queue.empty():
-            self.work_queue.get()
+        self.work_list = self.manager.list()
 
         for process in self.processes:
             if platform.system() == "Windows":
@@ -199,20 +191,21 @@ class Executor:
 
         self.cleanup()
 
-    def cleanup(self):
+    def cleanup(self) -> None:
         self.cb_queue.put(None)
         self.cb_thread_instance.join()
+        self.work_list = self.manager.list()
 
     def get_result(self) -> Generator[Any, None, None]:
-        for result in iter(self.results_queue.get, None):
-            yield result
+        gen: Iterator[Any] = iter(self.results_queue.get, None)
+        yield from gen
 
     def cb(
         self,
         action: Optional[str],
-        args: Optional[tuple[str, ...]] = None,
-        kwargs: Optional[dict[str, Any]] = None,
-    ):
+        args: Optional[Tuple[str, ...]] = None,
+        kwargs: Optional[Dict[str, Any]] = None,
+    ) -> None:
         self.cb_queue.put((action, args, kwargs))
 
 
@@ -228,7 +221,7 @@ class Job:
         cb_bar: Callable[..., None],
         cb_ask_bool: Callable[..., bool],
         cb_ask_str: Callable[..., str],
-    ):
+    ) -> None:
         self.opt_input = opt_input
         self.opt_comp = opt_comp
         self.opt_output = opt_output
@@ -239,8 +232,8 @@ class Job:
         self.cb_ask_bool = cb_ask_bool
         self.cb_ask_str = cb_ask_str
 
-        self.compress_fails: list[str] = []
-        self.out_urls: list[str] = []
+        self.compress_fails: List[str] = []
+        self.out_urls: List[str] = []
 
         self.executor = Executor(
             self.cb_msg,
@@ -277,7 +270,7 @@ class Job:
                 code = 2
                 self.executor.cb("Job cancelled.")
                 break
-            elif not success:
+            if not success:
                 code = 1
                 self.executor.cb("An error occured during this run.")
                 break
@@ -287,7 +280,7 @@ class Job:
 
         return code
 
-    def cancel(self, *args: Any, **kwargs: Any):
+    def cancel(self, *_: Any, **_kwargs: Any) -> None:
         self.executor.kill_workers()
 
     def verify_input(self) -> bool:
@@ -332,7 +325,7 @@ class Job:
             error_msg += "[X] Uploading to signal requires uuid and password.\n"
             error_msg += save_to_local_tip
 
-        output_presets = JsonManager.load_json(ROOT_DIR / "resources/output.json")
+        output_presets = OUTPUT_JSON
 
         input_option = self.opt_input.option
         output_option = self.opt_output.option
@@ -401,7 +394,9 @@ class Job:
 
         if self.opt_comp.scale_filter not in (
             "nearest",
+            "box",
             "bilinear",
+            "hamming",
             "bicubic",
             "lanczos",
         ):
@@ -409,7 +404,9 @@ class Job:
             error_msg += (
                 f"[X] scale_filter {self.opt_comp.scale_filter} is not valid option"
             )
-            error_msg += "    Valid options: nearest, bilinear, bicubic, lanczos"
+            error_msg += (
+                "    Valid options: nearest, box, bilinear, hamming, bicubic, lanczos"
+            )
 
         if self.opt_comp.quantize_method not in ("imagequant", "fastoctree", "none"):
             error_msg += "\n"
@@ -448,7 +445,7 @@ class Job:
                 or not any(path.iterdir())
             ):
                 continue
-            elif path_type == "Output" and (
+            if path_type == "Output" and (
                 path.name == "stickers_output"
                 or self.opt_comp.no_compress
                 or not any(path.iterdir())
@@ -456,7 +453,7 @@ class Job:
                 continue
 
             related_files = MetadataHandler.get_files_related_to_sticker_convert(path)
-            if any([i for i in path.iterdir() if i not in related_files]):
+            if any(i for i in path.iterdir() if i not in related_files):
                 msg = "WARNING: {} directory is set to {}.\n"
                 msg += 'It does not have default name of "{}",\n'
                 msg += "and It seems like it contains PERSONAL DATA.\n"
@@ -528,7 +525,7 @@ class Job:
         return True
 
     def download(self) -> bool:
-        downloaders: list[Callable[..., bool]] = []
+        downloaders: List[Callable[..., bool]] = []
 
         if self.opt_input.option == "signal":
             downloaders.append(DownloadSignal.start)
@@ -548,14 +545,13 @@ class Job:
             self.executor.cb("Nothing to download")
             return True
 
-        self.executor.start_workers(processes=1)
-
         for downloader in downloaders:
             self.executor.add_work(
                 work_func=downloader,
                 work_args=(self.opt_input.url, self.opt_input.dir, self.opt_cred),
             )
 
+        self.executor.start_workers(processes=1)
         self.executor.join_workers()
 
         # Return False if any of the job returns failure
@@ -600,7 +596,7 @@ class Job:
         input_dir = Path(self.opt_input.dir)
         output_dir = Path(self.opt_output.dir)
 
-        in_fs: list[Path] = []
+        in_fs: List[Path] = []
 
         # .txt: emoji.txt, title.txt
         # .m4a: line sticker sound effects
@@ -609,9 +605,7 @@ class Job:
 
             if not in_f.is_file():
                 continue
-            elif (
-                CodecInfo.get_file_ext(i) in (".txt", ".m4a") or Path(i).stem == "cover"
-            ):
+            if CodecInfo.get_file_ext(i) in (".txt", ".m4a") or Path(i).stem == "cover":
                 shutil.copy(in_f, output_dir / i.name)
             else:
                 in_fs.append(i)
@@ -623,8 +617,6 @@ class Job:
             "bar", kwargs={"set_progress_mode": "determinate", "steps": in_fs_count}
         )
 
-        self.executor.start_workers(processes=min(self.opt_comp.processes, in_fs_count))
-
         for i in in_fs:
             in_f = input_dir / i.name
             out_f = output_dir / Path(i).stem
@@ -633,6 +625,7 @@ class Job:
                 work_func=StickerConvert.convert, work_args=(in_f, out_f, self.opt_comp)
             )
 
+        self.executor.start_workers(processes=min(self.opt_comp.processes, in_fs_count))
         self.executor.join_workers()
 
         # Return False if any of the job returns failure
@@ -649,7 +642,7 @@ class Job:
 
         self.executor.cb("Exporting...")
 
-        exporters: list[Callable[..., list[str]]] = []
+        exporters: List[Callable[..., List[str]]] = []
 
         if self.opt_output.option == "whatsapp":
             exporters.append(CompressWastickers.start)
@@ -666,21 +659,22 @@ class Job:
         if self.opt_output.option == "imessage":
             exporters.append(XcodeImessage.start)
 
-        self.executor.start_workers(processes=1)
-
         for exporter in exporters:
             self.executor.add_work(
                 work_func=exporter,
                 work_args=(self.opt_output, self.opt_comp, self.opt_cred),
             )
 
+        self.executor.start_workers(processes=1)
         self.executor.join_workers()
 
         for result in self.executor.get_result():
             self.out_urls.extend(result)
 
         if self.out_urls:
-            with open(Path(self.opt_output.dir, "export-result.txt"), "w+") as f:
+            with open(
+                Path(self.opt_output.dir, "export-result.txt"), "w+", encoding="utf-8"
+            ) as f:
                 f.write("\n".join(self.out_urls))
         else:
             self.executor.cb("An error occured while exporting stickers")
@@ -694,14 +688,14 @@ class Job:
         msg += "##########\n"
         msg += "\n"
 
-        if self.compress_fails != []:
+        if self.compress_fails:
             msg += f'Warning: Could not compress the following {len(self.compress_fails)} file{"s" if len(self.compress_fails) > 1 else ""}:\n'
             msg += "\n".join(self.compress_fails)
             msg += "\n"
             msg += "\nConsider adjusting compression parameters"
             msg += "\n"
 
-        if self.out_urls != []:
+        if self.out_urls:
             msg += "Export results:\n"
             msg += "\n".join(self.out_urls)
         else:

sticker_convert/job_option.py

@@ -6,7 +6,7 @@ from dataclasses import dataclass
 from math import ceil
 from multiprocessing import cpu_count
 from pathlib import Path
-from typing import Any, Optional, Union
+from typing import Any, Dict, Optional, Tuple, Union
 
 
 def to_int(i: Union[float, str, None]) -> Optional[int]:
@@ -15,7 +15,7 @@ def to_int(i: Union[float, str, None]) -> Optional[int]:
 
 @dataclass
 class BaseOption:
-    def merge(self, config: "BaseOption"):
+    def merge(self, config: "BaseOption") -> None:
         for k, v in vars(config).items():
             if v is not None:
                 setattr(self, k, v)
@@ -23,8 +23,8 @@ class BaseOption:
     def __repr__(self) -> str:
         return json.dumps(self.to_dict(), indent=2)
 
-    def to_dict(self) -> dict[str, str]:
-        return dict()
+    def to_dict(self) -> Dict[str, str]:
+        return {}
 
 
 @dataclass
@@ -33,7 +33,7 @@ class InputOption(BaseOption):
     url: str = ""
     dir: Path = Path()
 
-    def to_dict(self) -> dict[Any, Any]:
+    def to_dict(self) -> Dict[Any, Any]:
         return {"option": self.option, "url": self.url, "dir": self.dir.as_posix()}
 
 
@@ -43,8 +43,8 @@ class CompOption(BaseOption):
     size_max_img: Optional[int] = None
     size_max_vid: Optional[int] = None
 
-    format_img: tuple[str, ...] = tuple()
-    format_vid: tuple[str, ...] = tuple()
+    format_img: Tuple[str, ...] = tuple()
+    format_vid: Tuple[str, ...] = tuple()
 
     fps_min: Optional[int] = None
     fps_max: Optional[int] = None
@@ -78,7 +78,7 @@ class CompOption(BaseOption):
     animated: Optional[bool] = None
     square: Optional[bool] = None
 
-    def to_dict(self) -> dict[Any, Any]:
+    def to_dict(self) -> Dict[Any, Any]:
         return {
             "preset": self.preset,
             "size_max": {"img": self.size_max_img, "vid": self.size_max_vid},
@@ -111,75 +111,79 @@ class CompOption(BaseOption):
             "square": self.square,
         }
 
-    def get_size_max(self) -> tuple[Optional[int], Optional[int]]:
+    def get_size_max(self) -> Tuple[Optional[int], Optional[int]]:
         return (self.size_max_img, self.size_max_vid)
 
-    def set_size_max(self, value: Optional[int]):
+    def set_size_max(self, value: Optional[int]) -> None:
         self.size_max_img, self.size_max_vid = to_int(value), to_int(value)
 
-    def get_format(self) -> tuple[tuple[str, ...], tuple[str, ...]]:
+    def get_format(self) -> Tuple[Tuple[str, ...], Tuple[str, ...]]:
         return (self.format_img, self.format_vid)
 
-    def set_format(self, value: tuple[str, ...]):
+    def set_format(self, value: Tuple[str, ...]) -> None:
         self.format_img, self.format_vid = value, value
 
-    def get_fps(self) -> tuple[Optional[int], Optional[int]]:
+    def get_fps(self) -> Tuple[Optional[int], Optional[int]]:
         return (self.fps_min, self.fps_max)
 
-    def set_fps(self, value: Optional[int]):
+    def set_fps(self, value: Optional[int]) -> None:
         self.fps_min, self.fps_max = to_int(value), to_int(value)
 
-    def get_res(self) -> tuple[tuple[Optional[int], Optional[int]], tuple[Optional[int], Optional[int]]]:
+    def get_res(
+        self,
+    ) -> Tuple[
+        Tuple[Optional[int], Optional[int]], Tuple[Optional[int], Optional[int]]
+    ]:
         return (self.get_res_w(), self.get_res_h())
 
-    def set_res(self, value: Optional[int]):
+    def set_res(self, value: Optional[int]) -> None:
         self.res_w_min = to_int(value)
         self.res_w_max = to_int(value)
         self.res_h_min = to_int(value)
         self.res_h_max = to_int(value)
 
-    def get_res_max(self) -> tuple[Optional[int], Optional[int]]:
+    def get_res_max(self) -> Tuple[Optional[int], Optional[int]]:
         return (self.res_w_max, self.res_h_max)
 
-    def set_res_max(self, value: Optional[int]):
+    def set_res_max(self, value: Optional[int]) -> None:
         self.res_w_max = to_int(value)
         self.res_h_max = to_int(value)
 
-    def get_res_min(self) -> tuple[Optional[int], Optional[int]]:
+    def get_res_min(self) -> Tuple[Optional[int], Optional[int]]:
         return (self.res_w_min, self.res_h_min)
 
-    def set_res_min(self, value: Optional[int]):
+    def set_res_min(self, value: Optional[int]) -> None:
         self.res_w_min = to_int(value)
         self.res_h_min = to_int(value)
 
-    def get_res_w(self) -> tuple[Optional[int], Optional[int]]:
+    def get_res_w(self) -> Tuple[Optional[int], Optional[int]]:
         return (self.res_w_min, self.res_w_max)
 
-    def set_res_w(self, value: Optional[int]):
+    def set_res_w(self, value: Optional[int]) -> None:
         self.res_w_min, self.res_w_max = to_int(value), to_int(value)
 
-    def get_res_h(self) -> tuple[Optional[int], Optional[int]]:
+    def get_res_h(self) -> Tuple[Optional[int], Optional[int]]:
         return (self.res_h_min, self.res_h_max)
 
-    def set_res_h(self, value: Optional[int]):
+    def set_res_h(self, value: Optional[int]) -> None:
         self.res_h_min, self.res_h_max = to_int(value), to_int(value)
 
-    def get_quality(self) -> tuple[Optional[int], Optional[int]]:
+    def get_quality(self) -> Tuple[Optional[int], Optional[int]]:
         return (self.quality_min, self.quality_max)
 
-    def set_quality(self, value: Optional[int]):
+    def set_quality(self, value: Optional[int]) -> None:
         self.quality_min, self.quality_max = to_int(value), to_int(value)
 
-    def get_color(self) -> tuple[Optional[int], Optional[int]]:
+    def get_color(self) -> Tuple[Optional[int], Optional[int]]:
         return (self.color_min, self.color_max)
 
-    def set_color(self, value: Optional[int]):
+    def set_color(self, value: Optional[int]) -> None:
         self.color_min, self.color_max = to_int(value), to_int(value)
 
-    def get_duration(self) -> tuple[Optional[int], Optional[int]]:
+    def get_duration(self) -> Tuple[Optional[int], Optional[int]]:
         return (self.duration_min, self.duration_max)
 
-    def set_duration(self, value: Optional[int]):
+    def set_duration(self, value: Optional[int]) -> None:
         self.duration_min, self.duration_max = to_int(value), to_int(value)
 
 
@@ -190,7 +194,7 @@ class OutputOption(BaseOption):
     title: str = ""
     author: str = ""
 
-    def to_dict(self) -> dict[Any, Any]:
+    def to_dict(self) -> Dict[Any, Any]:
         return {
             "option": self.option,
             "dir": self.dir.as_posix(),
@@ -212,7 +216,7 @@ class CredOption(BaseOption):
     kakao_phone_number: str = ""
     line_cookies: str = ""
 
-    def to_dict(self) -> dict[Any, Any]:
+    def to_dict(self) -> Dict[Any, Any]:
         return {
             "signal": {"uuid": self.signal_uuid, "password": self.signal_password},
             "telegram": {"token": self.telegram_token, "userid": self.telegram_userid},