cobweb-launcher 1.3.14__py3-none-any.whl → 3.1.0__py3-none-any.whl

cobweb/utils/oss.py

@@ -1,9 +1,9 @@
-# from typing import List
+
 from cobweb import setting
 from requests import Response
 from oss2 import Auth, Bucket, models, PartIterator
 from cobweb.exceptions import oss_db_exception
-from cobweb.base import Decorators
+from cobweb.base.decorators import decorator_oss_db
 
 
 class OssUtil:
@@ -23,6 +23,9 @@ class OssUtil:
         self.chunk_size = int(chunk_size or setting.OSS_CHUNK_SIZE)
         self.min_upload_size = int(min_upload_size or setting.OSS_MIN_UPLOAD_SIZE)
 
+        self.failed_count = 0
+        self._kw = kwargs
+
         self._auth = Auth(
             access_key_id=access_key or setting.OSS_ACCESS_KEY,
             access_key_secret=secret_key or setting.OSS_SECRET_KEY
@@ -31,26 +34,42 @@ class OssUtil:
             auth=self._auth,
             endpoint=self.endpoint,
             bucket_name=self.bucket,
-            **kwargs
+            **self._kw
         )
 
+    def failed(self):
+        self.failed_count += 1
+        if self.failed_count >= 5:
+            self._client = Bucket(
+                auth=self._auth,
+                endpoint=self.endpoint,
+                bucket_name=self.bucket,
+                **self._kw
+            )
+
     def exists(self, key: str) -> bool:
-        return self._client.object_exists(key)
+        try:
+            result = self._client.object_exists(key)
+            self.failed_count = 0
+            return result
+        except Exception as e:
+            self.failed()
+            raise e
 
     def head(self, key: str) -> models.HeadObjectResult:
         return self._client.head_object(key)
 
-    @Decorators.decorator_oss_db(exception=oss_db_exception.OssDBInitPartError)
+    @decorator_oss_db(exception=oss_db_exception.OssDBInitPartError)
     def init_part(self, key) -> models.InitMultipartUploadResult:
         """初始化分片上传"""
         return self._client.init_multipart_upload(key)
 
-    @Decorators.decorator_oss_db(exception=oss_db_exception.OssDBPutObjError)
+    @decorator_oss_db(exception=oss_db_exception.OssDBPutObjError)
     def put(self, key, data) -> models.PutObjectResult:
         """文件上传"""
         return self._client.put_object(key, data)
 
-    @Decorators.decorator_oss_db(exception=oss_db_exception.OssDBPutPartError)
+    @decorator_oss_db(exception=oss_db_exception.OssDBPutPartError)
     def put_part(self, key, upload_id, position, data) -> models.PutObjectResult:
         """分片上传"""
         return self._client.upload_part(key, upload_id, position, data)
@@ -59,13 +78,13 @@ class OssUtil:
         """获取分片列表"""
         return [part_info for part_info in PartIterator(self._client, key, upload_id)]
 
-    @Decorators.decorator_oss_db(exception=oss_db_exception.OssDBMergeError)
+    @decorator_oss_db(exception=oss_db_exception.OssDBMergeError)
     def merge(self, key, upload_id, parts=None) -> models.PutObjectResult:
         """合并分片"""
         headers = None if parts else {"x-oss-complete-all": "yes"}
         return self._client.complete_multipart_upload(key, upload_id, parts, headers=headers)
 
-    @Decorators.decorator_oss_db(exception=oss_db_exception.OssDBAppendObjError)
+    @decorator_oss_db(exception=oss_db_exception.OssDBAppendObjError)
     def append(self, key, position, data) -> models.AppendObjectResult:
         """追加上传"""
         return self._client.append_object(key, position, data)

{cobweb_launcher-1.3.14.dist-info → cobweb_launcher-3.1.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.3.14
+Version: 3.1.0
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

cobweb_launcher-3.1.0.dist-info/RECORD

@@ -0,0 +1,41 @@
+cobweb/__init__.py,sha256=UfNq1asNq7_a8IKf5WCbk0ju5fxT4wElAyaItf_a-d8,67
+cobweb/constant.py,sha256=zy3XYsc1qp2B76_Fn_hVQ8eGHlPBd3OFlZK2cryE6FY,2839
+cobweb/setting.py,sha256=yP9ZkVShTY4sCZ4DpzI_hO-FMSf0vGrQvkp7UCmZaa0,2338
+cobweb/base/__init__.py,sha256=epVQttTHQcux9kAtrGuO9HB_wP74L-pi74vhzc4QOCw,224
+cobweb/base/common_queue.py,sha256=W7PPZZFl52j3Mc916T0imHj7oAUelA6aKJwW-FecDPE,872
+cobweb/base/item.py,sha256=hYheVTV2Bozp4iciJpE2ZwBIXkaqBg4QQkRccP8yoVk,1049
+cobweb/base/log.py,sha256=L01hXdk3L2qEm9X1FOXQ9VmWIoHSELe0cyZvrdAN61A,2003
+cobweb/base/request.py,sha256=tEkgMVUfdQI-kZuzWuiit9P_q4Q9-_RZh9aXXpc0314,2352
+cobweb/base/response.py,sha256=eB1DWMXFCpn3cJ3yzgCRU1WeZAdayGDohRgdjdMUFN4,406
+cobweb/base/seed.py,sha256=A-F1urjbE5hYNWTCwq3sUV4nrxlK_RGMoCmjBmIwYsI,3158
+cobweb/crawlers/__init__.py,sha256=msvkB9mTpsgyj8JfNMsmwAcpy5kWk_2NrO1Adw2Hkw0,29
+cobweb/crawlers/crawler.py,sha256=73WFGVNvIFvH8cP5RfDhAhM-WaFL_mdwnAhNohDnBO0,696
+cobweb/db/__init__.py,sha256=uZwSkd105EAwYo95oZQXAfofUKHVIAZZIPpNMy-hm2Q,56
+cobweb/db/api_db.py,sha256=bDc5dJQxq4z04h70KUTHd0OqUOEY7Cm3wcNJZtTvJIM,3015
+cobweb/db/redis_db.py,sha256=rK6PnKLLVTeg6HMGVtlYl4_b5og3aO2J5jZrqU2Aoso,7721
+cobweb/exceptions/__init__.py,sha256=E9SHnJBbhD7fOgPFMswqyOf8SKRDrI_i25L0bSpohvk,32
+cobweb/exceptions/oss_db_exception.py,sha256=iP_AImjNHT3-Iv49zCFQ3rdLnlvuHa3h2BXApgrOYpA,636
+cobweb/launchers/__init__.py,sha256=XEW4hmBMPKp2bRkHBxxlplXwvKdlLVYBFg3etjtbRXo,222
+cobweb/launchers/distributor.py,sha256=Br5G6Jn6aJeaAX9y3yBfNVRvq2-kIIz9LsjRCXLmzGg,6447
+cobweb/launchers/launcher.py,sha256=WTyv5bgKE_VwZXuWia_F26fH2zi6mps0wsnC16bks9k,5253
+cobweb/launchers/uploader.py,sha256=2zAYMEpfUYo8zCzH6Jhwkj9Y4_aJYVNnx0D5H8srKW4,1806
+cobweb/pipelines/__init__.py,sha256=zSUsGtx6smbs2iXBXvYynReKSgky-3gjqaAtKVnA_OU,105
+cobweb/pipelines/pipeline.py,sha256=qwoOYMhlAB-MnEmMNpNeauTHoRTOr2wyBDYS4MF6B1c,261
+cobweb/pipelines/pipeline_console.py,sha256=NEh-4zhuVAQOqwXLsqeb-rcNZ9_KXFUpL3otUTL5qBs,754
+cobweb/pipelines/pipeline_loghub.py,sha256=xZ6D55BGdiM71WUv83jyLGbEyUwhBHLJRZoXthBxxTs,1019
+cobweb/schedulers/__init__.py,sha256=LEya11fdAv0X28YzbQTeC1LQZ156Fj4cyEMGqQHUWW0,49
+cobweb/schedulers/launcher_air.py,sha256=qdcxq41I9zN5snEpMUUNEQNTtUiZM1Hw_3N9zu4PuAs,3058
+cobweb/schedulers/launcher_api.py,sha256=3-A6k3Igvi-xnvP9M_3NkJKUweDJ_pY10ZrHClteD-g,8628
+cobweb/schedulers/scheduler.py,sha256=mN9XvaOCzNnBWQfzslTIM860ZGq2gyLtxpbVUd0Slqs,2240
+cobweb/schedulers/scheduler_with_redis.py,sha256=pCimXqqpJ2xI1fs-6Ecy2vs9vHC-PPFX4IBMLDtpbho,6394
+cobweb/utils/__init__.py,sha256=8Bu5iZrIOUMS4jv4hi0inRPtscf6MK0ZFa7gQ7ZFoqw,145
+cobweb/utils/bloom.py,sha256=vng-YbKgh9HbtpAWYf_nkUSbfVTOj40aqUUejRYlsCU,1752
+cobweb/utils/decorators.py,sha256=066JCY_RNMr2mXkhEv8XTtOOKkv9CFiBm0ZNCcC-2ag,1131
+cobweb/utils/dotting.py,sha256=mVICaa26R-dQ4JGmPK-kkR6QjX38QiRewXZnGb2DCIc,1784
+cobweb/utils/oss.py,sha256=6x_ugXanh1R-6ZylQzUDQh4OeFZHujhWFCOxbzy53JY,3984
+cobweb/utils/tools.py,sha256=5JEaaAwYoV9Sdla2UBIJn6faUBuXmxUMagm9ck6FVqs,1253
+cobweb_launcher-3.1.0.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
+cobweb_launcher-3.1.0.dist-info/METADATA,sha256=Cl_g5S9zmKx6qhAPllDGYYpow_slpwwXYxJxtZoOzck,6509
+cobweb_launcher-3.1.0.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
+cobweb_launcher-3.1.0.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
+cobweb_launcher-3.1.0.dist-info/RECORD,,

cobweb/base/basic.py

@@ -1,295 +0,0 @@
-import json
-import random
-import time
-import hashlib
-import requests
-
-
-class Params:
-
-    def __init__(self, retry=None, priority=None, version=None, status=None):
-        self.retry = retry or 0
-        self.priority = priority or 300
-        self.version = version or int(time.time())
-        self.status = status
-
-
-class Seed:
-    __SEED_PARAMS__ = [
-        "retry",
-        "priority",
-        "version",
-        "status"
-    ]
-
-    def __init__(
-            self,
-            seed,
-            sid=None,
-            retry=None,
-            priority=None,
-            version=None,
-            status=None,
-            **kwargs
-    ):
-        if any(isinstance(seed, t) for t in (str, bytes)):
-            try:
-                item = json.loads(seed)
-                self._init_seed(item)
-            except json.JSONDecodeError:
-                self.__setattr__("url", seed)
-        elif isinstance(seed, dict):
-            self._init_seed(seed)
-        else:
-            raise TypeError(Exception(
-                f"seed type error, "
-                f"must be str or dict! "
-                f"seed: {seed}"
-            ))
-
-        seed_params = {
-            "retry": retry,
-            "priority": priority,
-            "version": version,
-            "status": status,
-        }
-
-        if kwargs:
-            # for k, v in kwargs.items():
-            #     if k in seed_params.keys():
-            #         seed_params[k] = v
-            #     else:
-            #         self.__setattr__(k, v)
-            self._init_seed(kwargs)
-            seed_params.update({
-                k: v for k, v in kwargs.items()
-                if k in self.__SEED_PARAMS__
-            })
-        if sid or not getattr(self, "sid", None):
-            self._init_id(sid)
-        self.params = Params(**seed_params)
-
-    def __getattr__(self, name):
-        return None
-
-    def __setitem__(self, key, value):
-        setattr(self, key, value)
-
-    def __getitem__(self, item):
-        return getattr(self, item)
-
-    def __str__(self):
-        return json.dumps(self.__dict__, ensure_ascii=False)
-
-    def __repr__(self):
-        chars = [f"{k}={v}" for k, v in self.__dict__.items()]
-        return f'{self.__class__.__name__}({", ".join(chars)})'
-
-    def _init_seed(self, seed_info: dict):
-        for k, v in seed_info.items():
-            if k not in self.__SEED_PARAMS__:
-                self.__setattr__(k, v)
-
-    def _init_id(self, sid):
-        if not sid:
-            sid = hashlib.md5(self.to_string.encode()).hexdigest()
-        self.__setattr__("sid", sid)
-
-    @property
-    def to_dict(self) -> dict:
-        seed = self.__dict__.copy()
-        if seed.get("params"):
-            del seed["params"]
-        return seed
-
-    @property
-    def to_string(self) -> str:
-        return json.dumps(
-            self.to_dict,
-            ensure_ascii=False,
-            separators=(",", ":")
-        )
-
-    @property
-    def seed(self):
-        return self.to_string
-
-
-class Request:
-    __SEED_PARAMS__ = [
-        "retry",
-        "priority",
-        "version",
-        "status"
-    ]
-
-    __REQUEST_ATTRS__ = {
-        "params",
-        "headers",
-        "cookies",
-        "data",
-        "json",
-        "files",
-        "auth",
-        "timeout",
-        "proxies",
-        "hooks",
-        "stream",
-        "verify",
-        "cert",
-        "allow_redirects",
-    }
-
-    def __init__(
-            self,
-            # url,
-            seed,
-            random_ua=True,
-            check_status_code=True,
-            retry=None,
-            priority=None,
-            version=None,
-            status=None,
-            **kwargs
-    ):
-        # self.url = url
-        self.check_status_code = check_status_code
-        self.request_setting = {}
-
-        seed_params = {
-            "retry": retry,
-            "priority": priority,
-            "version": version,
-            "status": status,
-        }
-
-        if isinstance(seed, Seed):
-            kwargs.update(**seed.to_dict)
-        elif isinstance(seed, str):
-            kwargs.update(**json.loads(seed))
-
-        for k, v in kwargs.items():
-            if k in self.__class__.__REQUEST_ATTRS__:
-                self.request_setting[k] = v
-                continue
-            elif k in self.__SEED_PARAMS__:
-                seed_params[k] = v
-            self.__setattr__(k, v)
-
-        if not getattr(self, "method", None):
-            self.method = "POST" if self.request_setting.get("data") or self.request_setting.get("json") else "GET"
-
-        if random_ua:
-            self._build_header()
-
-        self.params = Params(**seed_params)
-        # self.seed = self.to_string
-
-    @property
-    def _random_ua(self) -> str:
-        v1 = random.randint(4, 15)
-        v2 = random.randint(3, 11)
-        v3 = random.randint(1, 16)
-        v4 = random.randint(533, 605)
-        v5 = random.randint(1000, 6000)
-        v6 = random.randint(10, 80)
-        user_agent = (f"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_{v1}_{v2}) AppleWebKit/{v4}.{v3} "
-                      f"(KHTML, like Gecko) Chrome/105.0.0.0 Safari/{v4}.{v3} Edg/105.0.{v5}.{v6}")
-        return user_agent
-
-    def _build_header(self) -> dict:
-        if not self.request_setting.get("headers"):
-            self.request_setting["headers"] = {"accept": "*/*", "user-agent": self._random_ua}
-        elif "user-agent" not in [key.lower() for key in self.request_setting["headers"].keys()]:
-            self.request_setting["headers"]["user-agent"] = self._random_ua
-
-    def download(self) -> requests.Response:
-        response = requests.request(self.method, self.url, **self.request_setting)
-        if self.check_status_code:
-            response.raise_for_status()
-        return response
-
-    def __getattr__(self, name):
-        return None
-
-    def __setitem__(self, key, value):
-        setattr(self, key, value)
-
-    def __getitem__(self, item):
-        return getattr(self, item)
-
-    @property
-    def to_dict(self):
-        _dict = self.__dict__.copy()
-        # _dict.pop('seed')
-        _dict.pop('params')
-        _dict.pop('check_status_code')
-        # _dict.pop('request_setting')
-        return _dict
-
-    @property
-    def to_string(self) -> str:
-        return json.dumps(
-            self.to_dict,
-            ensure_ascii=False,
-            separators=(",", ":")
-        )
-
-    @property
-    def seed(self):
-        return self.to_string
-
-
-class Response:
-
-    def __init__(
-            self,
-            seed,
-            response,
-            retry=None,
-            priority=None,
-            version=None,
-            status=None,
-            **kwargs
-    ):
-        self.seed = seed
-        self.response = response
-        seed_params = {
-            "retry": retry,
-            "priority": priority,
-            "version": version,
-            "status": status,
-        }
-        for k, v in kwargs.items():
-            if k in seed_params.keys():
-                seed_params[k] = v
-            else:
-                self.__setattr__(k, v)
-        self.params = Params(**seed_params)
-
-    @property
-    def to_dict(self):
-        _dict = self.__dict__.copy()
-        _dict.pop('seed')
-        _dict.pop('response')
-        _dict.pop('method')
-        _dict.pop('params')
-        _dict.pop('request_setting')
-        return _dict
-
-    @property
-    def to_string(self) -> str:
-        return json.dumps(
-            self.to_dict,
-            ensure_ascii=False,
-            separators=(",", ":")
-        )
-
-    def __getattr__(self, name):
-        return None
-
-    def __setitem__(self, key, value):
-        setattr(self, key, value)
-
-    def __getitem__(self, item):
-        return getattr(self, item)

cobweb/base/dotting.py

@@ -1,35 +0,0 @@
-import os
-import json
-from aliyun.log import LogClient, LogItem, PutLogsRequest
-
-
-class LoghubDot:
-
-    def __init__(self):
-        endpoint = os.getenv("DOTTING_ENDPOINT", "")
-        accessKeyId = os.getenv("DOTTING_ACCESS_KEY", "")
-        accessKey = os.getenv("DOTTING_SECRET_KEY", "")
-        self.client = LogClient(endpoint=endpoint, accessKeyId=accessKeyId, accessKey=accessKey) \
-            if endpoint and accessKeyId and accessKey else None
-
-    def build(self, topic, **kwargs):
-        if self.client:
-            temp = {}
-            log_items = []
-            log_item = LogItem()
-            for key, value in kwargs.items():
-                if not isinstance(value, str):
-                    temp[key] = json.dumps(value, ensure_ascii=False)
-                else:
-                    temp[key] = value
-            contents = sorted(temp.items())
-            log_item.set_contents(contents)
-            log_items.append(log_item)
-            request = PutLogsRequest(
-                project="databee-download-log",
-                logstore="download-logging",
-                topic=topic,
-                logitems=log_items,
-                compress=True
-            )
-            self.client.put_logs(request=request)

cobweb/launchers/launcher_air.py

@@ -1,88 +0,0 @@
-# import time
-#
-# from cobweb.base import logger
-# from cobweb.constant import LogTemplate
-# from .launcher import Launcher, check_pause
-#
-#
-# class LauncherAir(Launcher):
-#
-#     # def _scheduler(self):
-#     #     if self.start_seeds:
-#     #         self.__LAUNCHER_QUEUE__['todo'].push(self.start_seeds)
-#
-#     @check_pause
-#     def _insert(self):
-#         seeds = {}
-#         status = self.__LAUNCHER_QUEUE__['new'].length < self._new_queue_max_size
-#         for _ in range(self._new_queue_max_size):
-#             seed = self.__LAUNCHER_QUEUE__['new'].pop()
-#             if not seed:
-#                 break
-#             seeds[seed.to_string] = seed.params.priority
-#         if seeds:
-#             self.__LAUNCHER_QUEUE__['todo'].push(seeds)
-#         if status:
-#             time.sleep(self._new_queue_wait_seconds)
-#
-#     @check_pause
-#     def _delete(self):
-#         seeds = []
-#         status = self.__LAUNCHER_QUEUE__['done'].length < self._done_queue_max_size
-#
-#         for _ in range(self._done_queue_max_size):
-#             seed = self.__LAUNCHER_QUEUE__['done'].pop()
-#             if not seed:
-#                 break
-#             seeds.append(seed.to_string)
-#
-#         if seeds:
-#             self._remove_doing_seeds(seeds)
-#
-#         if status:
-#             time.sleep(self._done_queue_wait_seconds)
-#
-#     def _polling(self):
-#
-#         check_emtpy_times = 0
-#
-#         while not self._stop.is_set():
-#
-#             queue_not_empty_count = 0
-#             pooling_wait_seconds = 30
-#
-#             for q in self.__LAUNCHER_QUEUE__.values():
-#                 if q.length != 0:
-#                     queue_not_empty_count += 1
-#
-#             if queue_not_empty_count == 0:
-#                 pooling_wait_seconds = 3
-#                 if self._pause.is_set():
-#                     check_emtpy_times = 0
-#                     if not self._task_model:
-#                         logger.info("Done! Ready to close thread...")
-#                         self._stop.set()
-#                 elif check_emtpy_times > 2:
-#                     self.__DOING__ = {}
-#                     self._pause.set()
-#                 else:
-#                     logger.info(
-#                         "check whether the task is complete, "
-#                         f"reset times {3 - check_emtpy_times}"
-#                     )
-#                     check_emtpy_times += 1
-#             elif self._pause.is_set():
-#                 self._pause.clear()
-#                 self._execute()
-#             else:
-#                 logger.info(LogTemplate.launcher_air_polling.format(
-#                     task=self.task,
-#                     doing_len=len(self.__DOING__.keys()),
-#                     todo_len=self.__LAUNCHER_QUEUE__['todo'].length,
-#                     done_len=self.__LAUNCHER_QUEUE__['done'].length,
-#                     upload_len=self.__LAUNCHER_QUEUE__['upload'].length,
-#                 ))
-#
-#             time.sleep(pooling_wait_seconds)
-#
-#

cobweb/launchers/launcher_api.py

@@ -1,88 +0,0 @@
-import time
-
-from cobweb.base import TaskQueue, Decorators, Seed, Request
-from cobweb.schedulers import ApiScheduler
-from .launcher import Launcher
-
-
-class LauncherApi(Launcher):
-
-    def __init__(self, task, project, custom_setting=None, **kwargs):
-        super().__init__(task, project, custom_setting, **kwargs)
-        self._redis_download = "{%s:%s}:download" % (project, task)
-        self._redis_todo = "{%s:%s}:todo" % (project, task)
-        self._scheduler = ApiScheduler(task, project)
-
-    @Decorators.stop
-    def _schedule(self):
-        thread_sleep = self.scheduling_wait_time
-        for q, key, size, item_info, Cls in [
-            (TaskQueue.TODO, self._redis_todo, self.todo_queue_size, self._task_info["todo"], Seed),
-            (TaskQueue.DOWNLOAD, self._redis_download, self.download_queue_size, self._task_info["download"], Request),
-        ]:
-            if q.length < size:
-                for member, priority in self._scheduler.schedule(key, self.scheduling_size):
-                    q.push(Cls(member, priority=priority))
-                    self.add_working_item(key.split(":")[-1], member, priority)
-                thread_sleep = 0.1
-        time.sleep(thread_sleep)
-
-    @Decorators.stop
-    def _heartbeat(self):
-        if self._scheduler.working.is_set():
-            self._scheduler.set_heartbeat()
-        time.sleep(3)
-
-    @Decorators.stop
-    def _reset(self):
-        self._scheduler.reset(
-            keys=[self._redis_todo, self._redis_download],
-            reset_time=self.seed_reset_seconds
-        )
-        time.sleep(30)
-
-    @Decorators.pause
-    def _insert(self):
-        thread_sleep = 0.1
-        for q, key, size in [
-            (TaskQueue.SEED, self._redis_todo, self.seed_queue_size),
-            (TaskQueue.REQUEST, self._redis_download, self.request_queue_size),
-        ]:
-            item_info = {}
-            while (item := q.pop()) and len(item_info.keys()) < self.inserting_size:
-                item_info[item.seed] = item.params.priority
-            if q.length >= size:
-                thread_sleep = self.inserting_wait_time
-            self._scheduler.insert(key, item_info)
-        time.sleep(thread_sleep)
-
-    @Decorators.pause
-    def _refresh(self):
-        self._scheduler.refresh(self._redis_todo, self._task_info["todo"])
-        self._scheduler.refresh(self._redis_download, self._task_info["download"])
-        time.sleep(10)
-
-    @Decorators.pause
-    def _remove(self):
-        thread_sleep = self.removing_wait_time
-        for q, key, size in [
-            (TaskQueue.DELETE, self._redis_todo, self.delete_queue_size),
-            (TaskQueue.DONE, self._redis_download, self.done_queue_size),
-        ]:
-            items = []
-            while (item := q.pop()) and len(items) < self.removing_size:
-                items.append(item)
-            self._scheduler.delete(key, items)
-            self.remove_working_items(key.split(":")[-1], items)
-            if q.length >= size:
-                thread_sleep = 0.1
-        time.sleep(thread_sleep)
-
-    def _init_schedule_thread(self):
-        self._add_thread(func=self._heartbeat)
-        self._add_thread(func=self._reset)
-        self._add_thread(func=self._refresh)
-        self._add_thread(func=self._schedule)
-        self._add_thread(func=self._insert)
-        self._add_thread(func=self._remove)
-        # self._add_thread(func=self._polling)