cobweb-launcher 1.3.6__py3-none-any.whl → 1.3.8__py3-none-any.whl

cobweb/base/__init__.py

@@ -1,3 +1,4 @@
+import os
 import time
 import traceback
 import threading
@@ -12,7 +13,7 @@ from .basic import Seed, Request, Response
 from .item import BaseItem, ConsoleItem
 # from .seed import Seed
 from .log import logger
-# from .dotting import LoghubDot
+from .dotting import LoghubDot
 
 
 class TaskQueue:
@@ -25,8 +26,7 @@ class TaskQueue:
     DONE = Queue()          # 下载完成队列
     UPLOAD = Queue()        # 任务上传队列
     DELETE = Queue()        # 任务删除队列
-
-    # DOT = LoghubDot()
+    DOT = LoghubDot()
 
     @staticmethod
     def is_empty():
@@ -43,6 +43,7 @@ class TaskQueue:
     @staticmethod
     def process_task(it: Union[Seed, Request, Response, BaseItem], crawler_func: Callable):
         try:
+            start_time = time.time()
             iterators = crawler_func(it)
             if not isgenerator(iterators):
                 raise TypeError(f"{crawler_func.__name__} function isn't a generator")
@@ -57,12 +58,11 @@ class TaskQueue:
                     TaskQueue.SEED.push(tk)
                 else:
                     raise TypeError(f"{crawler_func.__name__} function return type isn't supported")
-                # TaskQueue.DOT.build(
-                #     topic=f"{self.project}:{self.task}",
-                #     cost_time=end_time - start_time,
-                #     **download_item.to_dict
-                # )
-                # todo: 数据打点
+                TaskQueue.DOT.build(
+                    topic=f"{os.getenv('PROJECT')}:{os.getenv('TASK')}",
+                    cost_time=round(time.time() - start_time, 2),
+                    **tk.to_dict
+                )
         except Exception as e:
             it.params.retry += 1
             if isinstance(it, Request):

cobweb/base/dotting.py

@@ -27,7 +27,7 @@ class LoghubDot:
             log_items.append(log_item)
             request = PutLogsRequest(
                 project="databee-download-log",
-                logstore="cobweb_log",
+                logstore="download-logging",
                 topic=topic,
                 logitems=log_items,
                 compress=True

cobweb/utils/oss.py

@@ -1,9 +1,9 @@
-from typing import List
+# from typing import List
 from cobweb import setting
 from requests import Response
 from oss2 import Auth, Bucket, models, PartIterator
 from cobweb.exceptions import oss_db_exception
-from cobweb.base.decorators import decorator_oss_db
+from cobweb.base import Decorators
 
 
 class OssUtil:
@@ -40,17 +40,17 @@ class OssUtil:
     def head(self, key: str) -> models.HeadObjectResult:
         return self._client.head_object(key)
 
-    @decorator_oss_db(exception=oss_db_exception.OssDBInitPartError)
+    @Decorators.decorator_oss_db(exception=oss_db_exception.OssDBInitPartError)
     def init_part(self, key) -> models.InitMultipartUploadResult:
         """初始化分片上传"""
         return self._client.init_multipart_upload(key)
 
-    @decorator_oss_db(exception=oss_db_exception.OssDBPutObjError)
+    @Decorators.decorator_oss_db(exception=oss_db_exception.OssDBPutObjError)
     def put(self, key, data) -> models.PutObjectResult:
         """文件上传"""
         return self._client.put_object(key, data)
 
-    @decorator_oss_db(exception=oss_db_exception.OssDBPutPartError)
+    @Decorators.decorator_oss_db(exception=oss_db_exception.OssDBPutPartError)
     def put_part(self, key, upload_id, position, data) -> models.PutObjectResult:
         """分片上传"""
         return self._client.upload_part(key, upload_id, position, data)
@@ -59,13 +59,13 @@ class OssUtil:
         """获取分片列表"""
         return [part_info for part_info in PartIterator(self._client, key, upload_id)]
 
-    @decorator_oss_db(exception=oss_db_exception.OssDBMergeError)
+    @Decorators.decorator_oss_db(exception=oss_db_exception.OssDBMergeError)
     def merge(self, key, upload_id, parts=None) -> models.PutObjectResult:
         """合并分片"""
         headers = None if parts else {"x-oss-complete-all": "yes"}
         return self._client.complete_multipart_upload(key, upload_id, parts, headers=headers)
 
-    @decorator_oss_db(exception=oss_db_exception.OssDBAppendObjError)
+    @Decorators.decorator_oss_db(exception=oss_db_exception.OssDBAppendObjError)
     def append(self, key, position, data) -> models.AppendObjectResult:
         """追加上传"""
         return self._client.append_object(key, position, data)

{cobweb_launcher-1.3.6.dist-info → cobweb_launcher-1.3.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.3.6
+Version: 1.3.8
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

cobweb_launcher-1.3.8.dist-info/RECORD

@@ -0,0 +1,40 @@
+cobweb/__init__.py,sha256=oaEfsGUuGP0s39UbFRwrnsjMUeuB6QvQIAwStKFyUTk,83
+cobweb/constant.py,sha256=eofONAntk9O6S-cb4KbYGYHL_u7nBlOqqFOw_HzJHAU,3588
+cobweb/setting.py,sha256=pY6LKsgWI3164GiGA1z_y26LVf5-3mpiEgmm86mKRdY,3135
+cobweb/base/__init__.py,sha256=HHxaDy0x6HKOq5BPhl8eNgqPx06-XYWe-WrM0GQZDDY,5108
+cobweb/base/basic.py,sha256=s5G4LBZiLUfoymV-gLSIqeH-OJ7q7-L35sBa6xEH3EI,7666
+cobweb/base/common_queue.py,sha256=Gor7sR3h1hlZWaI0XcNAbf0S15Ftjr3DFRWNTGL13uU,1137
+cobweb/base/dotting.py,sha256=lfFXXqnVP__hxlW3qH5Bnuq69KtnFaQLbcz1M8e2Ajg,1239
+cobweb/base/item.py,sha256=hYheVTV2Bozp4iciJpE2ZwBIXkaqBg4QQkRccP8yoVk,1049
+cobweb/base/log.py,sha256=L01hXdk3L2qEm9X1FOXQ9VmWIoHSELe0cyZvrdAN61A,2003
+cobweb/base/request.py,sha256=acGm3OzxsPed5VUTk7D9eeHZPMh7KUNQRUv44G5znZg,2659
+cobweb/base/response.py,sha256=eB1DWMXFCpn3cJ3yzgCRU1WeZAdayGDohRgdjdMUFN4,406
+cobweb/base/seed.py,sha256=PN5J4gKPEXylwyQeSGOBfauxHktxFr7RJe8nVX1hBw4,2987
+cobweb/crawlers/__init__.py,sha256=msvkB9mTpsgyj8JfNMsmwAcpy5kWk_2NrO1Adw2Hkw0,29
+cobweb/crawlers/crawler.py,sha256=ZQ6yVA1EaQRdKJEY3DNqShzp9HPMwlSXapnsRW9E5Wc,2987
+cobweb/db/__init__.py,sha256=uZwSkd105EAwYo95oZQXAfofUKHVIAZZIPpNMy-hm2Q,56
+cobweb/db/api_db.py,sha256=bDc5dJQxq4z04h70KUTHd0OqUOEY7Cm3wcNJZtTvJIM,3015
+cobweb/db/redis_db.py,sha256=FvMzckJtmhwKhZqKoS23iXmJti5P2dnMVD5rJ__5LUw,5139
+cobweb/exceptions/__init__.py,sha256=E9SHnJBbhD7fOgPFMswqyOf8SKRDrI_i25L0bSpohvk,32
+cobweb/exceptions/oss_db_exception.py,sha256=iP_AImjNHT3-Iv49zCFQ3rdLnlvuHa3h2BXApgrOYpA,636
+cobweb/launchers/__init__.py,sha256=m_XNG2bWuMbirPt3d0_s-Ezl1xycfUxeqZnwq_kkfuo,116
+cobweb/launchers/launcher.py,sha256=NFwpc_0Um0hbDm1A8glWA4fcW6mNYL1eon4t3JAQUlw,7411
+cobweb/launchers/launcher_air.py,sha256=yPr395HVIIHAq6lqRcYJu7c0KkfO9V8O-2sn0hC96p0,2990
+cobweb/launchers/launcher_api.py,sha256=TfLrLXazFWsOJLI7caMGfZozCttL1WTwTo3uUpN_FV0,3370
+cobweb/launchers/launcher_pro.py,sha256=2H-TcvQx-ga78GLNTa-GXMLYAj9nEeCJSWf8xl-1ISQ,3374
+cobweb/pipelines/__init__.py,sha256=zSUsGtx6smbs2iXBXvYynReKSgky-3gjqaAtKVnA_OU,105
+cobweb/pipelines/pipeline.py,sha256=Pycm22bHId9a3gdP81D5y7SsuMndYooTb5n4zQxP7dM,1321
+cobweb/pipelines/pipeline_console.py,sha256=NEh-4zhuVAQOqwXLsqeb-rcNZ9_KXFUpL3otUTL5qBs,754
+cobweb/pipelines/pipeline_loghub.py,sha256=xZ6D55BGdiM71WUv83jyLGbEyUwhBHLJRZoXthBxxTs,1019
+cobweb/schedulers/__init__.py,sha256=y7Lv_7b0zfTl0OhIONb_8u1K1C9gVlBA-xz_XG_kI9g,85
+cobweb/schedulers/scheduler_api.py,sha256=mC54QOS0PEu4SFvxfD5Qr9239hAxwMrKTg-33rirANE,2112
+cobweb/schedulers/scheduler_redis.py,sha256=Aw7de0sXigRAxJgqUhHWu30hMBzgEWjkj-3OXXqmldg,2118
+cobweb/utils/__init__.py,sha256=YvD4mIDBd9jmGA6WJBcwkgDU2jRFNBCEbarZCSUBAHE,114
+cobweb/utils/bloom.py,sha256=vng-YbKgh9HbtpAWYf_nkUSbfVTOj40aqUUejRYlsCU,1752
+cobweb/utils/oss.py,sha256=6Qlhdde7CcwD69bBe2rGWHY3-aptG9NXB_DZLhjgDRQ,3553
+cobweb/utils/tools.py,sha256=5JEaaAwYoV9Sdla2UBIJn6faUBuXmxUMagm9ck6FVqs,1253
+cobweb_launcher-1.3.8.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
+cobweb_launcher-1.3.8.dist-info/METADATA,sha256=vxO6miknf7w1rom7AF3FubCwBiiYCHgS5xXVGKNdQdk,6509
+cobweb_launcher-1.3.8.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
+cobweb_launcher-1.3.8.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
+cobweb_launcher-1.3.8.dist-info/RECORD,,

cobweb/base/decorators.py

@@ -1,40 +0,0 @@
-from functools import wraps
-
-
-# def check_redis_status(func):
-#     @wraps(func)
-#     def wrapper(*args, **kwargs):
-#         try:
-#             result = func(*args, **kwargs)
-#         except Exception:
-#             result = False
-#         return result
-#
-#     return wrapper
-
-
-def decorator_oss_db(exception, retries=3):
-    def decorator(func):
-        @wraps(func)
-        def wrapper(callback_func, *args, **kwargs):
-            result = None
-            for i in range(retries):
-                msg = None
-                try:
-                    return func(callback_func, *args, **kwargs)
-                except Exception as e:
-                    result = None
-                    msg = e
-                finally:
-                    if result:
-                        return result
-
-                    if i >= 2 and msg:
-                        raise exception(msg)
-
-        return wrapper
-
-    return decorator
-
-
-

cobweb/crawlers/base_crawler.py

@@ -1,144 +0,0 @@
-import threading
-import time
-import traceback
-
-from inspect import isgenerator
-from typing import Union, Callable, Mapping
-
-from cobweb.base import Queue, Seed, BaseItem, Request, Response, logger
-from cobweb.constant import DealModel, LogTemplate
-from cobweb.utils import download_log_info
-from cobweb import setting
-
-
-class Crawler(threading.Thread):
-
-    def __init__(
-            self,
-            upload_queue: Queue,
-            custom_func: Union[Mapping[str, Callable]],
-            launcher_queue: Union[Mapping[str, Queue]],
-    ):
-        super().__init__()
-
-        self.upload_queue = upload_queue
-        for func_name, _callable in custom_func.items():
-            if isinstance(_callable, Callable):
-                self.__setattr__(func_name, _callable)
-
-        self.launcher_queue = launcher_queue
-
-        self.spider_thread_num = setting.SPIDER_THREAD_NUM
-        self.max_retries = setting.SPIDER_MAX_RETRIES
-
-    @staticmethod
-    def request(seed: Seed) -> Union[Request, BaseItem]:
-        stream = True if setting.DOWNLOAD_MODEL else False
-        yield Request(seed.url, seed, stream=stream, timeout=5)
-
-    @staticmethod
-    def download(item: Request) -> Union[Seed, BaseItem, Response, str]:
-        response = item.download()
-        yield Response(item.seed, response, **item.to_dict)
-
-    @staticmethod
-    def parse(item: Response) -> BaseItem:
-        pass
-
-    def get_seed(self) -> Seed:
-        return self.launcher_queue['todo'].pop()
-
-    def distribute(self, item, seed):
-        if isinstance(item, BaseItem):
-            self.upload_queue.push(item)
-        elif isinstance(item, Seed):
-            self.launcher_queue['new'].push(item)
-        elif isinstance(item, str) and item == DealModel.poll:
-            self.launcher_queue['todo'].push(seed)
-        elif isinstance(item, str) and item == DealModel.done:
-            self.launcher_queue['done'].push(seed)
-        elif isinstance(item, str) and item == DealModel.fail:
-            seed.params.seed_status = DealModel.fail
-            self.launcher_queue['done'].push(seed)
-        else:
-            raise TypeError("yield value type error!")
-
-    def spider(self):
-        while True:
-            seed = self.get_seed()
-
-            if not seed:
-                continue
-
-            elif seed.params.retry >= self.max_retries:
-                seed.params.seed_status = DealModel.fail
-                self.launcher_queue['done'].push(seed)
-                continue
-
-            seed_detail_log_info = download_log_info(seed.to_dict)
-
-            try:
-                request_iterators = self.request(seed)
-
-                if not isgenerator(request_iterators):
-                    raise TypeError("request function isn't a generator!")
-
-                iterator_status = False
-
-                for request_item in request_iterators:
-
-                    iterator_status = True
-
-                    if isinstance(request_item, Request):
-                        iterator_status = False
-                        download_iterators = self.download(request_item)
-                        if not isgenerator(download_iterators):
-                            raise TypeError("download function isn't a generator")
-
-                        for download_item in download_iterators:
-                            iterator_status = True
-                            if isinstance(download_item, Response):
-                                iterator_status = False
-                                logger.info(LogTemplate.download_info.format(
-                                    detail=seed_detail_log_info,
-                                    retry=seed.params.retry,
-                                    priority=seed.params.priority,
-                                    seed_version=seed.params.seed_version,
-                                    identifier=seed.identifier or "",
-                                    status=download_item.response,
-                                    response=download_log_info(download_item.to_dict)
-                                ))
-                                parse_iterators = self.parse(download_item)
-                                if not isgenerator(parse_iterators):
-                                    raise TypeError("parse function isn't a generator")
-                                for parse_item in parse_iterators:
-                                    iterator_status = True
-                                    if isinstance(parse_item, Response):
-                                        raise TypeError("upload_item can't be a Response instance")
-                                    self.distribute(parse_item, seed)
-                            else:
-                                self.distribute(download_item, seed)
-                    else:
-                        self.distribute(request_item, seed)
-
-                if not iterator_status:
-                    raise ValueError("request/download/parse function yield value error!")
-
-            except Exception as e:
-                logger.info(LogTemplate.download_exception.format(
-                    detail=seed_detail_log_info,
-                    retry=seed.params.retry,
-                    priority=seed.params.priority,
-                    seed_version=seed.params.seed_version,
-                    identifier=seed.identifier or "",
-                    exception=''.join(traceback.format_exception(type(e), e, e.__traceback__))
-                ))
-                seed.params.retry += 1
-                self.launcher_queue['todo'].push(seed)
-            finally:
-                time.sleep(0.1)
-
-    def run(self):
-        for index in range(self.spider_thread_num):
-            threading.Thread(name=f"spider_{index}", target=self.spider).start()
-

cobweb/crawlers/file_crawler.py

@@ -1,98 +0,0 @@
-import os
-from typing import Union
-from cobweb import setting
-from cobweb.utils import OssUtil
-from cobweb.crawlers import Crawler
-from cobweb.base import Seed, BaseItem, Request, Response
-from cobweb.exceptions import OssDBPutPartError, OssDBMergeError
-
-
-oss_util = OssUtil(is_path_style=bool(int(os.getenv("PRIVATE_LINK", 0))))
-
-
-class FileCrawlerAir(Crawler):
-
-    @staticmethod
-    def download(item: Request) -> Union[Seed, BaseItem, Response, str]:
-        seed_dict = item.seed.to_dict
-        seed_dict["bucket_name"] = oss_util.bucket
-        try:
-            seed_dict["oss_path"] = key = item.seed.oss_path or getattr(item, "oss_path")
-
-            if oss_util.exists(key):
-                seed_dict["data_size"] = oss_util.head(key).content_length
-                yield Response(item.seed, "exists", **seed_dict)
-
-            else:
-                seed_dict.setdefault("end", "")
-                seed_dict.setdefault("start", 0)
-
-                if seed_dict["end"] or seed_dict["start"]:
-                    start, end = seed_dict["start"], seed_dict["end"]
-                    item.request_setting["headers"]['Range'] = f'bytes={start}-{end}'
-
-                if not item.seed.identifier:
-                    content = b""
-                    chunk_size = oss_util.chunk_size
-                    min_upload_size = oss_util.min_upload_size
-                    seed_dict.setdefault("position", 1)
-
-                    response = item.download()
-
-                    content_type = response.headers.get("content-type", "").split(";")[0]
-                    seed_dict["data_size"] = content_length = int(response.headers.get("content-length", 0))
-
-                    if content_type and content_type in setting.FILE_FILTER_CONTENT_TYPE:
-                        """过滤响应文件类型"""
-                        response.close()
-                        seed_dict["filter"] = True
-                        seed_dict["msg"] = f"response content type is {content_type}"
-                        yield Response(item.seed, response, **seed_dict)
-
-                    elif seed_dict['position'] == 1 and min_upload_size >= content_length > 0:
-                        """过小文件标识返回"""
-                        response.close()
-                        seed_dict["filter"] = True
-                        seed_dict["msg"] = "file size is too small"
-                        yield Response(item.seed, response, **seed_dict)
-
-                    elif seed_dict['position'] == 1 and chunk_size > content_length > min_upload_size:
-                        """小文件直接下载"""
-                        for part_data in response.iter_content(chunk_size):
-                            content += part_data
-                        response.close()
-                        oss_util.put(key, content)
-                        yield Response(item.seed, response, **seed_dict)
-
-                    else:
-                        """中大文件同步分片下载"""
-                        seed_dict.setdefault("upload_id", oss_util.init_part(key).upload_id)
-
-                        for part_data in response.iter_content(chunk_size):
-                            content += part_data
-                            if len(content) >= chunk_size:
-                                upload_data = content[:chunk_size]
-                                content = content[chunk_size:]
-                                oss_util.put_part(key, seed_dict["upload_id"], seed_dict['position'], content)
-                                seed_dict['start'] += len(upload_data)
-                                seed_dict['position'] += 1
-
-                        response.close()
-
-                        if content:
-                            oss_util.put_part(key, seed_dict["upload_id"], seed_dict['position'], content)
-                        oss_util.merge(key, seed_dict["upload_id"])
-                        seed_dict["data_size"] = oss_util.head(key).content_length
-                        yield Response(item.seed, response, **seed_dict)
-
-                elif item.seed.identifier == "merge":
-                    oss_util.merge(key, seed_dict["upload_id"])
-                    seed_dict["data_size"] = oss_util.head(key).content_length
-                    yield Response(item.seed, "merge", **seed_dict)
-
-        except OssDBPutPartError:
-            yield Seed(seed_dict)
-        except OssDBMergeError:
-            yield Seed(seed_dict, identifier="merge")
-
-

cobweb/pipelines/base_pipeline.py

@@ -1,54 +0,0 @@
-import time
-import threading
-
-from abc import ABC, abstractmethod
-from cobweb.base import BaseItem, Queue, logger
-
-
-class Pipeline(threading.Thread, ABC):
-
-    def __init__(
-            self,
-            done_queue: Queue,
-            upload_queue: Queue,
-            upload_queue_size: int,
-            upload_wait_seconds: int
-    ):
-        super().__init__()
-        self.done_queue = done_queue
-        self.upload_queue = upload_queue
-        self.upload_queue_size = upload_queue_size
-        self.upload_wait_seconds = upload_wait_seconds
-
-    @abstractmethod
-    def build(self, item: BaseItem) -> dict:
-        pass
-
-    @abstractmethod
-    def upload(self, table: str, data: list) -> bool:
-        pass
-
-    def run(self):
-        while True:
-            status = self.upload_queue.length < self.upload_queue_size
-            if status:
-                time.sleep(self.upload_wait_seconds)
-            data_info, seeds = {}, []
-            for _ in range(self.upload_queue_size):
-                item = self.upload_queue.pop()
-                if not item:
-                    break
-                data = self.build(item)
-                seeds.append(item.seed)
-                data_info.setdefault(item.table, []).append(data)
-            for table, datas in data_info.items():
-                try:
-                    self.upload(table, datas)
-                    status = True
-                except Exception as e:
-                    logger.info(e)
-                    status = False
-                if status:
-                    self.done_queue.push(seeds)
-
-

cobweb/pipelines/loghub_pipeline.py

@@ -1,34 +0,0 @@
-import json
-
-from cobweb import setting
-from cobweb.base import BaseItem
-from cobweb.pipelines import Pipeline
-from aliyun.log import LogClient, LogItem, PutLogsRequest
-
-
-class LoghubPipeline(Pipeline):
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.client = LogClient(**setting.LOGHUB_CONFIG)
-
-    def build(self, item: BaseItem):
-        log_item = LogItem()
-        temp = item.to_dict
-        for key, value in temp.items():
-            if not isinstance(value, str):
-                temp[key] = json.dumps(value, ensure_ascii=False)
-        contents = sorted(temp.items())
-        log_item.set_contents(contents)
-        return log_item
-
-    def upload(self, table, datas):
-        request = PutLogsRequest(
-            project=setting.LOGHUB_PROJECT,
-            logstore=table,
-            topic=setting.LOGHUB_TOPIC,
-            source=setting.LOGHUB_SOURCE,
-            logitems=datas,
-            compress=True
-        )
-        self.client.put_logs(request=request)

cobweb/utils/dotting.py

@@ -1,32 +0,0 @@
-import json
-
-from aliyun.log import LogClient, LogItem, PutLogsRequest
-from cobweb import setting
-
-
-class LoghubDot:
-
-    def __init__(self):
-        self.client = LogClient(**setting.LOGHUB_CONFIG)
-
-    def build(self, topic, **kwargs):
-
-        temp = {}
-        log_items = []
-        log_item = LogItem()
-        for key, value in kwargs.items():
-            if not isinstance(value, str):
-                temp[key] = json.dumps(value, ensure_ascii=False)
-            else:
-                temp[key] = value
-        contents = sorted(temp.items())
-        log_item.set_contents(contents)
-        log_items.append(log_item)
-        request = PutLogsRequest(
-            project="databee-download-log",
-            logstore="cobweb_log",
-            topic=topic,
-            logitems=log_items,
-            compress=True
-        )
-        self.client.put_logs(request=request)

cobweb_/__init__.py

@@ -1,2 +0,0 @@
-from .launchers import LauncherAir, LauncherPro, LauncherApi
-from .constant import CrawlerModel

cobweb_/base/__init__.py

@@ -1,9 +0,0 @@
-from .common_queue import Queue
-from .response import Response
-from .request import Request
-from .item import BaseItem, ConsoleItem
-from .seed import Seed
-
-from .log import logger
-from .decorators import decorator_oss_db
-

cobweb_/base/common_queue.py

@@ -1,30 +0,0 @@
-from collections import deque
-
-
-class Queue:
-
-    def __init__(self):
-        self._queue = deque()
-
-    @property
-    def length(self) -> int:
-        return len(self._queue)
-
-    def push(self, data, left: bool = False, direct_insertion: bool = False):
-        try:
-            if not data:
-                return None
-            if not direct_insertion and any(isinstance(data, t) for t in (list, tuple)):
-                self._queue.extendleft(data) if left else self._queue.extend(data)
-            else:
-                self._queue.appendleft(data) if left else self._queue.append(data)
-        except AttributeError:
-            pass
-
-    def pop(self, left: bool = True):
-        try:
-            return self._queue.popleft() if left else self._queue.pop()
-        except IndexError:
-            return None
-        except AttributeError:
-            return None

cobweb_/base/decorators.py

@@ -1,40 +0,0 @@
-from functools import wraps
-
-
-# def check_redis_status(func):
-#     @wraps(func)
-#     def wrapper(*args, **kwargs):
-#         try:
-#             result = func(*args, **kwargs)
-#         except Exception:
-#             result = False
-#         return result
-#
-#     return wrapper
-
-
-def decorator_oss_db(exception, retries=3):
-    def decorator(func):
-        @wraps(func)
-        def wrapper(callback_func, *args, **kwargs):
-            result = None
-            for i in range(retries):
-                msg = None
-                try:
-                    return func(callback_func, *args, **kwargs)
-                except Exception as e:
-                    result = None
-                    msg = e
-                finally:
-                    if result:
-                        return result
-
-                    if i >= 2 and msg:
-                        raise exception(msg)
-
-        return wrapper
-
-    return decorator
-
-
-