cobweb-launcher 1.3.6__py3-none-any.whl → 1.3.8__py3-none-any.whl

cobweb_/utils/bloom.py

@@ -1,58 +0,0 @@
-import math
-import time
-
-import mmh3
-import redis
-from cobweb import setting
-
-
-class BloomFilter:
-
-    def __init__(self, key, redis_config=None, capacity=None, error_rate=None):
-        redis_config = redis_config or setting.REDIS_CONFIG
-        capacity = capacity or setting.CAPACITY
-        error_rate = error_rate or setting.ERROR_RATE
-        redis_config['db'] = 3
-
-        self.key = key
-
-        pool = redis.ConnectionPool(**redis_config)
-        self._client = redis.Redis(connection_pool=pool)
-        self.bit_size = self.get_bit_size(capacity, error_rate)
-        self.hash_count = self.get_hash_count(self.bit_size, capacity)
-        self._init_bloom_key()
-
-    def add(self, value):
-        for seed in range(self.hash_count):
-            result = mmh3.hash(value, seed) % self.bit_size
-            self._client.setbit(self.key, result, 1)
-        return True
-
-    def exists(self, value):
-        if not self._client.exists(self.key):
-            return False
-        for seed in range(self.hash_count):
-            result = mmh3.hash(value, seed) % self.bit_size
-            if not self._client.getbit(self.key, result):
-                return False
-        return True
-
-    def _init_bloom_key(self):
-        lua_script = """
-        redis.call("SETBIT", KEYS[1], ARGV[1], ARGV[2])
-        redis.call("EXPIRE", KEYS[1], 604800)
-        """
-        if self._client.exists(self.key):
-            return True
-        execute = self._client.register_script(lua_script)
-        execute(keys=[self.key], args=[self.bit_size-1, 1])
-
-    @classmethod
-    def get_bit_size(cls, n, p):
-        return int(-(n * math.log(p)) / (math.log(2) ** 2))
-
-    @classmethod
-    def get_hash_count(cls, m, n):
-        return int((m / n) * math.log(2))
-
-

cobweb_/utils/dotting.py

@@ -1,32 +0,0 @@
-import json
-
-from aliyun.log import LogClient, LogItem, PutLogsRequest
-from cobweb import setting
-
-
-class LoghubDot:
-
-    def __init__(self):
-        self.client = LogClient(**setting.LOGHUB_CONFIG)
-
-    def build(self, topic, **kwargs):
-
-        temp = {}
-        log_items = []
-        log_item = LogItem()
-        for key, value in kwargs.items():
-            if not isinstance(value, str):
-                temp[key] = json.dumps(value, ensure_ascii=False)
-            else:
-                temp[key] = value
-        contents = sorted(temp.items())
-        log_item.set_contents(contents)
-        log_items.append(log_item)
-        request = PutLogsRequest(
-            project="databee-download-log",
-            logstore="cobweb_log",
-            topic=topic,
-            logitems=log_items,
-            compress=True
-        )
-        self.client.put_logs(request=request)

cobweb_/utils/oss.py

@@ -1,94 +0,0 @@
-from typing import List
-from cobweb import setting
-from requests import Response
-from oss2 import Auth, Bucket, models, PartIterator
-from cobweb.exceptions import oss_db_exception
-from cobweb.base.decorators import decorator_oss_db
-
-
-class OssUtil:
-
-    def __init__(
-        self,
-        bucket=None,
-        endpoint=None,
-        access_key=None,
-        secret_key=None,
-        chunk_size=None,
-        min_upload_size=None,
-        **kwargs
-    ):
-        self.bucket = bucket or setting.OSS_BUCKET
-        self.endpoint = endpoint or setting.OSS_ENDPOINT
-        self.chunk_size = int(chunk_size or setting.OSS_CHUNK_SIZE)
-        self.min_upload_size = int(min_upload_size or setting.OSS_MIN_UPLOAD_SIZE)
-
-        self._auth = Auth(
-            access_key_id=access_key or setting.OSS_ACCESS_KEY,
-            access_key_secret=secret_key or setting.OSS_SECRET_KEY
-        )
-        self._client = Bucket(
-            auth=self._auth,
-            endpoint=self.endpoint,
-            bucket_name=self.bucket,
-            **kwargs
-        )
-
-    def exists(self, key: str) -> bool:
-        return self._client.object_exists(key)
-
-    def head(self, key: str) -> models.HeadObjectResult:
-        return self._client.head_object(key)
-
-    @decorator_oss_db(exception=oss_db_exception.OssDBInitPartError)
-    def init_part(self, key) -> models.InitMultipartUploadResult:
-        """初始化分片上传"""
-        return self._client.init_multipart_upload(key)
-
-    @decorator_oss_db(exception=oss_db_exception.OssDBPutObjError)
-    def put(self, key, data) -> models.PutObjectResult:
-        """文件上传"""
-        return self._client.put_object(key, data)
-
-    @decorator_oss_db(exception=oss_db_exception.OssDBPutPartError)
-    def put_part(self, key, upload_id, position, data) -> models.PutObjectResult:
-        """分片上传"""
-        return self._client.upload_part(key, upload_id, position, data)
-
-    def list_part(self, key, upload_id):  # -> List[models.ListPartsResult]:
-        """获取分片列表"""
-        return [part_info for part_info in PartIterator(self._client, key, upload_id)]
-
-    @decorator_oss_db(exception=oss_db_exception.OssDBMergeError)
-    def merge(self, key, upload_id, parts=None) -> models.PutObjectResult:
-        """合并分片"""
-        headers = None if parts else {"x-oss-complete-all": "yes"}
-        return self._client.complete_multipart_upload(key, upload_id, parts, headers=headers)
-
-    @decorator_oss_db(exception=oss_db_exception.OssDBAppendObjError)
-    def append(self, key, position, data) -> models.AppendObjectResult:
-        """追加上传"""
-        return self._client.append_object(key, position, data)
-
-    def iter_data(self, data, chunk_size=None):
-        chunk_size = chunk_size or self.chunk_size
-        if isinstance(data, Response):
-            for part_data in data.iter_content(chunk_size):
-                yield part_data
-        if isinstance(data, bytes):
-            for i in range(0, len(data), chunk_size):
-                yield data[i:i + chunk_size]
-
-    def assemble(self, ready_data, data, chunk_size=None):
-        upload_data = b""
-        ready_data = ready_data + data
-        chunk_size = chunk_size or self.chunk_size
-        if len(ready_data) >= chunk_size:
-            upload_data = ready_data[:chunk_size]
-            ready_data = ready_data[chunk_size:]
-        return ready_data, upload_data
-
-    def content_length(self, key: str) -> int:
-        head = self.head(key)
-        return head.content_length
-

cobweb_/utils/tools.py

@@ -1,42 +0,0 @@
-import re
-import hashlib
-from typing import Union
-from importlib import import_module
-
-
-def md5(text: Union[str, bytes]) -> str:
-    if isinstance(text, str):
-        text = text.encode('utf-8')
-    return hashlib.md5(text).hexdigest()
-
-
-def build_path(site, url, file_type):
-    return f"{site}/{md5(url)}.{file_type}"
-
-
-def format_size(content_length: int) -> str:
-    units = ["KB", "MB", "GB", "TB"]
-    for i in range(4):
-        num = content_length / (1024 ** (i + 1))
-        if num < 1024:
-            return f"{round(num, 2)} {units[i]}"
-
-
-def dynamic_load_class(model_info):
-    if isinstance(model_info, str):
-        if "import" in model_info:
-            model_path, class_name = re.search(
-                r"from (.*?) import (.*?)$", model_info
-            ).groups()
-            model = import_module(model_path)
-            class_object = getattr(model, class_name)
-        else:
-            model_path, class_name = model_info.rsplit(".", 1)
-            model = import_module(model_path)
-            class_object = getattr(model, class_name)
-        return class_object
-    raise TypeError()
-
-
-# def download_log_info(item:dict) -> str:
-#     return "\n".join([" " * 12 + f"{str(k).ljust(14)}:    {str(v)}" for k, v in item.items()])

cobweb_launcher-1.3.6.dist-info/RECORD

@@ -1,111 +0,0 @@
-cobweb/__init__.py,sha256=oaEfsGUuGP0s39UbFRwrnsjMUeuB6QvQIAwStKFyUTk,83
-cobweb/constant.py,sha256=eofONAntk9O6S-cb4KbYGYHL_u7nBlOqqFOw_HzJHAU,3588
-cobweb/setting.py,sha256=pY6LKsgWI3164GiGA1z_y26LVf5-3mpiEgmm86mKRdY,3135
-cobweb/base/__init__.py,sha256=Na385Hhl9l2S8aPhcdJVPjmb02wkVM969bWQ84bCSQs,5095
-cobweb/base/basic.py,sha256=s5G4LBZiLUfoymV-gLSIqeH-OJ7q7-L35sBa6xEH3EI,7666
-cobweb/base/common_queue.py,sha256=Gor7sR3h1hlZWaI0XcNAbf0S15Ftjr3DFRWNTGL13uU,1137
-cobweb/base/decorators.py,sha256=wDCaQ94aAZGxks9Ljc0aXq6omDXT1_yzFy83ZW6VbVI,930
-cobweb/base/dotting.py,sha256=0SH8F2uAGWZjfODpTAXngYHz8JgfCm-RqpmQbfQ3NCY,1233
-cobweb/base/item.py,sha256=hYheVTV2Bozp4iciJpE2ZwBIXkaqBg4QQkRccP8yoVk,1049
-cobweb/base/log.py,sha256=L01hXdk3L2qEm9X1FOXQ9VmWIoHSELe0cyZvrdAN61A,2003
-cobweb/base/request.py,sha256=acGm3OzxsPed5VUTk7D9eeHZPMh7KUNQRUv44G5znZg,2659
-cobweb/base/response.py,sha256=eB1DWMXFCpn3cJ3yzgCRU1WeZAdayGDohRgdjdMUFN4,406
-cobweb/base/seed.py,sha256=PN5J4gKPEXylwyQeSGOBfauxHktxFr7RJe8nVX1hBw4,2987
-cobweb/crawlers/__init__.py,sha256=msvkB9mTpsgyj8JfNMsmwAcpy5kWk_2NrO1Adw2Hkw0,29
-cobweb/crawlers/base_crawler.py,sha256=ee_WSDnPQpPTk6wlFuY2UEx5L3hcsAZFcr6i3GLSry8,5751
-cobweb/crawlers/crawler.py,sha256=ZQ6yVA1EaQRdKJEY3DNqShzp9HPMwlSXapnsRW9E5Wc,2987
-cobweb/crawlers/file_crawler.py,sha256=2Sjbdgxzqd41WykKUQE3QQlGai3T8k-pmHNmPlTchjQ,4454
-cobweb/db/__init__.py,sha256=uZwSkd105EAwYo95oZQXAfofUKHVIAZZIPpNMy-hm2Q,56
-cobweb/db/api_db.py,sha256=bDc5dJQxq4z04h70KUTHd0OqUOEY7Cm3wcNJZtTvJIM,3015
-cobweb/db/redis_db.py,sha256=FvMzckJtmhwKhZqKoS23iXmJti5P2dnMVD5rJ__5LUw,5139
-cobweb/exceptions/__init__.py,sha256=E9SHnJBbhD7fOgPFMswqyOf8SKRDrI_i25L0bSpohvk,32
-cobweb/exceptions/oss_db_exception.py,sha256=iP_AImjNHT3-Iv49zCFQ3rdLnlvuHa3h2BXApgrOYpA,636
-cobweb/launchers/__init__.py,sha256=m_XNG2bWuMbirPt3d0_s-Ezl1xycfUxeqZnwq_kkfuo,116
-cobweb/launchers/launcher.py,sha256=NFwpc_0Um0hbDm1A8glWA4fcW6mNYL1eon4t3JAQUlw,7411
-cobweb/launchers/launcher_air.py,sha256=yPr395HVIIHAq6lqRcYJu7c0KkfO9V8O-2sn0hC96p0,2990
-cobweb/launchers/launcher_api.py,sha256=TfLrLXazFWsOJLI7caMGfZozCttL1WTwTo3uUpN_FV0,3370
-cobweb/launchers/launcher_pro.py,sha256=2H-TcvQx-ga78GLNTa-GXMLYAj9nEeCJSWf8xl-1ISQ,3374
-cobweb/pipelines/__init__.py,sha256=zSUsGtx6smbs2iXBXvYynReKSgky-3gjqaAtKVnA_OU,105
-cobweb/pipelines/base_pipeline.py,sha256=fYnWf79GmhufXpcnMa3te18SbmnVeYLwxfyo-zLd9CY,1577
-cobweb/pipelines/loghub_pipeline.py,sha256=cjPO6w6UJ0jNw2fVvdX0BCdlm58T7dmYXlxzXOBpvfY,1027
-cobweb/pipelines/pipeline.py,sha256=Pycm22bHId9a3gdP81D5y7SsuMndYooTb5n4zQxP7dM,1321
-cobweb/pipelines/pipeline_console.py,sha256=NEh-4zhuVAQOqwXLsqeb-rcNZ9_KXFUpL3otUTL5qBs,754
-cobweb/pipelines/pipeline_loghub.py,sha256=xZ6D55BGdiM71WUv83jyLGbEyUwhBHLJRZoXthBxxTs,1019
-cobweb/schedulers/__init__.py,sha256=y7Lv_7b0zfTl0OhIONb_8u1K1C9gVlBA-xz_XG_kI9g,85
-cobweb/schedulers/scheduler_api.py,sha256=mC54QOS0PEu4SFvxfD5Qr9239hAxwMrKTg-33rirANE,2112
-cobweb/schedulers/scheduler_redis.py,sha256=Aw7de0sXigRAxJgqUhHWu30hMBzgEWjkj-3OXXqmldg,2118
-cobweb/utils/__init__.py,sha256=YvD4mIDBd9jmGA6WJBcwkgDU2jRFNBCEbarZCSUBAHE,114
-cobweb/utils/bloom.py,sha256=vng-YbKgh9HbtpAWYf_nkUSbfVTOj40aqUUejRYlsCU,1752
-cobweb/utils/dotting.py,sha256=fZ-16TAf51dovGIKaHsQthgTkbSEMAteaJPluZhniJI,879
-cobweb/utils/oss.py,sha256=gyt8-UB07tVphZLQXMOf-JTJwU-mWq8KZkOXKkAf3uk,3513
-cobweb/utils/tools.py,sha256=5JEaaAwYoV9Sdla2UBIJn6faUBuXmxUMagm9ck6FVqs,1253
-cobweb_/__init__.py,sha256=CBd2oByCfc5EmH2dCZYVHkxXYZG-oWrLyTtZU5sEoP0,96
-cobweb_/constant.py,sha256=zy3XYsc1qp2B76_Fn_hVQ8eGHlPBd3OFlZK2cryE6FY,2839
-cobweb_/setting.py,sha256=47HZsw40HLpsmOmvij1lyQALPQQCN_tWlKZ0wbn2MtM,2216
-cobweb_/base/__init__.py,sha256=4gwWWQ0Q8cYG9cD7Lwf4XMqRGc5M_mapS3IczR6zeCE,222
-cobweb_/base/common_queue.py,sha256=W7PPZZFl52j3Mc916T0imHj7oAUelA6aKJwW-FecDPE,872
-cobweb_/base/decorators.py,sha256=wDCaQ94aAZGxks9Ljc0aXq6omDXT1_yzFy83ZW6VbVI,930
-cobweb_/base/item.py,sha256=hYheVTV2Bozp4iciJpE2ZwBIXkaqBg4QQkRccP8yoVk,1049
-cobweb_/base/log.py,sha256=L01hXdk3L2qEm9X1FOXQ9VmWIoHSELe0cyZvrdAN61A,2003
-cobweb_/base/request.py,sha256=tEkgMVUfdQI-kZuzWuiit9P_q4Q9-_RZh9aXXpc0314,2352
-cobweb_/base/response.py,sha256=eB1DWMXFCpn3cJ3yzgCRU1WeZAdayGDohRgdjdMUFN4,406
-cobweb_/base/seed.py,sha256=Uz_VBRlAxNYQcFHk3tsZFMlU96yPOedHaWGTvk-zKd8,2908
-cobweb_/crawlers/__init__.py,sha256=msvkB9mTpsgyj8JfNMsmwAcpy5kWk_2NrO1Adw2Hkw0,29
-cobweb_/crawlers/crawler.py,sha256=mPRc9GBfWi5AoSxB1jlARxvG_AzsPVRFil5O8RnOxCY,7018
-cobweb_/db/__init__.py,sha256=uZwSkd105EAwYo95oZQXAfofUKHVIAZZIPpNMy-hm2Q,56
-cobweb_/db/api_db.py,sha256=bDc5dJQxq4z04h70KUTHd0OqUOEY7Cm3wcNJZtTvJIM,3015
-cobweb_/db/redis_db.py,sha256=fumNZJiio-uQqRcSrymx8eJ1PqsdOwITe_Y-9JOXxrQ,4298
-cobweb_/exceptions/__init__.py,sha256=E9SHnJBbhD7fOgPFMswqyOf8SKRDrI_i25L0bSpohvk,32
-cobweb_/exceptions/oss_db_exception.py,sha256=iP_AImjNHT3-Iv49zCFQ3rdLnlvuHa3h2BXApgrOYpA,636
-cobweb_/launchers/__init__.py,sha256=qMuVlQcjErVK67HyKFZEsXf_rfZD5ODjx1QucSCKMOM,114
-cobweb_/launchers/launcher.py,sha256=sPts-xlgxoeIfl1fn1XR2XVZxLzt7He9xrYDfTHRAGo,7029
-cobweb_/launchers/launcher_air.py,sha256=KAk_M8F3029cXYe7m4nn3Nzyi89lbxJ2cqZjqW8iZ0E,2832
-cobweb_/launchers/launcher_api.py,sha256=Ih8f5xDcFlGBn6VSnlrpxcchMB48ugsj2NTWYgGYWfY,8669
-cobweb_/launchers/launcher_pro.py,sha256=NBJstQuB0o_jMiySJ14lk0Y3WAxxiScaQvXa1qtTSo4,8683
-cobweb_/pipelines/__init__.py,sha256=zSUsGtx6smbs2iXBXvYynReKSgky-3gjqaAtKVnA_OU,105
-cobweb_/pipelines/pipeline.py,sha256=4TJLX0sUHRxYndF5A4Vs5btUGI-wigkOcFvhTW1hLXI,2009
-cobweb_/pipelines/pipeline_console.py,sha256=NEh-4zhuVAQOqwXLsqeb-rcNZ9_KXFUpL3otUTL5qBs,754
-cobweb_/pipelines/pipeline_loghub.py,sha256=xZ6D55BGdiM71WUv83jyLGbEyUwhBHLJRZoXthBxxTs,1019
-cobweb_/utils/__init__.py,sha256=Ev2LZZ1-S56iQYDqFZrqadizEv4Gk8Of-DraH-_WnKY,109
-cobweb_/utils/bloom.py,sha256=vng-YbKgh9HbtpAWYf_nkUSbfVTOj40aqUUejRYlsCU,1752
-cobweb_/utils/dotting.py,sha256=fZ-16TAf51dovGIKaHsQthgTkbSEMAteaJPluZhniJI,879
-cobweb_/utils/oss.py,sha256=gyt8-UB07tVphZLQXMOf-JTJwU-mWq8KZkOXKkAf3uk,3513
-cobweb_/utils/tools.py,sha256=5JEaaAwYoV9Sdla2UBIJn6faUBuXmxUMagm9ck6FVqs,1253
-cobweb_new/__init__.py,sha256=CBd2oByCfc5EmH2dCZYVHkxXYZG-oWrLyTtZU5sEoP0,96
-cobweb_new/constant.py,sha256=wy2bLpGZrl1MtgVv-Z1Tmtj5uWh-KGzDrrGKdVStxV4,3236
-cobweb_new/setting.py,sha256=Ya3X4HbvDfSmMF2kSJwaaP1naxrWETTFW88T11agP7k,3035
-cobweb_new/base/__init__.py,sha256=L74KN3qZn6s33EXyxQ_vB3FF8mA7pZJd_ekkWiUKd5Y,2229
-cobweb_new/base/common_queue.py,sha256=Po6yY8HqpC6Wt6csd3Co3lBd7ygN2vmEECczgyc_sM8,1292
-cobweb_new/base/decorators.py,sha256=8VDpANSIhxhrFnwgQzAxM_8ZyDXKdn3zTH0oZIXqRPE,1801
-cobweb_new/base/item.py,sha256=hYheVTV2Bozp4iciJpE2ZwBIXkaqBg4QQkRccP8yoVk,1049
-cobweb_new/base/log.py,sha256=L01hXdk3L2qEm9X1FOXQ9VmWIoHSELe0cyZvrdAN61A,2003
-cobweb_new/base/request.py,sha256=tEkgMVUfdQI-kZuzWuiit9P_q4Q9-_RZh9aXXpc0314,2352
-cobweb_new/base/response.py,sha256=eB1DWMXFCpn3cJ3yzgCRU1WeZAdayGDohRgdjdMUFN4,406
-cobweb_new/base/seed.py,sha256=KBVxVU4jMB6oiw8HPtu-nDUVUZ6jiTjzR917jTYGCZs,2977
-cobweb_new/crawlers/__init__.py,sha256=msvkB9mTpsgyj8JfNMsmwAcpy5kWk_2NrO1Adw2Hkw0,29
-cobweb_new/crawlers/crawler-new.py,sha256=TAYMH2E3BTkjU6bFLlIMVfsR3cV2ggjA0moUpaXOe1Y,2762
-cobweb_new/crawlers/crawler.py,sha256=xiFNM0t69f5xlm59hPbO2MpqtdirVAUhD84-CLpyHPM,6349
-cobweb_new/db/__init__.py,sha256=uZwSkd105EAwYo95oZQXAfofUKHVIAZZIPpNMy-hm2Q,56
-cobweb_new/db/api_db.py,sha256=bDc5dJQxq4z04h70KUTHd0OqUOEY7Cm3wcNJZtTvJIM,3015
-cobweb_new/db/redis_db.py,sha256=FvMzckJtmhwKhZqKoS23iXmJti5P2dnMVD5rJ__5LUw,5139
-cobweb_new/exceptions/__init__.py,sha256=E9SHnJBbhD7fOgPFMswqyOf8SKRDrI_i25L0bSpohvk,32
-cobweb_new/exceptions/oss_db_exception.py,sha256=iP_AImjNHT3-Iv49zCFQ3rdLnlvuHa3h2BXApgrOYpA,636
-cobweb_new/launchers/__init__.py,sha256=qMuVlQcjErVK67HyKFZEsXf_rfZD5ODjx1QucSCKMOM,114
-cobweb_new/launchers/launcher.py,sha256=87P_2rRjzqyQXcG_EJ5Y6lMAk7saM8k1WBJcl9ANX6k,8309
-cobweb_new/launchers/launcher_air.py,sha256=KAk_M8F3029cXYe7m4nn3Nzyi89lbxJ2cqZjqW8iZ0E,2832
-cobweb_new/launchers/launcher_api.py,sha256=qPazoC7U-UmgebbiTkhl6f4yQmN34XMl6HawekhAhEo,5789
-cobweb_new/launchers/launcher_pro.py,sha256=QLjAiN8qMk4NklSY7ldBAR5OEEUB8sECuCCwRrFEC68,3414
-cobweb_new/launchers/tesss.py,sha256=pDe0wwhXbdjjmtfc7JLPfVOvs9yuc7Y8wLT1b1ueeEs,912
-cobweb_new/pipelines/__init__.py,sha256=zSUsGtx6smbs2iXBXvYynReKSgky-3gjqaAtKVnA_OU,105
-cobweb_new/pipelines/pipeline.py,sha256=3IRHHqrHblZ_18Cps2bGK6iugDjs-dde7p3AbarfiN8,1958
-cobweb_new/pipelines/pipeline_console.py,sha256=NEh-4zhuVAQOqwXLsqeb-rcNZ9_KXFUpL3otUTL5qBs,754
-cobweb_new/pipelines/pipeline_loghub.py,sha256=xZ6D55BGdiM71WUv83jyLGbEyUwhBHLJRZoXthBxxTs,1019
-cobweb_new/utils/__init__.py,sha256=c9macpjc15hrCUCdzO5RR_sgK_B9kvJKreSGprZ1ld4,112
-cobweb_new/utils/bloom.py,sha256=vng-YbKgh9HbtpAWYf_nkUSbfVTOj40aqUUejRYlsCU,1752
-cobweb_new/utils/oss.py,sha256=gyt8-UB07tVphZLQXMOf-JTJwU-mWq8KZkOXKkAf3uk,3513
-cobweb_new/utils/tools.py,sha256=5JEaaAwYoV9Sdla2UBIJn6faUBuXmxUMagm9ck6FVqs,1253
-cobweb_launcher-1.3.6.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
-cobweb_launcher-1.3.6.dist-info/METADATA,sha256=4q-fHNZ6nArBE6t6q0aTy8WoQYXFUYMW-ZnLZzO51yI,6509
-cobweb_launcher-1.3.6.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
-cobweb_launcher-1.3.6.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
-cobweb_launcher-1.3.6.dist-info/RECORD,,

cobweb_new/__init__.py

@@ -1,2 +0,0 @@
-from .launchers import LauncherAir, LauncherPro, LauncherApi
-from .constant import CrawlerModel

cobweb_new/base/__init__.py

@@ -1,72 +0,0 @@
-import time
-from inspect import isgenerator
-from typing import Callable, Union
-
-from .common_queue import Queue
-from .response import Response
-from .request import Request
-from .item import BaseItem, ConsoleItem
-from .seed import Seed
-
-from .log import logger
-# from .decorators import decorator_oss_db, stop, pause
-import decorators
-
-
-class TaskQueue:
-
-    SEED = Queue()          # 添加任务种子队列
-    TODO = Queue()          # 任务种子队列
-    REQUEST = Queue()       # 请求队列
-
-    DOWNLOAD = Queue()      # 下载任务队列
-    RESPONSE = Queue()      # 响应队列
-    DONE = Queue()          # 下载完成队列
-
-    UPLOAD = Queue()        # 任务上传队列
-
-    DELETE = Queue()        # 任务删除队列
-
-    def __init__(self, db):
-        self.db = db
-
-    @staticmethod
-    def is_empty():
-        total_length = TaskQueue.SEED.length
-        total_length += TaskQueue.TODO.length
-        total_length += TaskQueue.REQUEST.length
-        total_length += TaskQueue.DOWNLOAD.length
-        total_length += TaskQueue.RESPONSE.length
-        total_length += TaskQueue.UPLOAD.length
-        total_length += TaskQueue.DONE.length
-        total_length += TaskQueue.DELETE.length
-        return not bool(total_length)
-    # @staticmethod
-    # def distribute(it):
-
-    @staticmethod
-    def process_task(it: Union[Seed, Request, Response, BaseItem], crawler_func: Callable):
-        try:
-            iterators = crawler_func(it)
-            if not isgenerator(iterators):
-                raise TypeError(f"{crawler_func.__name__} function isn't a generator")
-            for tk in iterators:
-                if isinstance(tk, Request):
-                    TaskQueue.DOWNLOAD.push(tk)
-                elif isinstance(tk, Response):
-                    TaskQueue.RESPONSE.push(tk)
-                elif isinstance(tk, BaseItem):
-                    TaskQueue.UPLOAD.push(tk)
-                elif isinstance(tk, Seed):
-                    TaskQueue.SEED.push(tk)
-        except Exception as e:
-            if not isinstance(it, BaseItem):
-                it.seed.params.retry += 1
-
-            time.sleep(5)
-
-
-class Distribute:
-    """
-    数据分发器，将数据分发到各个队列中
-    """

cobweb_new/base/common_queue.py

@@ -1,53 +0,0 @@
-import time
-from collections import deque
-
-
-class Queue:
-
-    def __init__(self):
-        self._queue = deque()
-
-    @property
-    def length(self) -> int:
-        return len(self._queue)
-
-    def push(self, data, left: bool = False, direct_insertion: bool = False):
-        try:
-            if not data:
-                return None
-            if not direct_insertion and any(isinstance(data, t) for t in (list, tuple)):
-                self._queue.extendleft(data) if left else self._queue.extend(data)
-            else:
-                self._queue.appendleft(data) if left else self._queue.append(data)
-        except AttributeError:
-            pass
-
-    def pop(self, left: bool = True):
-        try:
-            return self._queue.popleft() if left else self._queue.pop()
-        except IndexError:
-            return None
-        except AttributeError:
-            return None
-
-    def clear(self):
-        self._queue.clear()
-
-    def get(self):
-        try:
-            yield self._queue.popleft()
-        except IndexError:
-            time.sleep(1)
-            yield None
-        except AttributeError:
-            yield None
-
-
-class RedisQueue(Queue):
-
-    def __init__(self, db):
-        super().__init__()
-        self.db = db
-
-    def pop(self, left: bool = True):
-        ...

cobweb_new/base/decorators.py

@@ -1,72 +0,0 @@
-import time
-import threading
-from functools import wraps
-
-
-def add_thread(num=1):
-    def decorator(func):
-        @wraps(func)
-        def wrapper(self, *args):
-            for i in range(num):
-                name = func.__name__ + "_" + str(i) if num > 1 else func.__name__
-                self._threads.append(threading.Thread(name=name, target=func, args=(self,) + args))
-        return wrapper
-
-    return decorator
-
-
-def pause(func):
-    @wraps(func)
-    def wrapper(self, *args, **kwargs):
-        while not self.pause.is_set():
-            try:
-                func(self, *args, **kwargs)
-            except Exception as e:
-                pass
-                # logger.info(f"{func.__name__}: " + str(e))
-            finally:
-                time.sleep(0.1)
-
-    return wrapper
-
-
-def stop(func):
-    @wraps(func)
-    def wrapper(self, *args, **kwargs):
-        while not self.stop.is_set():
-            try:
-                func(self, *args, **kwargs)
-            except Exception as e:
-                # logger.info(f"{func.__name__}: " + str(e))
-                pass
-            finally:
-                time.sleep(0.1)
-
-    return wrapper
-
-
-def decorator_oss_db(exception, retries=3):
-    def decorator(func):
-        @wraps(func)
-        def wrapper(callback_func, *args, **kwargs):
-            result = None
-            for i in range(retries):
-                msg = None
-                try:
-                    return func(callback_func, *args, **kwargs)
-                except Exception as e:
-                    result = None
-                    msg = e
-                finally:
-                    if result:
-                        return result
-
-                    if i >= 2 and msg:
-                        raise exception(msg)
-
-        return wrapper
-
-    return decorator
-
-
-

cobweb_new/base/item.py

@@ -1,46 +0,0 @@
-from .seed import Seed
-from collections import namedtuple
-
-
-class Item(type):
-
-    def __new__(cls, name, bases, dct):
-        new_class_instance = type.__new__(cls, name, bases, dct)
-        if name != "BaseItem":
-            table = getattr(new_class_instance, "__TABLE__")
-            fields = getattr(new_class_instance, "__FIELDS__")
-            new_class_instance.Data = namedtuple(table, fields)
-        return new_class_instance
-
-
-class BaseItem(metaclass=Item):
-
-    __TABLE__ = ""
-    __FIELDS__ = ""
-
-    def __init__(self, seed: Seed, **kwargs):
-        self.seed = seed
-
-        data = {}
-        for key, value in kwargs.items():
-            if key not in self.__FIELDS__:
-                self.__setattr__(key, value)
-            else:
-                data[key] = value
-
-        self.data = self.Data(**data)
-
-    @property
-    def to_dict(self):
-        return self.data._asdict()
-
-    @property
-    def table(self):
-        return self.Data.__name__
-
-
-class ConsoleItem(BaseItem):
-
-    __TABLE__ = "console"
-    __FIELDS__ = "data"
-

cobweb_new/base/log.py

@@ -1,94 +0,0 @@
-import logging
-
-
-class ColorCodes:
-    # Text Reset
-    RESET = "\033[0m"
-
-    # Regular Colors
-    RED = "\033[31m"
-    GREEN = "\033[32m"
-    YELLOW = "\033[33m"
-    BLUE = "\033[34m"
-    PURPLE = "\033[35m"
-    CYAN = "\033[36m"
-    WHITE = "\033[37m"
-
-    # Bright Colors
-    BRIGHT_RED = "\033[91m"
-    BRIGHT_GREEN = "\033[92m"
-    BRIGHT_YELLOW = "\033[93m"
-    BRIGHT_BLUE = "\033[94m"
-    BRIGHT_PURPLE = "\033[95m"
-    BRIGHT_CYAN = "\033[96m"
-    BRIGHT_WHITE = "\033[97m"
-
-    # Background Colors
-    BG_RED = "\033[41m"
-    BG_GREEN = "\033[42m"
-    BG_YELLOW = "\033[43m"
-    BG_BLUE = "\033[44m"
-    BG_PURPLE = "\033[45m"
-    BG_CYAN = "\033[46m"
-    BG_WHITE = "\033[47m"
-
-    # Bright Background Colors
-    BG_BRIGHT_RED = "\033[101m"
-    BG_BRIGHT_GREEN = "\033[102m"
-    BG_BRIGHT_YELLOW = "\033[103m"
-    BG_BRIGHT_BLUE = "\033[104m"
-    BG_BRIGHT_PURPLE = "\033[105m"
-    BG_BRIGHT_CYAN = "\033[106m"
-    BG_BRIGHT_WHITE = "\033[107m"
-
-    # Text Styles
-    BOLD = "\033[1m"
-    DIM = "\033[2m"
-    ITALIC = "\033[3m"
-    UNDERLINE = "\033[4m"
-    BLINK = "\033[5m"
-    REVERSE = "\033[7m"
-    HIDDEN = "\033[8m"
-
-
-class Log:
-    logging.getLogger('oss2.api').setLevel(logging.WARNING)
-    logging.basicConfig(
-        level=logging.INFO,
-        format=f'%(asctime)s %(name)s [%(filename)s:%(lineno)d %(funcName)s]'
-               f' %(levelname)s -> %(message)s'
-    )
-    log = logging.getLogger()
-
-    def set_log_name(self, name):
-        self.__class__.log = logging.getLogger(name)
-
-    @property
-    def debug(self):
-        return self.__class__.log.debug
-
-    @property
-    def info(self):
-        return self.__class__.log.info
-
-    @property
-    def warning(self):
-        return self.__class__.log.warning
-
-    @property
-    def exception(self):
-        return self.__class__.log.exception
-
-    @property
-    def error(self):
-        return self.__class__.log.error
-
-    @property
-    def critical(self):
-        return self.__class__.log.critical
-
-
-logger = Log()
-
-
-