cobweb-launcher 1.2.49__py3-none-any.whl → 1.3.1__py3-none-any.whl

cobweb_launcher-1.3.1.dist-info/RECORD

@@ -0,0 +1,108 @@
+cobweb/__init__.py,sha256=CBd2oByCfc5EmH2dCZYVHkxXYZG-oWrLyTtZU5sEoP0,96
+cobweb/constant.py,sha256=eofONAntk9O6S-cb4KbYGYHL_u7nBlOqqFOw_HzJHAU,3588
+cobweb/setting.py,sha256=pY6LKsgWI3164GiGA1z_y26LVf5-3mpiEgmm86mKRdY,3135
+cobweb/base/__init__.py,sha256=yrf__ULml-wN2CuLDgu7o1slbFhiSJwvOu_dsYstPLU,4849
+cobweb/base/basic.py,sha256=ukWBLDdBgJhV15pO13ZxcYYXVLmnWFctOyHuDvZQh4M,6390
+cobweb/base/common_queue.py,sha256=Gor7sR3h1hlZWaI0XcNAbf0S15Ftjr3DFRWNTGL13uU,1137
+cobweb/base/decorators.py,sha256=wDCaQ94aAZGxks9Ljc0aXq6omDXT1_yzFy83ZW6VbVI,930
+cobweb/base/dotting.py,sha256=0SH8F2uAGWZjfODpTAXngYHz8JgfCm-RqpmQbfQ3NCY,1233
+cobweb/base/item.py,sha256=hYheVTV2Bozp4iciJpE2ZwBIXkaqBg4QQkRccP8yoVk,1049
+cobweb/base/log.py,sha256=L01hXdk3L2qEm9X1FOXQ9VmWIoHSELe0cyZvrdAN61A,2003
+cobweb/base/request.py,sha256=acGm3OzxsPed5VUTk7D9eeHZPMh7KUNQRUv44G5znZg,2659
+cobweb/base/response.py,sha256=eB1DWMXFCpn3cJ3yzgCRU1WeZAdayGDohRgdjdMUFN4,406
+cobweb/base/seed.py,sha256=PN5J4gKPEXylwyQeSGOBfauxHktxFr7RJe8nVX1hBw4,2987
+cobweb/crawlers/__init__.py,sha256=msvkB9mTpsgyj8JfNMsmwAcpy5kWk_2NrO1Adw2Hkw0,29
+cobweb/crawlers/base_crawler.py,sha256=ee_WSDnPQpPTk6wlFuY2UEx5L3hcsAZFcr6i3GLSry8,5751
+cobweb/crawlers/crawler.py,sha256=rdXLgY91kKDSjOvDml3CUqJTR9RFifcLEYNiF6-gXEs,2964
+cobweb/crawlers/file_crawler.py,sha256=2Sjbdgxzqd41WykKUQE3QQlGai3T8k-pmHNmPlTchjQ,4454
+cobweb/db/__init__.py,sha256=uZwSkd105EAwYo95oZQXAfofUKHVIAZZIPpNMy-hm2Q,56
+cobweb/db/api_db.py,sha256=bDc5dJQxq4z04h70KUTHd0OqUOEY7Cm3wcNJZtTvJIM,3015
+cobweb/db/redis_db.py,sha256=FvMzckJtmhwKhZqKoS23iXmJti5P2dnMVD5rJ__5LUw,5139
+cobweb/exceptions/__init__.py,sha256=E9SHnJBbhD7fOgPFMswqyOf8SKRDrI_i25L0bSpohvk,32
+cobweb/exceptions/oss_db_exception.py,sha256=iP_AImjNHT3-Iv49zCFQ3rdLnlvuHa3h2BXApgrOYpA,636
+cobweb/launchers/__init__.py,sha256=uzfPkLbY2m0wsIR_s93VFxmO0U49GgUG7hXPzMYdye0,118
+cobweb/launchers/launcher.py,sha256=0BCXzZ05PFB6aQejwVBuMPwoxhlc2BLYL541HO5yiYM,7238
+cobweb/launchers/launcher_air.py,sha256=KAk_M8F3029cXYe7m4nn3Nzyi89lbxJ2cqZjqW8iZ0E,2832
+cobweb/launchers/launcher_api.py,sha256=qPazoC7U-UmgebbiTkhl6f4yQmN34XMl6HawekhAhEo,5789
+cobweb/launchers/launcher_pro.py,sha256=OMbo4Lu9yQagoznD3DawCT3hizBTvJBVKu10XfDy7Lw,3411
+cobweb/pipelines/__init__.py,sha256=zSUsGtx6smbs2iXBXvYynReKSgky-3gjqaAtKVnA_OU,105
+cobweb/pipelines/base_pipeline.py,sha256=fYnWf79GmhufXpcnMa3te18SbmnVeYLwxfyo-zLd9CY,1577
+cobweb/pipelines/loghub_pipeline.py,sha256=cjPO6w6UJ0jNw2fVvdX0BCdlm58T7dmYXlxzXOBpvfY,1027
+cobweb/pipelines/pipeline.py,sha256=FTl2JOoF3ysWa-mxb1pbmayEM4IXQH5a4L5CymlBQNU,1293
+cobweb/pipelines/pipeline_console.py,sha256=NEh-4zhuVAQOqwXLsqeb-rcNZ9_KXFUpL3otUTL5qBs,754
+cobweb/pipelines/pipeline_loghub.py,sha256=xZ6D55BGdiM71WUv83jyLGbEyUwhBHLJRZoXthBxxTs,1019
+cobweb/utils/__init__.py,sha256=Ev2LZZ1-S56iQYDqFZrqadizEv4Gk8Of-DraH-_WnKY,109
+cobweb/utils/bloom.py,sha256=vng-YbKgh9HbtpAWYf_nkUSbfVTOj40aqUUejRYlsCU,1752
+cobweb/utils/dotting.py,sha256=fZ-16TAf51dovGIKaHsQthgTkbSEMAteaJPluZhniJI,879
+cobweb/utils/oss.py,sha256=gyt8-UB07tVphZLQXMOf-JTJwU-mWq8KZkOXKkAf3uk,3513
+cobweb/utils/tools.py,sha256=5JEaaAwYoV9Sdla2UBIJn6faUBuXmxUMagm9ck6FVqs,1253
+cobweb_/__init__.py,sha256=CBd2oByCfc5EmH2dCZYVHkxXYZG-oWrLyTtZU5sEoP0,96
+cobweb_/constant.py,sha256=zy3XYsc1qp2B76_Fn_hVQ8eGHlPBd3OFlZK2cryE6FY,2839
+cobweb_/setting.py,sha256=47HZsw40HLpsmOmvij1lyQALPQQCN_tWlKZ0wbn2MtM,2216
+cobweb_/base/__init__.py,sha256=4gwWWQ0Q8cYG9cD7Lwf4XMqRGc5M_mapS3IczR6zeCE,222
+cobweb_/base/common_queue.py,sha256=W7PPZZFl52j3Mc916T0imHj7oAUelA6aKJwW-FecDPE,872
+cobweb_/base/decorators.py,sha256=wDCaQ94aAZGxks9Ljc0aXq6omDXT1_yzFy83ZW6VbVI,930
+cobweb_/base/item.py,sha256=hYheVTV2Bozp4iciJpE2ZwBIXkaqBg4QQkRccP8yoVk,1049
+cobweb_/base/log.py,sha256=L01hXdk3L2qEm9X1FOXQ9VmWIoHSELe0cyZvrdAN61A,2003
+cobweb_/base/request.py,sha256=tEkgMVUfdQI-kZuzWuiit9P_q4Q9-_RZh9aXXpc0314,2352
+cobweb_/base/response.py,sha256=eB1DWMXFCpn3cJ3yzgCRU1WeZAdayGDohRgdjdMUFN4,406
+cobweb_/base/seed.py,sha256=Uz_VBRlAxNYQcFHk3tsZFMlU96yPOedHaWGTvk-zKd8,2908
+cobweb_/crawlers/__init__.py,sha256=msvkB9mTpsgyj8JfNMsmwAcpy5kWk_2NrO1Adw2Hkw0,29
+cobweb_/crawlers/crawler.py,sha256=mPRc9GBfWi5AoSxB1jlARxvG_AzsPVRFil5O8RnOxCY,7018
+cobweb_/db/__init__.py,sha256=uZwSkd105EAwYo95oZQXAfofUKHVIAZZIPpNMy-hm2Q,56
+cobweb_/db/api_db.py,sha256=bDc5dJQxq4z04h70KUTHd0OqUOEY7Cm3wcNJZtTvJIM,3015
+cobweb_/db/redis_db.py,sha256=fumNZJiio-uQqRcSrymx8eJ1PqsdOwITe_Y-9JOXxrQ,4298
+cobweb_/exceptions/__init__.py,sha256=E9SHnJBbhD7fOgPFMswqyOf8SKRDrI_i25L0bSpohvk,32
+cobweb_/exceptions/oss_db_exception.py,sha256=iP_AImjNHT3-Iv49zCFQ3rdLnlvuHa3h2BXApgrOYpA,636
+cobweb_/launchers/__init__.py,sha256=qMuVlQcjErVK67HyKFZEsXf_rfZD5ODjx1QucSCKMOM,114
+cobweb_/launchers/launcher.py,sha256=sPts-xlgxoeIfl1fn1XR2XVZxLzt7He9xrYDfTHRAGo,7029
+cobweb_/launchers/launcher_air.py,sha256=KAk_M8F3029cXYe7m4nn3Nzyi89lbxJ2cqZjqW8iZ0E,2832
+cobweb_/launchers/launcher_api.py,sha256=Ih8f5xDcFlGBn6VSnlrpxcchMB48ugsj2NTWYgGYWfY,8669
+cobweb_/launchers/launcher_pro.py,sha256=NBJstQuB0o_jMiySJ14lk0Y3WAxxiScaQvXa1qtTSo4,8683
+cobweb_/pipelines/__init__.py,sha256=zSUsGtx6smbs2iXBXvYynReKSgky-3gjqaAtKVnA_OU,105
+cobweb_/pipelines/pipeline.py,sha256=4TJLX0sUHRxYndF5A4Vs5btUGI-wigkOcFvhTW1hLXI,2009
+cobweb_/pipelines/pipeline_console.py,sha256=NEh-4zhuVAQOqwXLsqeb-rcNZ9_KXFUpL3otUTL5qBs,754
+cobweb_/pipelines/pipeline_loghub.py,sha256=xZ6D55BGdiM71WUv83jyLGbEyUwhBHLJRZoXthBxxTs,1019
+cobweb_/utils/__init__.py,sha256=Ev2LZZ1-S56iQYDqFZrqadizEv4Gk8Of-DraH-_WnKY,109
+cobweb_/utils/bloom.py,sha256=vng-YbKgh9HbtpAWYf_nkUSbfVTOj40aqUUejRYlsCU,1752
+cobweb_/utils/dotting.py,sha256=fZ-16TAf51dovGIKaHsQthgTkbSEMAteaJPluZhniJI,879
+cobweb_/utils/oss.py,sha256=gyt8-UB07tVphZLQXMOf-JTJwU-mWq8KZkOXKkAf3uk,3513
+cobweb_/utils/tools.py,sha256=5JEaaAwYoV9Sdla2UBIJn6faUBuXmxUMagm9ck6FVqs,1253
+cobweb_new/__init__.py,sha256=CBd2oByCfc5EmH2dCZYVHkxXYZG-oWrLyTtZU5sEoP0,96
+cobweb_new/constant.py,sha256=wy2bLpGZrl1MtgVv-Z1Tmtj5uWh-KGzDrrGKdVStxV4,3236
+cobweb_new/setting.py,sha256=Ya3X4HbvDfSmMF2kSJwaaP1naxrWETTFW88T11agP7k,3035
+cobweb_new/base/__init__.py,sha256=L74KN3qZn6s33EXyxQ_vB3FF8mA7pZJd_ekkWiUKd5Y,2229
+cobweb_new/base/common_queue.py,sha256=Po6yY8HqpC6Wt6csd3Co3lBd7ygN2vmEECczgyc_sM8,1292
+cobweb_new/base/decorators.py,sha256=8VDpANSIhxhrFnwgQzAxM_8ZyDXKdn3zTH0oZIXqRPE,1801
+cobweb_new/base/item.py,sha256=hYheVTV2Bozp4iciJpE2ZwBIXkaqBg4QQkRccP8yoVk,1049
+cobweb_new/base/log.py,sha256=L01hXdk3L2qEm9X1FOXQ9VmWIoHSELe0cyZvrdAN61A,2003
+cobweb_new/base/request.py,sha256=tEkgMVUfdQI-kZuzWuiit9P_q4Q9-_RZh9aXXpc0314,2352
+cobweb_new/base/response.py,sha256=eB1DWMXFCpn3cJ3yzgCRU1WeZAdayGDohRgdjdMUFN4,406
+cobweb_new/base/seed.py,sha256=KBVxVU4jMB6oiw8HPtu-nDUVUZ6jiTjzR917jTYGCZs,2977
+cobweb_new/crawlers/__init__.py,sha256=msvkB9mTpsgyj8JfNMsmwAcpy5kWk_2NrO1Adw2Hkw0,29
+cobweb_new/crawlers/crawler-new.py,sha256=TAYMH2E3BTkjU6bFLlIMVfsR3cV2ggjA0moUpaXOe1Y,2762
+cobweb_new/crawlers/crawler.py,sha256=xiFNM0t69f5xlm59hPbO2MpqtdirVAUhD84-CLpyHPM,6349
+cobweb_new/db/__init__.py,sha256=uZwSkd105EAwYo95oZQXAfofUKHVIAZZIPpNMy-hm2Q,56
+cobweb_new/db/api_db.py,sha256=bDc5dJQxq4z04h70KUTHd0OqUOEY7Cm3wcNJZtTvJIM,3015
+cobweb_new/db/redis_db.py,sha256=FvMzckJtmhwKhZqKoS23iXmJti5P2dnMVD5rJ__5LUw,5139
+cobweb_new/exceptions/__init__.py,sha256=E9SHnJBbhD7fOgPFMswqyOf8SKRDrI_i25L0bSpohvk,32
+cobweb_new/exceptions/oss_db_exception.py,sha256=iP_AImjNHT3-Iv49zCFQ3rdLnlvuHa3h2BXApgrOYpA,636
+cobweb_new/launchers/__init__.py,sha256=qMuVlQcjErVK67HyKFZEsXf_rfZD5ODjx1QucSCKMOM,114
+cobweb_new/launchers/launcher.py,sha256=87P_2rRjzqyQXcG_EJ5Y6lMAk7saM8k1WBJcl9ANX6k,8309
+cobweb_new/launchers/launcher_air.py,sha256=KAk_M8F3029cXYe7m4nn3Nzyi89lbxJ2cqZjqW8iZ0E,2832
+cobweb_new/launchers/launcher_api.py,sha256=qPazoC7U-UmgebbiTkhl6f4yQmN34XMl6HawekhAhEo,5789
+cobweb_new/launchers/launcher_pro.py,sha256=QLjAiN8qMk4NklSY7ldBAR5OEEUB8sECuCCwRrFEC68,3414
+cobweb_new/launchers/tesss.py,sha256=pDe0wwhXbdjjmtfc7JLPfVOvs9yuc7Y8wLT1b1ueeEs,912
+cobweb_new/pipelines/__init__.py,sha256=zSUsGtx6smbs2iXBXvYynReKSgky-3gjqaAtKVnA_OU,105
+cobweb_new/pipelines/pipeline.py,sha256=3IRHHqrHblZ_18Cps2bGK6iugDjs-dde7p3AbarfiN8,1958
+cobweb_new/pipelines/pipeline_console.py,sha256=NEh-4zhuVAQOqwXLsqeb-rcNZ9_KXFUpL3otUTL5qBs,754
+cobweb_new/pipelines/pipeline_loghub.py,sha256=xZ6D55BGdiM71WUv83jyLGbEyUwhBHLJRZoXthBxxTs,1019
+cobweb_new/utils/__init__.py,sha256=c9macpjc15hrCUCdzO5RR_sgK_B9kvJKreSGprZ1ld4,112
+cobweb_new/utils/bloom.py,sha256=vng-YbKgh9HbtpAWYf_nkUSbfVTOj40aqUUejRYlsCU,1752
+cobweb_new/utils/oss.py,sha256=gyt8-UB07tVphZLQXMOf-JTJwU-mWq8KZkOXKkAf3uk,3513
+cobweb_new/utils/tools.py,sha256=5JEaaAwYoV9Sdla2UBIJn6faUBuXmxUMagm9ck6FVqs,1253
+cobweb_launcher-1.3.1.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
+cobweb_launcher-1.3.1.dist-info/METADATA,sha256=WiWGFXMCuA4T5UJ3S5gCbpqRyN5Ja6ZmK9lgtOm7BEI,6509
+cobweb_launcher-1.3.1.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
+cobweb_launcher-1.3.1.dist-info/top_level.txt,sha256=A0GPGeX6QtxXg7AJno3SVRTHtVCCqeRIOrpwDoXg9qs,15
+cobweb_launcher-1.3.1.dist-info/RECORD,,

cobweb_launcher-1.3.1.dist-info/top_level.txt

@@ -0,0 +1,2 @@
+cobweb
+cobweb_

cobweb_new/__init__.py

@@ -0,0 +1,2 @@
+from .launchers import LauncherAir, LauncherPro, LauncherApi
+from .constant import CrawlerModel

cobweb_new/base/__init__.py

@@ -0,0 +1,72 @@
+import time
+from inspect import isgenerator
+from typing import Callable, Union
+
+from .common_queue import Queue
+from .response import Response
+from .request import Request
+from .item import BaseItem, ConsoleItem
+from .seed import Seed
+
+from .log import logger
+# from .decorators import decorator_oss_db, stop, pause
+import decorators
+
+
+class TaskQueue:
+
+    SEED = Queue()          # 添加任务种子队列
+    TODO = Queue()          # 任务种子队列
+    REQUEST = Queue()       # 请求队列
+
+    DOWNLOAD = Queue()      # 下载任务队列
+    RESPONSE = Queue()      # 响应队列
+    DONE = Queue()          # 下载完成队列
+
+    UPLOAD = Queue()        # 任务上传队列
+
+    DELETE = Queue()        # 任务删除队列
+
+    def __init__(self, db):
+        self.db = db
+
+    @staticmethod
+    def is_empty():
+        total_length = TaskQueue.SEED.length
+        total_length += TaskQueue.TODO.length
+        total_length += TaskQueue.REQUEST.length
+        total_length += TaskQueue.DOWNLOAD.length
+        total_length += TaskQueue.RESPONSE.length
+        total_length += TaskQueue.UPLOAD.length
+        total_length += TaskQueue.DONE.length
+        total_length += TaskQueue.DELETE.length
+        return not bool(total_length)
+    # @staticmethod
+    # def distribute(it):
+
+    @staticmethod
+    def process_task(it: Union[Seed, Request, Response, BaseItem], crawler_func: Callable):
+        try:
+            iterators = crawler_func(it)
+            if not isgenerator(iterators):
+                raise TypeError(f"{crawler_func.__name__} function isn't a generator")
+            for tk in iterators:
+                if isinstance(tk, Request):
+                    TaskQueue.DOWNLOAD.push(tk)
+                elif isinstance(tk, Response):
+                    TaskQueue.RESPONSE.push(tk)
+                elif isinstance(tk, BaseItem):
+                    TaskQueue.UPLOAD.push(tk)
+                elif isinstance(tk, Seed):
+                    TaskQueue.SEED.push(tk)
+        except Exception as e:
+            if not isinstance(it, BaseItem):
+                it.seed.params.retry += 1
+
+            time.sleep(5)
+
+
+class Distribute:
+    """
+    数据分发器，将数据分发到各个队列中
+    """

cobweb_new/base/common_queue.py

@@ -0,0 +1,53 @@
+import time
+from collections import deque
+
+
+class Queue:
+
+    def __init__(self):
+        self._queue = deque()
+
+    @property
+    def length(self) -> int:
+        return len(self._queue)
+
+    def push(self, data, left: bool = False, direct_insertion: bool = False):
+        try:
+            if not data:
+                return None
+            if not direct_insertion and any(isinstance(data, t) for t in (list, tuple)):
+                self._queue.extendleft(data) if left else self._queue.extend(data)
+            else:
+                self._queue.appendleft(data) if left else self._queue.append(data)
+        except AttributeError:
+            pass
+
+    def pop(self, left: bool = True):
+        try:
+            return self._queue.popleft() if left else self._queue.pop()
+        except IndexError:
+            return None
+        except AttributeError:
+            return None
+
+    def clear(self):
+        self._queue.clear()
+
+    def get(self):
+        try:
+            yield self._queue.popleft()
+        except IndexError:
+            time.sleep(1)
+            yield None
+        except AttributeError:
+            yield None
+
+
+class RedisQueue(Queue):
+
+    def __init__(self, db):
+        super().__init__()
+        self.db = db
+
+    def pop(self, left: bool = True):
+        ...

cobweb_new/base/decorators.py

@@ -0,0 +1,72 @@
+import time
+import threading
+from functools import wraps
+
+
+def add_thread(num=1):
+    def decorator(func):
+        @wraps(func)
+        def wrapper(self, *args):
+            for i in range(num):
+                name = func.__name__ + "_" + str(i) if num > 1 else func.__name__
+                self._threads.append(threading.Thread(name=name, target=func, args=(self,) + args))
+        return wrapper
+
+    return decorator
+
+
+def pause(func):
+    @wraps(func)
+    def wrapper(self, *args, **kwargs):
+        while not self.pause.is_set():
+            try:
+                func(self, *args, **kwargs)
+            except Exception as e:
+                pass
+                # logger.info(f"{func.__name__}: " + str(e))
+            finally:
+                time.sleep(0.1)
+
+    return wrapper
+
+
+def stop(func):
+    @wraps(func)
+    def wrapper(self, *args, **kwargs):
+        while not self.stop.is_set():
+            try:
+                func(self, *args, **kwargs)
+            except Exception as e:
+                # logger.info(f"{func.__name__}: " + str(e))
+                pass
+            finally:
+                time.sleep(0.1)
+
+    return wrapper
+
+
+def decorator_oss_db(exception, retries=3):
+    def decorator(func):
+        @wraps(func)
+        def wrapper(callback_func, *args, **kwargs):
+            result = None
+            for i in range(retries):
+                msg = None
+                try:
+                    return func(callback_func, *args, **kwargs)
+                except Exception as e:
+                    result = None
+                    msg = e
+                finally:
+                    if result:
+                        return result
+
+                    if i >= 2 and msg:
+                        raise exception(msg)
+
+        return wrapper
+
+    return decorator
+
+
+

cobweb_new/base/item.py

@@ -0,0 +1,46 @@
+from .seed import Seed
+from collections import namedtuple
+
+
+class Item(type):
+
+    def __new__(cls, name, bases, dct):
+        new_class_instance = type.__new__(cls, name, bases, dct)
+        if name != "BaseItem":
+            table = getattr(new_class_instance, "__TABLE__")
+            fields = getattr(new_class_instance, "__FIELDS__")
+            new_class_instance.Data = namedtuple(table, fields)
+        return new_class_instance
+
+
+class BaseItem(metaclass=Item):
+
+    __TABLE__ = ""
+    __FIELDS__ = ""
+
+    def __init__(self, seed: Seed, **kwargs):
+        self.seed = seed
+
+        data = {}
+        for key, value in kwargs.items():
+            if key not in self.__FIELDS__:
+                self.__setattr__(key, value)
+            else:
+                data[key] = value
+
+        self.data = self.Data(**data)
+
+    @property
+    def to_dict(self):
+        return self.data._asdict()
+
+    @property
+    def table(self):
+        return self.Data.__name__
+
+
+class ConsoleItem(BaseItem):
+
+    __TABLE__ = "console"
+    __FIELDS__ = "data"
+

cobweb_new/base/log.py

@@ -0,0 +1,94 @@
+import logging
+
+
+class ColorCodes:
+    # Text Reset
+    RESET = "\033[0m"
+
+    # Regular Colors
+    RED = "\033[31m"
+    GREEN = "\033[32m"
+    YELLOW = "\033[33m"
+    BLUE = "\033[34m"
+    PURPLE = "\033[35m"
+    CYAN = "\033[36m"
+    WHITE = "\033[37m"
+
+    # Bright Colors
+    BRIGHT_RED = "\033[91m"
+    BRIGHT_GREEN = "\033[92m"
+    BRIGHT_YELLOW = "\033[93m"
+    BRIGHT_BLUE = "\033[94m"
+    BRIGHT_PURPLE = "\033[95m"
+    BRIGHT_CYAN = "\033[96m"
+    BRIGHT_WHITE = "\033[97m"
+
+    # Background Colors
+    BG_RED = "\033[41m"
+    BG_GREEN = "\033[42m"
+    BG_YELLOW = "\033[43m"
+    BG_BLUE = "\033[44m"
+    BG_PURPLE = "\033[45m"
+    BG_CYAN = "\033[46m"
+    BG_WHITE = "\033[47m"
+
+    # Bright Background Colors
+    BG_BRIGHT_RED = "\033[101m"
+    BG_BRIGHT_GREEN = "\033[102m"
+    BG_BRIGHT_YELLOW = "\033[103m"
+    BG_BRIGHT_BLUE = "\033[104m"
+    BG_BRIGHT_PURPLE = "\033[105m"
+    BG_BRIGHT_CYAN = "\033[106m"
+    BG_BRIGHT_WHITE = "\033[107m"
+
+    # Text Styles
+    BOLD = "\033[1m"
+    DIM = "\033[2m"
+    ITALIC = "\033[3m"
+    UNDERLINE = "\033[4m"
+    BLINK = "\033[5m"
+    REVERSE = "\033[7m"
+    HIDDEN = "\033[8m"
+
+
+class Log:
+    logging.getLogger('oss2.api').setLevel(logging.WARNING)
+    logging.basicConfig(
+        level=logging.INFO,
+        format=f'%(asctime)s %(name)s [%(filename)s:%(lineno)d %(funcName)s]'
+               f' %(levelname)s -> %(message)s'
+    )
+    log = logging.getLogger()
+
+    def set_log_name(self, name):
+        self.__class__.log = logging.getLogger(name)
+
+    @property
+    def debug(self):
+        return self.__class__.log.debug
+
+    @property
+    def info(self):
+        return self.__class__.log.info
+
+    @property
+    def warning(self):
+        return self.__class__.log.warning
+
+    @property
+    def exception(self):
+        return self.__class__.log.exception
+
+    @property
+    def error(self):
+        return self.__class__.log.error
+
+    @property
+    def critical(self):
+        return self.__class__.log.critical
+
+
+logger = Log()
+
+
+

cobweb_new/base/request.py

@@ -0,0 +1,82 @@
+import random
+import requests
+
+
+class Request:
+
+    __REQUEST_ATTRS__ = {
+        "params",
+        "headers",
+        "cookies",
+        "data",
+        "json",
+        "files",
+        "auth",
+        "timeout",
+        "proxies",
+        "hooks",
+        "stream",
+        "verify",
+        "cert",
+        "allow_redirects",
+    }
+
+    def __init__(
+            self,
+            url,
+            seed,
+            random_ua=True,
+            check_status_code=True,
+            **kwargs
+    ):
+        self.url = url
+        self.seed = seed
+        self.check_status_code = check_status_code
+        self.request_setting = {}
+
+        for k, v in kwargs.items():
+            if k in self.__class__.__REQUEST_ATTRS__:
+                self.request_setting[k] = v
+                continue
+            self.__setattr__(k, v)
+
+        if not getattr(self, "method", None):
+            self.method = "POST" if self.request_setting.get("data") or self.request_setting.get("json") else "GET"
+
+        if random_ua:
+            self._build_header()
+
+    @property
+    def _random_ua(self) -> str:
+        v1 = random.randint(4, 15)
+        v2 = random.randint(3, 11)
+        v3 = random.randint(1, 16)
+        v4 = random.randint(533, 605)
+        v5 = random.randint(1000, 6000)
+        v6 = random.randint(10, 80)
+        user_agent = (f"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_{v1}_{v2}) AppleWebKit/{v4}.{v3} "
+                      f"(KHTML, like Gecko) Chrome/105.0.0.0 Safari/{v4}.{v3} Edg/105.0.{v5}.{v6}")
+        return user_agent
+
+    def _build_header(self) -> dict:
+        if not self.request_setting.get("headers"):
+            self.request_setting["headers"] = {"accept": "*/*", "user-agent": self._random_ua}
+        elif "user-agent" not in [key.lower() for key in self.request_setting["headers"].keys()]:
+            self.request_setting["headers"]["user-agent"] = self._random_ua
+
+    def download(self) -> requests.Response:
+        response = requests.request(self.method, self.url, **self.request_setting)
+        if self.check_status_code:
+            response.raise_for_status()
+        return response
+
+    @property
+    def to_dict(self):
+        _dict = self.__dict__.copy()
+        _dict.pop('url')
+        _dict.pop('seed')
+        _dict.pop('check_status_code')
+        _dict.pop('request_setting')
+        return _dict
+
+

cobweb_new/base/response.py

@@ -0,0 +1,23 @@
+
+
+class Response:
+
+    def __init__(
+            self,
+            seed,
+            response,
+            **kwargs
+    ):
+        self.seed = seed
+        self.response = response
+
+        for k, v in kwargs.items():
+            self.__setattr__(k, v)
+
+    @property
+    def to_dict(self):
+        _dict = self.__dict__.copy()
+        _dict.pop('seed')
+        _dict.pop('response')
+        return _dict
+

cobweb_new/base/seed.py

@@ -0,0 +1,118 @@
+import json
+import time
+import hashlib
+
+
+class SeedParams:
+
+    def __init__(self, retry, priority, seed_version, seed_status=None):
+        self.retry = retry or 0
+        self.priority = priority or 300
+        self.seed_version = seed_version or int(time.time())
+        self.seed_status = seed_status
+
+
+class Seed:
+
+    __SEED_PARAMS__ = [
+        "retry",
+        "priority",
+        "seed_version",
+        "seed_status"
+    ]
+
+    def __init__(
+            self,
+            seed,
+            sid=None,
+            retry=None,
+            priority=None,
+            seed_version=None,
+            seed_status=None,
+            **kwargs
+    ):
+        if any(isinstance(seed, t) for t in (str, bytes)):
+            try:
+                item = json.loads(seed)
+                self._init_seed(item)
+            except json.JSONDecodeError:
+                self.__setattr__("url", seed)
+        elif isinstance(seed, dict):
+            self._init_seed(seed)
+        else:
+            raise TypeError(Exception(
+                f"seed type error, "
+                f"must be str or dict! "
+                f"seed: {seed}"
+            ))
+
+        seed_params = {
+            "retry": retry,
+            "priority": priority,
+            "seed_version": seed_version,
+            "seed_status": seed_status,
+        }
+
+        if kwargs:
+            self._init_seed(kwargs)
+            seed_params.update({
+                k:v for k, v in kwargs.items()
+                if k in self.__SEED_PARAMS__
+            })
+        if sid or not getattr(self, "sid", None):
+            self._init_id(sid)
+        self.params = SeedParams(**seed_params)
+
+    def __getattr__(self, name):
+        return None
+
+    def __setitem__(self, key, value):
+        setattr(self, key, value)
+
+    def __getitem__(self, item):
+        return getattr(self, item)
+
+    def __str__(self):
+        return json.dumps(self.__dict__, ensure_ascii=False)
+
+    def __repr__(self):
+        chars = [f"{k}={v}" for k, v in self.__dict__.items()]
+        return f'{self.__class__.__name__}({", ".join(chars)})'
+
+    def _init_seed(self, seed_info:dict):
+        for k, v in seed_info.items():
+            if k not in self.__SEED_PARAMS__:
+                self.__setattr__(k, v)
+
+    def _init_id(self, sid):
+        if not sid:
+            sid = hashlib.md5(self.to_string.encode()).hexdigest()
+        self.__setattr__("sid", sid)
+
+    @property
+    def to_dict(self) -> dict:
+        seed = self.__dict__.copy()
+        if seed.get("params"):
+            del seed["params"]
+        return seed
+
+    @property
+    def to_string(self) -> str:
+        return json.dumps(
+            self.to_dict,
+            ensure_ascii=False,
+            separators=(",", ":")
+        )
+
+    # @property
+    # def get_all(self):
+    #     return json.dumps(
+    #         self.__dict__,
+    #         ensure_ascii=False,
+    #         separators=(",", ":")
+    #     )
+
+    @property
+    def seed(self):
+        return self
+