cobweb-launcher 0.1.20__py3-none-any.whl → 0.1.21__py3-none-any.whl

cobweb/config.py

@@ -0,0 +1,164 @@
+import json
+from collections import namedtuple
+from base.utils import struct_table_name
+
+StorerInfo = namedtuple(
+    "StorerInfo",
+    "DB, table, fields, length, config"
+)
+SchedulerInfo = namedtuple(
+    "SchedulerInfo",
+    "DB, table, sql, length, size, config",
+)
+RedisInfo = namedtuple(
+    "RedisInfo",
+    "host, port, username, password, db",
+    defaults=("localhost", 6379, None, None, 0)
+)
+
+# redis_info = dict(
+#     host="localhost",
+#     port=6379,
+#     username=None,
+#     password=None,
+#     db=0
+# )
+
+
+class SchedulerDB:
+
+    @staticmethod
+    def default():
+        from db.scheduler.default import Default
+        return SchedulerInfo(DB=Default, table="", sql="", length=100, size=500000, config=None)
+
+    @staticmethod
+    def textfile(table, sql=None, length=100, size=500000):
+        from db.scheduler.textfile import Textfile
+        return SchedulerInfo(DB=Textfile, table=table, sql=sql, length=length, size=size, config=None)
+
+    @staticmethod
+    def diy(DB, table, sql=None, length=100, size=500000, config=None):
+        from base.interface import SchedulerInterface
+        if not isinstance(DB, SchedulerInterface):
+            raise Exception("DB must be inherit from SchedulerInterface")
+        return SchedulerInfo(DB=DB, table=table, sql=sql, length=length, size=size, config=config)
+
+    # @staticmethod
+    # def info(scheduler_info):
+    #     if not scheduler_info:
+    #         return SchedulerDB.default()
+    #
+    #     if isinstance(scheduler_info, SchedulerInfo):
+    #         return scheduler_info
+    #
+    #     if isinstance(scheduler_info, str):
+    #         scheduler = json.loads(scheduler_info)
+    #         if isinstance(scheduler, dict):
+    #             db_name = scheduler["DB"]
+    #             if db_name in dir(SchedulerDB):
+    #                 del scheduler["DB"]
+    #             else:
+    #                 db_name = "diy"
+    #             func = getattr(SchedulerDB, db_name)
+    #             return func(**scheduler)
+
+
+class StorerDB:
+
+    @staticmethod
+    def console(table, fields, length=200):
+        from db.storer.console import Console
+        table = struct_table_name(table)
+        return StorerInfo(DB=Console, table=table, fields=fields, length=length, config=None)
+
+    @staticmethod
+    def textfile(table, fields, length=200):
+        from db.storer.textfile import Textfile
+        table = struct_table_name(table)
+        return StorerInfo(DB=Textfile, table=table, fields=fields, length=length, config=None)
+
+    @staticmethod
+    def loghub(table, fields, length=200, config=None):
+        from db.storer.loghub import Loghub
+        table = struct_table_name(table)
+        return StorerInfo(DB=Loghub, table=table, fields=fields, length=length, config=config)
+
+    @staticmethod
+    def diy(DB, table, fields, length=200, config=None):
+        from base.interface import StorerInterface
+        if not isinstance(DB, StorerInterface):
+            raise Exception("DB must be inherit from StorerInterface")
+        table = struct_table_name(table)
+        return StorerInfo(DB=DB, table=table, fields=fields, length=length, config=config)
+
+    # @staticmethod
+    # def info(storer_info):
+    #     if not storer_info:
+    #         return None
+    #
+    #     if isinstance(storer_info, str):
+    #         storer_info = json.loads(storer_info)
+    #
+    #     if any(isinstance(storer_info, t) for t in (dict, StorerInfo)):
+    #         storer_info = [storer_info]
+    #
+    #     if not isinstance(storer_info, list):
+    #         raise Exception("StorerDB.info storer_info")
+    #
+    #     storer_info_list = []
+    #     for storer in storer_info:
+    #         if isinstance(storer, StorerInfo):
+    #             storer_info_list.append(storer)
+    #         else:
+    #             db_name = storer["DB"]
+    #             if db_name in dir(StorerDB):
+    #                 del storer["DB"]
+    #             else:
+    #                 db_name = "diy"
+    #             func = getattr(StorerDB, db_name)
+    #             storer_info_list.append(func(**storer))
+    #     return storer_info_list
+
+
+
+def deal(config, tag):
+    if isinstance(config, dict):
+        if tag == 0:
+            return RedisInfo(**config)
+        elif tag == 1:
+            db_name = config["DB"]
+            if db_name in dir(SchedulerDB):
+                del config["DB"]
+            else:
+                db_name = "diy"
+            func = getattr(SchedulerDB, db_name)
+            return func(**config)
+        elif tag == 2:
+            db_name = config["DB"]
+            if db_name in dir(StorerDB):
+                del config["DB"]
+            else:
+                db_name = "diy"
+            func = getattr(StorerDB, db_name)
+            return func(**config)
+        raise ValueError("tag must be in [0, 1, 2]")
+    elif any(isinstance(config, t) for t in (StorerInfo, SchedulerInfo, RedisInfo)):
+        return config
+    raise TypeError("config must be in [StorerInfo, SchedulerInfo, RedisInfo]")
+
+
+def info(configs, tag = 0):
+    if configs is None:
+        return SchedulerDB.default() if tag == 1 else None
+
+    if isinstance(configs, str):
+        configs = json.loads(configs)
+
+    if tag == 0:
+        return deal(configs, tag)
+
+    if not isinstance(configs, list):
+        configs = [configs]
+
+    return [deal(config, tag) for config in configs]

cobweb/db/redis_db.py

@@ -119,7 +119,7 @@ class RedisDB:
     def get_seed(self, length: int = 200):
         cs = time.time()
 
-        if self._get_lock(key=self.update_lock, t=Setting.SCHEDULER_LOCK_TIME):
+        if self._get_lock(key=self.update_lock):
 
             update_item, result = {}, []
 

cobweb/equip/single/launcher.py

@@ -171,7 +171,8 @@ def launcher(task):
                 target=spider.spider_task,
                 args=(
                     stop, func, item,
-                    redis_db.del_seed
+                    redis_db.del_seed,
+                    redis_db.add_seed
                 )
             ).start()
 

cobweb/equip/single/models.py

@@ -53,7 +53,7 @@ class Spider:
         self.storage = storage
         self.queue = queue
 
-    def spider_task(self, stop, func, item, del_seed):
+    def spider_task(self, stop, func, item, del_seed, add_seed):
         while not stop.is_set():
 
             seed = self.queue.pop()
@@ -72,6 +72,7 @@ class Spider:
 
                 store_queue = None
                 store_data = list()
+                add_seed_list = list()
                 iterators = func(item, seed)
 
                 if not isgenerator(iterators):
@@ -94,7 +95,7 @@ class Spider:
                             store_queue = it.queue()
                         store_data.append(it.struct_data)
                     elif isinstance(it, Seed):
-                        self.queue.push(it)
+                        add_seed_list.append(it)
 
                     elif isinstance(it, str) and it == DealModel.polling:
                         self.queue.push(seed)
@@ -111,10 +112,12 @@ class Spider:
                 if not status:
                     seed._retry += 1
                     self.queue.push(seed)
-
                 elif store_queue and store_data:
                     store_data.append(seed)
                     store_queue.push(store_data)
+                elif add_seed_list:
+                    del_seed(seed, spider_status=True)
+                    add_seed(add_seed_list)
 
             except Exception as e:
                 seed._retry += 1

cobweb/new.py

@@ -0,0 +1,20 @@
+
+
+
+
+class Launcher:
+
+    def __init__(self):
+        pass
+
+    def register(self, task_name, func):
+        pass
+
+    def launch(self, task_name):
+
+        def decorator(func):
+            # 注册爬虫程序
+            self.register(task_name, func)
+            return func
+
+        return decorator

cobweb/task.py

@@ -6,7 +6,6 @@ from .utils import parse_info, struct_start_seeds
 def init_task_env():
     Setting.RESET_SCORE = int(os.getenv("RESET_SCORE", 600))
     Setting.CHECK_LOCK_TIME = int(os.getenv("CHECK_LOCK_TIME", 30))
-    Setting.SCHEDULER_LOCK_TIME = int(os.getenv("SCHEDULER_LOCK_TIME", 15))
     Setting.DEAL_MODEL = os.getenv("DEAL_MODEL", DealModel.failure)
     Setting.LAUNCHER_MODEL = os.getenv("LAUNCHER_MODEL", LauncherModel.task)
     Setting.SCHEDULER_WAIT_TIME = float(os.getenv("SCHEDULER_WAIT_TIME", 5))

{cobweb_launcher-0.1.20.dist-info → cobweb_launcher-0.1.21.dist-info}/METADATA

@@ -1,19 +1,20 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 0.1.20
+Version: 0.1.21
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP
 Author-email: 2604868278@qq.com
 License: MIT
 Keywords: cobweb-launcher, cobweb
+Platform: UNKNOWN
 Classifier: Programming Language :: Python :: 3
 Requires-Python: >=3.7
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: requests >=2.19.1
-Requires-Dist: oss2 >=2.18.1
-Requires-Dist: redis >=4.4.4
+Requires-Dist: requests (>=2.19.1)
+Requires-Dist: oss2 (>=2.18.1)
+Requires-Dist: redis (>=4.4.4)
 Requires-Dist: aliyun-log-python-sdk
 
 # cobweb
@@ -43,3 +44,5 @@ need deal
 
 
 ![img.png](https://image-luyuan.oss-cn-hangzhou.aliyuncs.com/image/D2388CDC-B9E5-4CE4-9F2C-7D173763B6A8.png)
+
+

{cobweb_launcher-0.1.20.dist-info → cobweb_launcher-0.1.21.dist-info}/RECORD

@@ -1,15 +1,17 @@
 cobweb/__init__.py,sha256=zVims_awYxKNqULuKSjFXdZAaDJaqzFBJJo_SqeakMs,286
 cobweb/bbb.py,sha256=zKeCeBVFQfhEv6M8CCktUTM7tXDZmAu6ZN0-ET44pUY,5707
+cobweb/config.py,sha256=iWrep4vW9UyUi3hvgUWb4RL2IpEwpo_ttY0EUDKWN4g,5362
 cobweb/constant.py,sha256=oT2Y-qm2du2QGLA2D53aGi52Eda9FdV3hBomYRpbXl8,613
 cobweb/decorators.py,sha256=eYQI9rddPVJihAlomLTmbtQhIOzPw8dCrOFpxAq2pLY,318
 cobweb/interface.py,sha256=um_k2AAQl1HTOvfUlq914DjkpfZVwt2m1B65EpPKrmE,802
 cobweb/log.py,sha256=Gb3_y4IzTo5pJohTggBCU9rK6-ZN3hgTOHkoXHyN6CU,2384
+cobweb/new.py,sha256=-ZHoLZE4ZGRMBU5YWmWHMfLn2ipuTKEaCTO1pU2fX5o,303
 cobweb/setting.py,sha256=UAu_dLuIFYO98MxtlZ5sZqJcwKAUKq4Bu4KoKlV50Mc,288
-cobweb/task.py,sha256=wOGnVJ82BOiOwCmqhsoCcAgD6GAYaXodJvR4yRyyBAc,2082
+cobweb/task.py,sha256=SyWC43C7hqQAqH-1HECXEEgp_6L6lwDhYe1kZNnHUvA,2006
 cobweb/utils.py,sha256=ivmRqJJNtwdOKYT4G7qQCWnL8ar9c-shxeDZzGB2E9c,2651
 cobweb/db/__init__.py,sha256=jC-uOThYLtiDUG6cTJRkDITgOzR4nIOeaZQeZhuk-v0,139
 cobweb/db/oss_db.py,sha256=59HwMMfoGUr6l_BI4p84YHYoQwEn22cCM_1GZK34Uo4,4244
-cobweb/db/redis_db.py,sha256=PJqiEurv3oFB5JEflOqIJYgH8A2VoAPyaA6ZxlUabfE,8210
+cobweb/db/redis_db.py,sha256=go9IPQQZAl_jXiHDTruWF7N8svs9QEpQPOr7JhuLCQs,8179
 cobweb/db/scheduler/__init__.py,sha256=w5uIGEB1wLJ-H9RqGpzRwOEWW-BBVSk6Cc7FxZIlWCs,51
 cobweb/db/scheduler/default.py,sha256=XDtxNyu5KTpVAbfCOW8mR1zNFNHiMuaQ4sAhZuIYBoM,79
 cobweb/db/scheduler/textfile.py,sha256=P5pk75DUnbXbLNPOaMIbHh2lbwBGBlv0mitX58yK-MU,786
@@ -29,14 +31,14 @@ cobweb/equip/distributed/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZ
 cobweb/equip/distributed/launcher.py,sha256=1LzxibGXWR20XpXawakiRpEMaa9yfaj2rFSKnmEwjFc,7475
 cobweb/equip/distributed/models.py,sha256=qTGzxLdb2arsZSZK2HE4-MrqhraUhc2Ol5wBvlv_aWA,5008
 cobweb/equip/single/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-cobweb/equip/single/launcher.py,sha256=KRsw7yxklvFM85cel-EyLsNPLyrC9Hd26BMSx6-4Hac,6785
-cobweb/equip/single/models.py,sha256=VwaGgZXq72a48zsZja100Lj2GBHF1GkYTddTi1R-RxM,5422
+cobweb/equip/single/launcher.py,sha256=lZt4WNar0_QQjUSDhaJnxvHDiuQReXeFxquVgJjI5T4,6824
+cobweb/equip/single/models.py,sha256=uH6VXHiUSEOV0PN0s63NKRveTCRXy9_QH6qJRQgYv9Q,5610
 cobweb/single/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cobweb/single/launcher.py,sha256=IoJbn87j7t7Pib_FxoWZmmX8asXOqNGb-9ospw6EYJI,7302
 cobweb/single/models.py,sha256=wIEV35666lxdzqjDqBHPjjh-r6zD0x24rtQYz7d4Oxw,4332
 cobweb/single/nest.py,sha256=49K6KQ934INfPrWQsrq9rIFpQauLbLGOFbDaHvoQzOk,5015
-cobweb_launcher-0.1.20.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
-cobweb_launcher-0.1.20.dist-info/METADATA,sha256=OQv2x6SduQV3xYh5VvWtUKOcBVNi1rgbb4ukD4lXhAc,1220
-cobweb_launcher-0.1.20.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-cobweb_launcher-0.1.20.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
-cobweb_launcher-0.1.20.dist-info/RECORD,,
+cobweb_launcher-0.1.21.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
+cobweb_launcher-0.1.21.dist-info/METADATA,sha256=10RvainuQ8tnyG0H_wFa6NMBlRZmjfL-oVXLEFHDFB8,1246
+cobweb_launcher-0.1.21.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
+cobweb_launcher-0.1.21.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
+cobweb_launcher-0.1.21.dist-info/RECORD,,

{cobweb_launcher-0.1.20.dist-info → cobweb_launcher-0.1.21.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.43.0)
+Generator: bdist_wheel (0.37.0)
 Root-Is-Purelib: true
 Tag: py3-none-any