cobweb-launcher 0.0.5__py3-none-any.whl → 0.0.7__py3-none-any.whl

cobweb/__init__.py

@@ -1,19 +1,8 @@
-# from .distributed.launcher import launcher
-# from .distributed import models
 from .bbb import Seed, Queue, DBItem
 from .task import Task
 from .log import log
 from .interface import SchedulerInterface, StorerInterface
 from .db.redis_db import RedisDB
-# from . import db
-# from .base.bbb import Seed
-# from .base import interface
-# from .base import config
-# from .base import utils
-# from .base import log as logger
-#
-# from .db.scheduler import *
-# from .db.storer import *
-# from .db.base import *
+from .distributed.launcher import launcher
 
 

cobweb/bbb.py

@@ -60,7 +60,7 @@ class Seed:
             self,
             seed_info=None,
             priority=300,
-            version=None,
+            version=0,
             retry=0,
             **kwargs
     ):

cobweb/db/redis_db.py

@@ -9,19 +9,9 @@ class RedisDB:
             self,
             project: str,
             task_name: str,
-            host=None,
-            port=None,
-            username=None,
-            password=None,
-            db=0
+            config: dict
     ):
-        pool = redis.ConnectionPool(
-            host=host,
-            port=port,
-            username=username,
-            password=password,
-            db=db
-        )
+        pool = redis.ConnectionPool(**config)
         self.heartbeat_key = f"{project}:{task_name}:heartbeat"  # redis type string
         self.spider_key = f"{project}:{task_name}:seed_info:spider"  # redis type zset, .format(priority)
         self.storer_key = f"{project}:{task_name}:seed_info:storer:%s"  # redis type set,

cobweb/db/storer/loghub.py

@@ -5,8 +5,8 @@ from cobweb import log, StorerInterface
 
 class Loghub(StorerInterface):
 
-    def __init__(self, table, length, queue, config):
-        super().__init__(table, length, queue, config)
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
         self.client = None
 
     def init_loghub_clint(self):

cobweb/distributed/launcher.py

@@ -2,10 +2,11 @@ import time
 import threading
 from threading import Thread
 from importlib import import_module
-from models import Scheduler, Spider, Storer
-from cobweb.utils import struct_queue_name, restore_table_name
-from cobweb import log, Queue, DBItem, RedisDB, StorerInterface
 
+from cobweb import log, Queue, DBItem, RedisDB, StorerInterface
+from cobweb.utils import struct_queue_name, restore_table_name
+from .models import Scheduler, Spider, Storer
+from collections import namedtuple
 
 # def start_seeds(seeds):
 #     if not seeds:
@@ -29,14 +30,14 @@ from cobweb import log, Queue, DBItem, RedisDB, StorerInterface
 #         storer_data[db_name]["db_args_list"].append(info[1:])
 #     return storer_data
 
-def get_storer_db(db):
+def get_scheduler_db(db):
     if isinstance(db, str):
         if "." in db:
             model_path = db.split(".")
             model = import_module(db)
             obj = getattr(model, db)
         else:
-            model = import_module(f"cobweb.db.storer.{db.lower()}")
+            model = import_module(f"cobweb.db.scheduler.{db.lower()}")
             obj = getattr(model, db.capitalize())
         return obj
         # if db.lower() in dir(StorerDB):
@@ -48,6 +49,25 @@ def get_storer_db(db):
     raise TypeError()
 
 
+def get_storer_db(db):
+    if isinstance(db, str):
+        if "." in db:
+            model_path = db.split(".")
+            model = import_module(db)
+            obj = getattr(model, db)
+        else:
+            model = import_module(f"cobweb.db.storer.{db.lower()}")
+            obj = getattr(model, db.capitalize())
+        return obj, db.lower()
+        # if db.lower() in dir(StorerDB):
+        #     return getattr(StorerDB, db)
+        # else:
+        #     pass
+    elif issubclass(db, StorerInterface):
+        return db, db.__name__.lower()
+    raise TypeError()
+
+
 def check(stop, last, spider, scheduler, storer_list, ready_seed_length, spider_queue_length):
     time.sleep(5)
     while True:
@@ -101,18 +121,22 @@ def launcher(task):
         stop = threading.Event()
 
         # 初始化redis信息
-        redis_db = RedisDB(task.project, task.task_name, *task.redis_info)
+        redis_db = RedisDB(task.project, task.task_name, task.redis_info)
 
         log.info("初始化cobweb!")
 
         seed_queue = Queue()
 
+        if task.scheduler_info is None:
+            task.scheduler_info = dict()
+
         # 调度器动态继承
         sql = task.scheduler_info.get("sql")
         table = task.scheduler_info.get("table")
         size = task.scheduler_info.get("size")
         scheduler_config = task.scheduler_info.get("config")
-        DB = task.scheduler_info["db"]
+        scheduler_db = task.scheduler_info.get("db", "default")
+        DB = get_scheduler_db(scheduler_db)
         # SchedulerDB, table, sql, length, size, config = task.scheduler_info
         SchedulerTmp = type(DB.__name__, (Scheduler, DB), {})
 
@@ -134,12 +158,12 @@ def launcher(task):
         item = type("Item", (object,), {"redis_client": redis_db})()
 
         for storer_info in storer_info_list:
-            db = storer_info["db"]
+            storer_db = storer_info["db"]
             fields = storer_info["fields"]
             storer_table = storer_info.get("table", "console")
             storer_config = storer_info.get("config")
 
-            StorerDB, db_name = get_storer_db(db)
+            StorerDB, db_name = get_storer_db(storer_db)
             StorerTmp = type(StorerDB.__name__, (Storer, StorerDB), {})
 
             if not getattr(item, db_name, None):
@@ -154,9 +178,9 @@ def launcher(task):
             # 初始话存储器
             table_name = restore_table_name(table_name=storer_table)
             storer = StorerTmp(
-                table=table_name, queue=queue,
+                table=table_name, fields=fields,
                 length=task.storer_queue_length,
-                config=storer_config
+                queue=queue, config=storer_config
             )
             storer_list.append(storer)
 

{cobweb_launcher-0.0.5.dist-info → cobweb_launcher-0.0.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 0.0.5
+Version: 0.0.7
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb_launcher-0.0.5.dist-info → cobweb_launcher-0.0.7.dist-info}/RECORD

@@ -1,28 +1,28 @@
-cobweb/__init__.py,sha256=tTpeU8vMuM_yDytUdNuqVpVSmkSMgRb2nCZ8bFAYLOw,503
-cobweb/bbb.py,sha256=8croagwF3xVdb1Xq3f5g22WydQ-pHHWWnA50895dH3E,5617
+cobweb/__init__.py,sha256=IKQkcts73m_K-b6TGs_IxkdkncnZRU0O_N0MJct2COI,218
+cobweb/bbb.py,sha256=DWqnCiBhzD4UzPxgDHo4mhvYIB5JnYFrAtpF_vcTtT0,5614
 cobweb/interface.py,sha256=um_k2AAQl1HTOvfUlq914DjkpfZVwt2m1B65EpPKrmE,802
 cobweb/log.py,sha256=Gb3_y4IzTo5pJohTggBCU9rK6-ZN3hgTOHkoXHyN6CU,2384
 cobweb/task.py,sha256=awZWFwON34WAJs08TKPaYTbyRjmoNOCBkLCNf2l9C-Q,1282
 cobweb/utils.py,sha256=tHSj_T1Ct7Y-QdIo5w4hCwkO59RlQiq9yGxUOjojMOg,2158
 cobweb/db/__init__.py,sha256=4m9lqmxZCRbaih3Z3rl_BT0GugMd0dkOIgu_P9aeC84,63
 cobweb/db/oss_db.py,sha256=l-Xbqawg1HJgedz9MumXQrr1jMK6_EePXCis11CZEkE,3810
-cobweb/db/redis_db.py,sha256=pezhpWM6KOFGlHX9zD3VAYMVV3mnOieSZI5Xcr7vK3I,7634
+cobweb/db/redis_db.py,sha256=keVlFpUT7spfNwZ4g_5teROo_uOsjfDWtR-WvAcqZIE,7415
 cobweb/db/scheduler/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cobweb/db/scheduler/default.py,sha256=OxmFX7OvMEhKEq-NF7A8I9cA4V4qWw5vayS-yIbng0A,114
 cobweb/db/scheduler/textfile.py,sha256=atRDeNT-e5toNvyGsCXAxL1FJi77uSYktdCzH_hXGo8,821
 cobweb/db/storer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cobweb/db/storer/console.py,sha256=f7yZFo4qTieaB9JxbGfrVAclAb2H_wji82dWoZp7HUw,182
-cobweb/db/storer/loghub.py,sha256=SxKnI4B7cn-Uws91WyjFXessbBwA3o1-pWTcMllBDac,1771
+cobweb/db/storer/loghub.py,sha256=4VqZacXWhidzINHXQu2_-E0HOBRCcc86f6LkKfnXD5I,1731
 cobweb/db/storer/redis.py,sha256=7Q2XEQwBL6X_M1uvxzzuSBt6iw9piKw-_FWKm2INZDQ,412
 cobweb/db/storer/textfile.py,sha256=3mDHMvF6Sh5fn3IHzWQxyTUd45V-zUoH8vY3EoRlMx0,415
 cobweb/distributed/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-cobweb/distributed/launcher.py,sha256=UmDYpl3PWUzoyoWBL68AErmnIO9VrWHohy73Q4Nz4r8,8339
+cobweb/distributed/launcher.py,sha256=WEEfZDdOXpEN5ljrYEGQWzPtDM7XcVczEnDO81yCoWg,9135
 cobweb/distributed/models.py,sha256=zRZfSOiP-OyvTVdI5_KScyf_jZmYmaYbxJvohf7ffDA,4390
 cobweb/single/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cobweb/single/models.py,sha256=lu8teNWnWcUwZFra8XmqyhzOAf3UyuEztwBr1Ne6pUs,2898
 cobweb/single/nest.py,sha256=mL8q9a5BjtoeUyzXCIVw_vyUsNY8ltbvQpYIIpZEDFU,5012
-cobweb_launcher-0.0.5.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
-cobweb_launcher-0.0.5.dist-info/METADATA,sha256=Gym0QTYBbJfcZTxe3yuFLR94jwy1HYYV2c2yvnv2FNk,1225
-cobweb_launcher-0.0.5.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
-cobweb_launcher-0.0.5.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
-cobweb_launcher-0.0.5.dist-info/RECORD,,
+cobweb_launcher-0.0.7.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
+cobweb_launcher-0.0.7.dist-info/METADATA,sha256=ymYuybr2CYckZMJVx6diBVimAeGlu3X8OvkuxNeveRQ,1225
+cobweb_launcher-0.0.7.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
+cobweb_launcher-0.0.7.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
+cobweb_launcher-0.0.7.dist-info/RECORD,,