cobweb-launcher 0.0.5__tar.gz → 0.0.6__tar.gz

{cobweb-launcher-0.0.5 → cobweb-launcher-0.0.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 0.0.5
+Version: 0.0.6
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

cobweb-launcher-0.0.6/cobweb/__init__.py

@@ -0,0 +1,8 @@
+from .bbb import Seed, Queue, DBItem
+from .task import Task
+from .log import log
+from .interface import SchedulerInterface, StorerInterface
+from .db.redis_db import RedisDB
+from .distributed.launcher import launcher
+
+

{cobweb-launcher-0.0.5 → cobweb-launcher-0.0.6}/cobweb/db/redis_db.py

@@ -9,19 +9,9 @@ class RedisDB:
             self,
             project: str,
             task_name: str,
-            host=None,
-            port=None,
-            username=None,
-            password=None,
-            db=0
+            config: dict
     ):
-        pool = redis.ConnectionPool(
-            host=host,
-            port=port,
-            username=username,
-            password=password,
-            db=db
-        )
+        pool = redis.ConnectionPool(**config)
         self.heartbeat_key = f"{project}:{task_name}:heartbeat"  # redis type string
         self.spider_key = f"{project}:{task_name}:seed_info:spider"  # redis type zset, .format(priority)
         self.storer_key = f"{project}:{task_name}:seed_info:storer:%s"  # redis type set,

{cobweb-launcher-0.0.5 → cobweb-launcher-0.0.6}/cobweb/db/storer/loghub.py

@@ -5,8 +5,8 @@ from cobweb import log, StorerInterface
 
 class Loghub(StorerInterface):
 
-    def __init__(self, table, length, queue, config):
-        super().__init__(table, length, queue, config)
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
         self.client = None
 
     def init_loghub_clint(self):

{cobweb-launcher-0.0.5 → cobweb-launcher-0.0.6}/cobweb/distributed/launcher.py

@@ -2,10 +2,11 @@ import time
 import threading
 from threading import Thread
 from importlib import import_module
-from models import Scheduler, Spider, Storer
-from cobweb.utils import struct_queue_name, restore_table_name
-from cobweb import log, Queue, DBItem, RedisDB, StorerInterface
 
+from cobweb import log, Queue, DBItem, RedisDB, StorerInterface
+from cobweb.utils import struct_queue_name, restore_table_name
+from .models import Scheduler, Spider, Storer
+from collections import namedtuple
 
 # def start_seeds(seeds):
 #     if not seeds:
@@ -29,14 +30,14 @@ from cobweb import log, Queue, DBItem, RedisDB, StorerInterface
 #         storer_data[db_name]["db_args_list"].append(info[1:])
 #     return storer_data
 
-def get_storer_db(db):
+def get_scheduler_db(db):
     if isinstance(db, str):
         if "." in db:
             model_path = db.split(".")
             model = import_module(db)
             obj = getattr(model, db)
         else:
-            model = import_module(f"cobweb.db.storer.{db.lower()}")
+            model = import_module(f"cobweb.db.scheduler.{db.lower()}")
             obj = getattr(model, db.capitalize())
         return obj
         # if db.lower() in dir(StorerDB):
@@ -48,6 +49,25 @@ def get_storer_db(db):
     raise TypeError()
 
 
+def get_storer_db(db):
+    if isinstance(db, str):
+        if "." in db:
+            model_path = db.split(".")
+            model = import_module(db)
+            obj = getattr(model, db)
+        else:
+            model = import_module(f"cobweb.db.storer.{db.lower()}")
+            obj = getattr(model, db.capitalize())
+        return obj, db.lower()
+        # if db.lower() in dir(StorerDB):
+        #     return getattr(StorerDB, db)
+        # else:
+        #     pass
+    elif issubclass(db, StorerInterface):
+        return db, db.__name__.lower()
+    raise TypeError()
+
+
 def check(stop, last, spider, scheduler, storer_list, ready_seed_length, spider_queue_length):
     time.sleep(5)
     while True:
@@ -101,18 +121,22 @@ def launcher(task):
         stop = threading.Event()
 
         # 初始化redis信息
-        redis_db = RedisDB(task.project, task.task_name, *task.redis_info)
+        redis_db = RedisDB(task.project, task.task_name, task.redis_info)
 
         log.info("初始化cobweb!")
 
         seed_queue = Queue()
 
+        if task.scheduler_info is None:
+            task.scheduler_info = dict()
+
         # 调度器动态继承
         sql = task.scheduler_info.get("sql")
         table = task.scheduler_info.get("table")
         size = task.scheduler_info.get("size")
         scheduler_config = task.scheduler_info.get("config")
-        DB = task.scheduler_info["db"]
+        scheduler_db = task.scheduler_info.get("db", "default")
+        DB = get_scheduler_db(scheduler_db)
         # SchedulerDB, table, sql, length, size, config = task.scheduler_info
         SchedulerTmp = type(DB.__name__, (Scheduler, DB), {})
 
@@ -134,12 +158,12 @@ def launcher(task):
         item = type("Item", (object,), {"redis_client": redis_db})()
 
         for storer_info in storer_info_list:
-            db = storer_info["db"]
+            storer_db = storer_info["db"]
             fields = storer_info["fields"]
             storer_table = storer_info.get("table", "console")
             storer_config = storer_info.get("config")
 
-            StorerDB, db_name = get_storer_db(db)
+            StorerDB, db_name = get_storer_db(storer_db)
             StorerTmp = type(StorerDB.__name__, (Storer, StorerDB), {})
 
             if not getattr(item, db_name, None):
@@ -154,9 +178,9 @@ def launcher(task):
             # 初始话存储器
             table_name = restore_table_name(table_name=storer_table)
             storer = StorerTmp(
-                table=table_name, queue=queue,
+                table=table_name, fields=fields,
                 length=task.storer_queue_length,
-                config=storer_config
+                queue=queue, config=storer_config
             )
             storer_list.append(storer)
 

{cobweb-launcher-0.0.5 → cobweb-launcher-0.0.6}/cobweb_launcher.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 0.0.5
+Version: 0.0.6
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-0.0.5 → cobweb-launcher-0.0.6}/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
 
 setup(
     name="cobweb-launcher",
-    version="0.0.5",
+    version="0.0.6",
     packages=find_packages(),
     url="https://github.com/Juannie-PP/cobweb",
     license="MIT",

cobweb-launcher-0.0.5/cobweb/__init__.py

@@ -1,19 +0,0 @@
-# from .distributed.launcher import launcher
-# from .distributed import models
-from .bbb import Seed, Queue, DBItem
-from .task import Task
-from .log import log
-from .interface import SchedulerInterface, StorerInterface
-from .db.redis_db import RedisDB
-# from . import db
-# from .base.bbb import Seed
-# from .base import interface
-# from .base import config
-# from .base import utils
-# from .base import log as logger
-#
-# from .db.scheduler import *
-# from .db.storer import *
-# from .db.base import *
-
-