cobweb-launcher 0.0.4__py3-none-any.whl → 0.0.6__py3-none-any.whl

cobweb/__init__.py

@@ -1,10 +1,8 @@
-from .base.config import StorerInfo, SchedulerInfo, RedisInfo, SchedulerDB, StorerDB
-from .base.interface import StorerInterface, SchedulerInterface
+from .bbb import Seed, Queue, DBItem
+from .task import Task
+from .log import log
+from .interface import SchedulerInterface, StorerInterface
+from .db.redis_db import RedisDB
 from .distributed.launcher import launcher
-from .distributed import models
-from .base.task import Task
-from .base.bbb import Seed
-from .db.base import *
-from .db.storer import *
-from .db.scheduler import *
+
 

cobweb/{base/bbb.py → bbb.py}

@@ -2,8 +2,8 @@
 import json
 import time
 import hashlib
-from log import log
-from utils import struct_queue_name
+from .log import log
+from .utils import struct_queue_name
 from collections import deque, namedtuple
 
 

cobweb/db/__init__.py

@@ -0,0 +1,2 @@
+from . import oss_db, redis_db
+from . import scheduler, storer

cobweb/db/{base/oss_db.py → oss_db.py}

@@ -2,7 +2,7 @@ import oss2
 from typing import Union
 from oss2.models import PartInfo
 from requests import Response
-from base.log import log
+from cobweb import log
 
 
 class OssDB:

cobweb/db/{base/redis_db.py → redis_db.py}

@@ -1,6 +1,6 @@
 import time
 import redis
-from base.bbb import Seed
+from cobweb import Seed
 
 
 class RedisDB:
@@ -9,19 +9,9 @@ class RedisDB:
             self,
             project: str,
             task_name: str,
-            host=None,
-            port=None,
-            username=None,
-            password=None,
-            db=0
+            config: dict
     ):
-        pool = redis.ConnectionPool(
-            host=host,
-            port=port,
-            username=username,
-            password=password,
-            db=db
-        )
+        pool = redis.ConnectionPool(**config)
         self.heartbeat_key = f"{project}:{task_name}:heartbeat"  # redis type string
         self.spider_key = f"{project}:{task_name}:seed_info:spider"  # redis type zset, .format(priority)
         self.storer_key = f"{project}:{task_name}:seed_info:storer:%s"  # redis type set,

cobweb/db/scheduler/default.py

@@ -1,4 +1,4 @@
-from base.interface import SchedulerInterface
+from cobweb import SchedulerInterface
 
 
 class Default(SchedulerInterface):

cobweb/db/scheduler/textfile.py

@@ -1,6 +1,4 @@
-from base.log import log
-from base.bbb import Seed
-from base.interface import SchedulerInterface
+from cobweb import log, Seed, SchedulerInterface
 
 
 class Textfile(SchedulerInterface):

cobweb/db/storer/console.py

@@ -1,5 +1,4 @@
-from base.log import log
-from base.interface import StorerInterface
+from cobweb import log, StorerInterface
 
 
 class Console(StorerInterface):

cobweb/db/storer/loghub.py

@@ -1,13 +1,12 @@
 import json
-from base.log import log
-from base.interface import StorerInterface
 from aliyun.log import LogClient, LogItem, PutLogsRequest
+from cobweb import log, StorerInterface
 
 
 class Loghub(StorerInterface):
 
-    def __init__(self, table, fields, length, queue, config):
-        super().__init__(table, fields, length, queue, config)
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
         self.client = None
 
     def init_loghub_clint(self):

cobweb/db/storer/redis.py

@@ -1,5 +1,4 @@
-from base.log import log
-from base.interface import StorerInterface
+from cobweb import log, StorerInterface
 
 
 class Redis(StorerInterface):

cobweb/db/storer/textfile.py

@@ -1,5 +1,4 @@
-from base.log import log
-from base.interface import StorerInterface
+from cobweb import log, StorerInterface
 
 
 class Textfile(StorerInterface):

cobweb/distributed/launcher.py

@@ -1,34 +1,71 @@
 import time
 import threading
 from threading import Thread
-from base.log import log
-from db.base.redis_db import RedisDB
-from base.bbb import Queue, Seed, DBItem
-from base.utils import struct_queue_name, restore_table_name
-from models import Scheduler, Spider, Storer
-
-
-def start_seeds(seeds):
-    if not seeds:
-        return None
-    if any(isinstance(seeds, t) for t in (list, tuple)):
-        return [Seed(seed) for seed in seeds]
-    elif any(isinstance(seeds, t) for t in (str, dict)):
-        return Seed(seeds)
-
-
-def parse_storer_info(storer_info):
-    storer_data = {}
-    storer_info_list = []
-    if storer_info.__class__.__name__ == 'StorerInfo':
-        storer_info_list.append(storer_info)
-    elif any(isinstance(storer_info, t) for t in (list, tuple)):
-        storer_info_list = storer_info
-    for info in storer_info_list:
-        db_name = info.DB.__name__
-        storer_data.setdefault(db_name, {"StorerDB": info.DB, "db_args_list": []})
-        storer_data[db_name]["db_args_list"].append(info[1:])
-    return storer_data
+from importlib import import_module
+
+from cobweb import log, Queue, DBItem, RedisDB, StorerInterface
+from cobweb.utils import struct_queue_name, restore_table_name
+from .models import Scheduler, Spider, Storer
+from collections import namedtuple
+
+# def start_seeds(seeds):
+#     if not seeds:
+#         return None
+#     if any(isinstance(seeds, t) for t in (list, tuple)):
+#         return [Seed(seed) for seed in seeds]
+#     elif any(isinstance(seeds, t) for t in (str, dict)):
+#         return Seed(seeds)
+
+
+# def parse_storer_info(storer_info):
+#     storer_data = {}
+#     storer_info_list = []
+#     if storer_info.__class__.__name__ == 'StorerInfo':
+#         storer_info_list.append(storer_info)
+#     elif any(isinstance(storer_info, t) for t in (list, tuple)):
+#         storer_info_list = storer_info
+#     for info in storer_info_list:
+#         db_name = info.DB.__name__
+#         storer_data.setdefault(db_name, {"StorerDB": info.DB, "db_args_list": []})
+#         storer_data[db_name]["db_args_list"].append(info[1:])
+#     return storer_data
+
+def get_scheduler_db(db):
+    if isinstance(db, str):
+        if "." in db:
+            model_path = db.split(".")
+            model = import_module(db)
+            obj = getattr(model, db)
+        else:
+            model = import_module(f"cobweb.db.scheduler.{db.lower()}")
+            obj = getattr(model, db.capitalize())
+        return obj
+        # if db.lower() in dir(StorerDB):
+        #     return getattr(StorerDB, db)
+        # else:
+        #     pass
+    elif issubclass(db, StorerInterface):
+        return db
+    raise TypeError()
+
+
+def get_storer_db(db):
+    if isinstance(db, str):
+        if "." in db:
+            model_path = db.split(".")
+            model = import_module(db)
+            obj = getattr(model, db)
+        else:
+            model = import_module(f"cobweb.db.storer.{db.lower()}")
+            obj = getattr(model, db.capitalize())
+        return obj, db.lower()
+        # if db.lower() in dir(StorerDB):
+        #     return getattr(StorerDB, db)
+        # else:
+        #     pass
+    elif issubclass(db, StorerInterface):
+        return db, db.__name__.lower()
+    raise TypeError()
 
 
 def check(stop, last, spider, scheduler, storer_list, ready_seed_length, spider_queue_length):
@@ -84,54 +121,96 @@ def launcher(task):
         stop = threading.Event()
 
         # 初始化redis信息
-        redis_db = RedisDB(task.project, task.task_name, *task.redis_info)
+        redis_db = RedisDB(task.project, task.task_name, task.redis_info)
 
         log.info("初始化cobweb!")
 
         seed_queue = Queue()
 
+        if task.scheduler_info is None:
+            task.scheduler_info = dict()
+
         # 调度器动态继承
-        SchedulerDB, table, sql, length, size, config = task.scheduler_info
-        SchedulerTmp = type(SchedulerDB.__name__, (Scheduler, SchedulerDB), {})
+        sql = task.scheduler_info.get("sql")
+        table = task.scheduler_info.get("table")
+        size = task.scheduler_info.get("size")
+        scheduler_config = task.scheduler_info.get("config")
+        scheduler_db = task.scheduler_info.get("db", "default")
+        DB = get_scheduler_db(scheduler_db)
+        # SchedulerDB, table, sql, length, size, config = task.scheduler_info
+        SchedulerTmp = type(DB.__name__, (Scheduler, DB), {})
 
         # 初始化调度器
-        scheduler = SchedulerTmp(table, sql, length, size, seed_queue, config)
+        scheduler = SchedulerTmp(
+            table=table, sql=sql, size=size, queue=seed_queue,
+            length=task.scheduler_queue_length, config=scheduler_config
+        )
 
         # 初始化采集器
         spider = Spider(seed_queue, task.max_retries)
 
         # 解析存储器信息
-        storer_data = parse_storer_info(task.storer_info)
+        storer_info_list = task.storer_info
+        if not isinstance(storer_info_list, list):
+            storer_info_list = [storer_info_list]
 
         # new item
         item = type("Item", (object,), {"redis_client": redis_db})()
-        for db_name in storer_data.keys():
-            # 存储器动态继承
-            StorerDB = storer_data[db_name]["StorerDB"]
+
+        for storer_info in storer_info_list:
+            storer_db = storer_info["db"]
+            fields = storer_info["fields"]
+            storer_table = storer_info.get("table", "console")
+            storer_config = storer_info.get("config")
+
+            StorerDB, db_name = get_storer_db(storer_db)
             StorerTmp = type(StorerDB.__name__, (Storer, StorerDB), {})
-            db_args_list = storer_data[db_name]["db_args_list"]
-            for storer_db_args in db_args_list:
-                table, fields, length, config = storer_db_args
-                if not getattr(item, db_name, None):
-                    instance = type(db_name, (DBItem,), {})
-                    setattr(item, db_name, instance)
-                # 创建存储xxx, 创建存储队列
-                storer_item_instance = getattr(item, db_name)
-                storer_item_instance.init_item(table, fields)
-                #
-                storer_queue = struct_queue_name(db_name, table)
-                queue = getattr(storer_item_instance, storer_queue)
-                # 初始话存储器
-                table_name = restore_table_name(table_name=table)
-                storer = StorerTmp(table_name, fields, length, queue, config)
-                storer_list.append(storer)
+
+            if not getattr(item, db_name, None):
+                instance = type(db_name, (DBItem,), {})
+                setattr(item, db_name, instance)
+
+            storer_item_instance = getattr(item, db_name)
+            storer_item_instance.init_item(storer_table, fields)
+
+            storer_queue = struct_queue_name(db_name, storer_table)
+            queue = getattr(storer_item_instance, storer_queue)
+            # 初始话存储器
+            table_name = restore_table_name(table_name=storer_table)
+            storer = StorerTmp(
+                table=table_name, fields=fields,
+                length=task.storer_queue_length,
+                queue=queue, config=storer_config
+            )
+            storer_list.append(storer)
+
+        # for db_name in storer_data.keys():
+        #     # 存储器动态继承
+        #     StorerDB = storer_data[db_name]["StorerDB"]
+        #     StorerTmp = type(StorerDB.__name__, (Storer, StorerDB), {})
+        #     db_args_list = storer_data[db_name]["db_args_list"]
+        #     for storer_db_args in db_args_list:
+        #         table, fields, length, config = storer_db_args
+        #         if not getattr(item, db_name, None):
+        #             instance = type(db_name, (DBItem,), {})
+        #             setattr(item, db_name, instance)
+        #         # 创建存储xxx, 创建存储队列
+        #         storer_item_instance = getattr(item, db_name)
+        #         storer_item_instance.init_item(table, fields)
+        #         #
+        #         storer_queue = struct_queue_name(db_name, table)
+        #         queue = getattr(storer_item_instance, storer_queue)
+        #         # 初始话存储器
+        #         table_name = restore_table_name(table_name=table)
+        #         storer = StorerTmp(table_name, fields, length, queue, config)
+        #         storer_list.append(storer)
 
         Thread(target=redis_db.check_spider_queue, args=(stop, len(storer_list))).start()
         Thread(target=redis_db.set_heartbeat, args=(stop,)).start()
 
         # 推送初始种子
-        seeds = start_seeds(task.start_seed)
-        redis_db.add_seed(seeds)
+        # seeds = start_seeds(task.start_seed)
+        redis_db.add_seed(task.seeds)
         # 启动调度器, 调度至redis队列
         Thread(
             # name="xxxx_schedule_seeds",
@@ -190,5 +269,5 @@ def launcher(task):
     return decorator
 
 
-
-
+# model = get_storer_db("console")
+# print()

cobweb/distributed/models.py

@@ -1,8 +1,6 @@
 import time
 from hashlib import md5
-from base.log import log
-from base.bbb import Queue, Seed
-from base.interface import SchedulerInterface, StorerInterface
+from cobweb import log, Queue, Seed, StorerInterface, SchedulerInterface
 # from pympler import asizeof
 
 

cobweb/{base/interface.py → interface.py}

@@ -1,15 +1,5 @@
-import json
 from abc import ABC, abstractmethod
-
-
-def parse_config(config):
-    if not config:
-        return None
-    if isinstance(config, str):
-        return json.loads(config)
-    if isinstance(config, dict):
-        return config
-    raise TypeError("config type is not in [string, dict]!")
+from .utils import parse_info
 
 
 class SchedulerInterface(ABC):
@@ -20,7 +10,7 @@ class SchedulerInterface(ABC):
         self.length = length
         self.size = size
         self.queue = queue
-        self.config = parse_config(config)
+        self.config = parse_info(config)
         self.stop = False
 
     @abstractmethod
@@ -35,7 +25,7 @@ class StorerInterface(ABC):
         self.fields = fields
         self.length = length
         self.queue = queue
-        self.config = parse_config(config)
+        self.config = parse_info(config)
         # self.redis_db = redis_db
 
     @abstractmethod

cobweb/task.py

@@ -0,0 +1,44 @@
+from .utils import parse_info, struct_start_seeds
+
+
+class Task:
+
+    def __init__(
+            self,
+            seeds=None,
+            project=None,
+            task_name=None,
+            redis_info=None,
+            storer_info=None,
+            scheduler_info=None,
+            spider_num=None,
+            max_retries=None,
+            storer_queue_length=None,
+            scheduler_queue_length=None,
+    ):
+        """
+
+        :param seeds:
+        :param project:
+        :param task_name:
+        :param redis_info:
+        :param storer_info:
+        :param scheduler_info: dict(DB="", table="", size="", config="")
+        :param spider_num:
+        :param max_retries:
+        :param storer_queue_length:
+        :param scheduler_queue_length:
+        """
+        self.seeds = struct_start_seeds(seeds)
+        self.project = project or "test"
+        self.task_name = task_name or "spider"
+
+        self.redis_info = parse_info(redis_info)
+        self.storer_info = parse_info(storer_info)
+        self.scheduler_info = parse_info(scheduler_info)
+
+        self.spider_num = spider_num or 1
+        self.max_retries = max_retries or 5
+        self.storer_queue_length = storer_queue_length or 100
+        self.scheduler_queue_length = scheduler_queue_length or 100
+

cobweb/utils.py

@@ -0,0 +1,85 @@
+import json
+import sys
+from typing import Iterable
+
+# from cobweb import Seed
+
+
+def struct_table_name(table_name):
+    return table_name.replace(".", "__p__").replace(":", "__c__")
+
+
+def restore_table_name(table_name):
+    return table_name.replace("__p__", ".").replace("__c__", ":")
+
+
+def struct_queue_name(db_name, table_name):
+    return sys.intern(f"__{db_name}_{table_name}_queue__")
+
+
+# class StorerDB:
+#
+#     @staticmethod
+#     def console(self):
+#         from db.storer.console import Console
+#         table = struct_table_name(table)
+#         return StorerInfo(DB=Console, table=table, length=length, config=None)
+#
+#     @staticmethod
+#     def textfile(table, length=200):
+#         from db.storer.textfile import Textfile
+#         table = struct_table_name(table)
+#         return StorerInfo(DB=Textfile, table=table, length=length, config=None)
+#
+#     @staticmethod
+#     def loghub(table, length=200, config=None):
+#         from db.storer.loghub import Loghub
+#         table = struct_table_name(table)
+#         return StorerInfo(DB=Loghub, table=table, length=length, config=config)
+
+
+def parse_info(info):
+    if not info:
+        return info
+
+    if isinstance(info, dict):
+        return info
+
+    if isinstance(info, str):
+        return json.loads(info)
+
+    if isinstance(info, Iterable):
+        result = list()
+        for ii in info:
+            if isinstance(ii, str):
+                result.append(json.loads(ii))
+            elif isinstance(ii, dict):
+                result.append(ii)
+            else:
+                raise TypeError("must be in [str, dict]")
+
+        return result
+
+
+def struct_start_seeds(seeds):
+    from .bbb import Seed
+    if not seeds:
+        return None
+    if any(isinstance(seeds, t) for t in (list, tuple)):
+        return [Seed(seed) for seed in seeds]
+    elif any(isinstance(seeds, t) for t in (str, dict)):
+        return Seed(seeds)
+
+
+# def get_storer_db(db):
+#
+#     if isinstance(db, str):
+#         model = import_module(f" db.storer.{db.lower()}")
+#
+#         # if db.lower() in dir(StorerDB):
+#         #     return getattr(StorerDB, db)
+#         # else:
+#         #     pass
+
+
+

{cobweb_launcher-0.0.4.dist-info → cobweb_launcher-0.0.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 0.0.4
+Version: 0.0.6
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

cobweb_launcher-0.0.6.dist-info/RECORD

@@ -0,0 +1,28 @@
+cobweb/__init__.py,sha256=IKQkcts73m_K-b6TGs_IxkdkncnZRU0O_N0MJct2COI,218
+cobweb/bbb.py,sha256=8croagwF3xVdb1Xq3f5g22WydQ-pHHWWnA50895dH3E,5617
+cobweb/interface.py,sha256=um_k2AAQl1HTOvfUlq914DjkpfZVwt2m1B65EpPKrmE,802
+cobweb/log.py,sha256=Gb3_y4IzTo5pJohTggBCU9rK6-ZN3hgTOHkoXHyN6CU,2384
+cobweb/task.py,sha256=awZWFwON34WAJs08TKPaYTbyRjmoNOCBkLCNf2l9C-Q,1282
+cobweb/utils.py,sha256=tHSj_T1Ct7Y-QdIo5w4hCwkO59RlQiq9yGxUOjojMOg,2158
+cobweb/db/__init__.py,sha256=4m9lqmxZCRbaih3Z3rl_BT0GugMd0dkOIgu_P9aeC84,63
+cobweb/db/oss_db.py,sha256=l-Xbqawg1HJgedz9MumXQrr1jMK6_EePXCis11CZEkE,3810
+cobweb/db/redis_db.py,sha256=keVlFpUT7spfNwZ4g_5teROo_uOsjfDWtR-WvAcqZIE,7415
+cobweb/db/scheduler/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+cobweb/db/scheduler/default.py,sha256=OxmFX7OvMEhKEq-NF7A8I9cA4V4qWw5vayS-yIbng0A,114
+cobweb/db/scheduler/textfile.py,sha256=atRDeNT-e5toNvyGsCXAxL1FJi77uSYktdCzH_hXGo8,821
+cobweb/db/storer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+cobweb/db/storer/console.py,sha256=f7yZFo4qTieaB9JxbGfrVAclAb2H_wji82dWoZp7HUw,182
+cobweb/db/storer/loghub.py,sha256=4VqZacXWhidzINHXQu2_-E0HOBRCcc86f6LkKfnXD5I,1731
+cobweb/db/storer/redis.py,sha256=7Q2XEQwBL6X_M1uvxzzuSBt6iw9piKw-_FWKm2INZDQ,412
+cobweb/db/storer/textfile.py,sha256=3mDHMvF6Sh5fn3IHzWQxyTUd45V-zUoH8vY3EoRlMx0,415
+cobweb/distributed/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+cobweb/distributed/launcher.py,sha256=WEEfZDdOXpEN5ljrYEGQWzPtDM7XcVczEnDO81yCoWg,9135
+cobweb/distributed/models.py,sha256=zRZfSOiP-OyvTVdI5_KScyf_jZmYmaYbxJvohf7ffDA,4390
+cobweb/single/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+cobweb/single/models.py,sha256=lu8teNWnWcUwZFra8XmqyhzOAf3UyuEztwBr1Ne6pUs,2898
+cobweb/single/nest.py,sha256=mL8q9a5BjtoeUyzXCIVw_vyUsNY8ltbvQpYIIpZEDFU,5012
+cobweb_launcher-0.0.6.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
+cobweb_launcher-0.0.6.dist-info/METADATA,sha256=TDawOhbZk0ugagG8ZitV2rv_3iVjaKDu7S_847R6VH4,1225
+cobweb_launcher-0.0.6.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
+cobweb_launcher-0.0.6.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
+cobweb_launcher-0.0.6.dist-info/RECORD,,