cobweb-launcher 0.1.8__tar.gz → 0.1.9__tar.gz

{cobweb-launcher-0.1.8 → cobweb-launcher-0.1.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 0.1.8
+Version: 0.1.9
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP
@@ -11,6 +11,10 @@ Classifier: Programming Language :: Python :: 3
 Requires-Python: >=3.7
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: requests>=2.19.1
+Requires-Dist: oss2>=2.18.1
+Requires-Dist: redis>=4.4.4
+Requires-Dist: aliyun-log-python-sdk
 
 # cobweb
 

cobweb-launcher-0.1.9/cobweb/__init__.py

@@ -0,0 +1,7 @@
+from .bbb import Seed, Queue, DBItem
+from .task import Task
+from .log import log
+from .db.redis_db import RedisDB
+
+from .equip.distributed.launcher import launcher
+from .equip.single.launcher import launcher as single_launcher

cobweb-launcher-0.1.9/cobweb/constant.py

@@ -0,0 +1,24 @@
+
+
+class LauncherModel:
+    task = "launcher model: task"
+    resident = "launcher model: resident"
+
+
+class LogModel:
+    simple = "log model: simple"
+    common = "log model: common"
+    detailed = "log model: detailed"
+
+
+class DealModel:
+    failure = "deal model: failure"
+    success = "deal model: success"
+    polling = "deal model: polling"
+
+
+class Setting:
+    RESET_SCORE = None
+    CHECK_LOCK_TIME = None
+    DEAL_MODEL = None
+    LAUNCHER_MODEL = None

cobweb-launcher-0.1.9/cobweb/db/__init__.py

@@ -0,0 +1,3 @@
+from .. import log, Seed, decorators
+from ..constant import Setting, DealModel
+from ..interface import SchedulerInterface, StorerInterface

{cobweb-launcher-0.1.8 → cobweb-launcher-0.1.9}/cobweb/db/redis_db.py

@@ -1,7 +1,8 @@
 import time
 import redis
-from cobweb import Seed, log
-from cobweb.decorators import check_redis_status
+from . import log, decorators, Seed, Setting, DealModel
+# from cobweb.decorators import decorators.check_redis_status
+# from cobweb.constant import Setting, DealModel
 
 
 class RedisDB:
@@ -11,9 +12,6 @@ class RedisDB:
             project: str,
             task_name: str,
             config: dict,
-            model: int,
-            cs_lct: int,
-            rs_time: int,
     ):
         pool = redis.ConnectionPool(**config)
         self.heartbeat_key = f"{project}:{task_name}:heartbeat"  # redis type string
@@ -25,11 +23,8 @@ class RedisDB:
         self.check_lock = f"{project}:{task_name}:check_seed_lock"  # redis type string
         self.scheduler_lock = f"{project}:{task_name}:scheduler_lock"  # redis type string
         self.client = redis.Redis(connection_pool=pool)
-        self.model = model
-        self.cs_lct = cs_lct
-        self.rs_time = rs_time
 
-    @check_redis_status
+    @decorators.check_redis_status
     def _get_lock(self, key, t=15, timeout=3, sleep_time=0.1):
         begin_time = int(time.time())
         while True:
@@ -55,7 +50,7 @@ class RedisDB:
             log.info("ttl: " + str(ttl))
             return False
 
-    @check_redis_status
+    @decorators.check_redis_status
     def _deal_seed(self, seeds, is_add: bool):
         if not seeds:
             return None
@@ -73,15 +68,15 @@ class RedisDB:
         if item_info:
             self.client.zadd(self.spider_key, mapping=item_info, nx=is_add, xx=not is_add)
 
-    @check_redis_status
+    @decorators.check_redis_status
     def add_seed(self, seeds):
         self._deal_seed(seeds, is_add=True)
 
-    @check_redis_status
+    @decorators.check_redis_status
     def reset_seed(self, seeds):
         self._deal_seed(seeds, is_add=False)
 
-    @check_redis_status
+    @decorators.check_redis_status
     def del_seed(self, seeds, spider_status: bool = True):
         if not seeds:
             return None
@@ -92,18 +87,16 @@ class RedisDB:
         seeds = [seed if isinstance(seed, Seed) else Seed(seed) for seed in seeds]
 
         if seeds:
-            # redis_key = self.succeed_key if spider_status else self.failed_key
             redis_key = None
-            if spider_status:
-                if isinstance(self.model, int) and self.model == 2:
-                    redis_key = self.succeed_key
-            else:
+            if spider_status and Setting.DEAL_MODEL in [DealModel.success, DealModel.polling]:
+                redis_key = self.succeed_key
+            elif not spider_status:
                 redis_key = self.failed_key
             if redis_key:
                 self.client.sadd(redis_key, *(str(seed) for seed in seeds))
             self.client.zrem(self.spider_key, *(seed.format_seed for seed in seeds))
 
-    @check_redis_status
+    @decorators.check_redis_status
     def set_storer(self, key, seeds):
         if not seeds:
             return None
@@ -122,7 +115,7 @@ class RedisDB:
             self.client.zadd(self.storer_key % key, mapping=item_info)
             log.info(f"zadd storer key: length {len(item_info.keys())}")
 
-    @check_redis_status
+    @decorators.check_redis_status
     def get_seed(self, length: int = 200):
         cs = time.time()
 
@@ -148,14 +141,14 @@ class RedisDB:
             log.info("push seeds into queue time: " + str(time.time() - cs))
             return result
 
-    @check_redis_status
+    @decorators.check_redis_status
     def check_spider_queue(self, stop, storer_num):
         while not stop.is_set():
             # 每15s获取check锁,等待600s后仍获取不到锁则重试;获取到锁后，设置锁的存活时间为${cs_lct}s
-            if self._get_lock(key=self.check_lock, t=self.cs_lct, timeout=600, sleep_time=3):
+            if self._get_lock(key=self.check_lock, t=Setting.CHECK_LOCK_TIME, timeout=600, sleep_time=3):
                 heartbeat = True if self.client.exists(self.heartbeat_key) else False
                 # 重启重制score值，否则获取${rs_time}分钟前的分数值
-                score = -int(time.time()) + self.rs_time if heartbeat else "-inf"
+                score = -int(time.time()) + Setting.RESET_SCORE if heartbeat else "-inf"
 
                 keys = self.client.keys(self.storer_key % "*")
 
@@ -170,7 +163,7 @@ class RedisDB:
                             break
                         for key in keys:
                             self.client.zrem(key, *members)
-                        if self.model == 2:
+                        if Setting.DEAL_MODEL in [DealModel.success, DealModel.polling]:
                             self.client.sadd(self.succeed_key, *members)
                         self.client.zrem(self.spider_key, *members)
                         self.client.zrem(intersection_key, *members)
@@ -193,31 +186,28 @@ class RedisDB:
                 if not heartbeat:
                     self.client.setex(self.heartbeat_key, 15, "")
 
-                # self.client.delete(self.check_lock)
-                # time.sleep(3)
-
-    @check_redis_status
+    @decorators.check_redis_status
     def set_heartbeat(self, stop):
         time.sleep(5)
         while not stop.is_set():
             self.client.setex(self.heartbeat_key, 5, "")
             time.sleep(3)
 
-    # @check_redis_status
+    # @decorators.check_redis_status
     # def heartbeat(self):
     #     """
     #     返回心跳key剩余存活时间
     #     """
     #     return self.client.ttl(self.heartbeat_key)
 
-    @check_redis_status
+    @decorators.check_redis_status
     def spider_queue_length(self):
         return self.client.zcard(self.spider_key)
 
-    @check_redis_status
+    @decorators.check_redis_status
     def ready_seed_length(self):
         return self.client.zcount(self.spider_key, min=0, max="+inf")
 
-    @check_redis_status
+    @decorators.check_redis_status
     def get_scheduler_lock(self):
         return self._get_lock(self.scheduler_lock)

cobweb-launcher-0.1.9/cobweb/db/scheduler/__init__.py

@@ -0,0 +1 @@
+from .. import log, Seed, SchedulerInterface as Inf

cobweb-launcher-0.1.9/cobweb/db/scheduler/default.py

@@ -0,0 +1,8 @@
+from . import Inf
+
+
+class Default(Inf):
+
+    def schedule(self):
+        pass
+

{cobweb-launcher-0.1.8 → cobweb-launcher-0.1.9}/cobweb/db/scheduler/textfile.py

@@ -1,7 +1,7 @@
-from cobweb import log, Seed, SchedulerInterface
+from . import Inf, log, Seed
 
 
-class Textfile(SchedulerInterface):
+class Textfile(Inf):
 
     index = None
 

cobweb-launcher-0.1.9/cobweb/db/storer/__init__.py

@@ -0,0 +1 @@
+from .. import log, Seed, StorerInterface as Inf

{cobweb-launcher-0.1.8 → cobweb-launcher-0.1.9}/cobweb/db/storer/console.py

@@ -1,7 +1,7 @@
-from cobweb import log, StorerInterface
+from . import Inf, log
 
 
-class Console(StorerInterface):
+class Console(Inf):
 
     def store(self, data_list):
         for item in data_list:

{cobweb-launcher-0.1.8 → cobweb-launcher-0.1.9}/cobweb/db/storer/loghub.py

@@ -1,9 +1,9 @@
 import json
+from . import Inf, log
 from aliyun.log import LogClient, LogItem, PutLogsRequest
-from cobweb import log, StorerInterface
 
 
-class Loghub(StorerInterface):
+class Loghub(Inf):
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)

cobweb-launcher-0.1.8/cobweb/db/storer/redis.py → cobweb-launcher-0.1.9/cobweb/db/storer/textfile.py

@@ -1,7 +1,7 @@
-from cobweb import log, StorerInterface
+from . import Inf, log
 
 
-class Redis(StorerInterface):
+class Textfile(Inf):
 
     def store(self, data_list):
         try:

{cobweb-launcher-0.1.8 → cobweb-launcher-0.1.9}/cobweb/decorators.py

@@ -1,5 +1,5 @@
 from functools import wraps
-from cobweb import log
+from .log import log
 
 
 def check_redis_status(func):

cobweb-launcher-0.1.9/cobweb/equip/__init__.py

@@ -0,0 +1,8 @@
+from .. import Queue, DBItem, RedisDB, Seed, log
+from ..constant import Setting, DealModel
+from utils import (
+    struct_queue_name as sqn,
+    restore_table_name as rtn,
+    parse_import_model as pim,
+    issubclass_cobweb_inf as ici
+)

{cobweb-launcher-0.1.8/cobweb → cobweb-launcher-0.1.9/cobweb/equip}/distributed/launcher.py

@@ -1,15 +1,9 @@
 import time
 import threading
-from threading import Thread
 
+from .. import log, sqn, rtn, pim
+from .. import Queue, DBItem, RedisDB, Setting
 from .models import Scheduler, Spider, Storer
-from cobweb import log, Queue, DBItem, RedisDB
-from cobweb.setting import MODEL, RESET_SCORE, CHECK_LOCK_TIME
-from cobweb.utils import (
-    struct_queue_name as sqn,
-    restore_table_name as rtn,
-    parse_import_model as pim,
-)
 
 
 def check(stop, last, spider, scheduler, storer_list, ready_seed_length, spider_queue_length):
@@ -37,11 +31,10 @@ def check(stop, last, spider, scheduler, storer_list, ready_seed_length, spider_
             )
         if (
                 scheduler.stop and
-                # not redis_ready_seed_length and
                 not memory_seed_queue_length and
                 not running_spider_thread_num
         ):
-            if not MODEL:
+            if not Setting.LAUNCHER_MODEL:
                 log.info("spider is done?")
             last.set()
             time.sleep(3)
@@ -58,7 +51,7 @@ def check(stop, last, spider, scheduler, storer_list, ready_seed_length, spider_
                     not redis_ready_seed_length and
                     not redis_spider_seed_length
             ):
-                if MODEL:
+                if Setting.LAUNCHER_MODEL:
                     log.info("waiting for push seeds...")
                     status = "waiting"
                     time.sleep(30)
@@ -89,20 +82,6 @@ def launcher(task):
     :param task: 任务配置信息
     """
     def decorator(func):
-        """
-        Item:
-            Textfile()
-            Loghub()
-            Console()
-        e.g.
-        task.fields = "a,b"
-        func(item, seed)
-            a = "a"
-            b = "b"
-            data = {"a": "a", "b": "b"}
-            yield item.Loghub(**data)
-            yield item.Loghub(a=a, b=b)
-        """
         storer_list = []
 
         # 程序结束事件
@@ -111,10 +90,7 @@ def launcher(task):
         stop = threading.Event()
 
         # 初始化redis信息
-        redis_db = RedisDB(
-            task.project, task.task_name, task.redis_info,
-            model=MODEL, cs_lct=CHECK_LOCK_TIME, rs_time=RESET_SCORE
-        )
+        redis_db = RedisDB(task.project, task.task_name, task.redis_info)
 
         log.info("初始化cobweb!")
 
@@ -139,9 +115,6 @@ def launcher(task):
             length=task.scheduler_queue_length, config=scheduler_config
         )
 
-        # 初始化采集器
-        spider = Spider(seed_queue, task.max_retries)
-
         # 解析存储器信息
         storer_info_list = task.storer_info or []
         if not isinstance(storer_info_list, list):
@@ -178,14 +151,17 @@ def launcher(task):
             )
             storer_list.append(storer)
 
-        Thread(target=redis_db.check_spider_queue, args=(stop, len(storer_list))).start()
-        Thread(target=redis_db.set_heartbeat, args=(stop,)).start()
+        # 初始化采集器
+        spider = Spider(seed_queue, storer_list and True, task.max_retries)
+
+        threading.Thread(target=redis_db.check_spider_queue, args=(stop, len(storer_list))).start()
+        threading.Thread(target=redis_db.set_heartbeat, args=(stop,)).start()
 
         # 推送初始种子
         # seeds = start_seeds(task.start_seed)
         redis_db.add_seed(task.seeds)
         # 启动调度器, 调度至redis队列
-        Thread(
+        threading.Thread(
             # name="xxxx_schedule_seeds",
             target=scheduler.schedule_seed,
             args=(
@@ -196,7 +172,7 @@ def launcher(task):
         ).start()
 
         # 启动调度器, 调度任务队列
-        Thread(
+        threading.Thread(
             # name="xxxx_schedule_task",
             target=scheduler.schedule_task,
             args=(
@@ -207,7 +183,7 @@ def launcher(task):
 
         # 启动采集器
         for index in range(task.spider_num):
-            Thread(
+            threading.Thread(
                 # name=f"xxxx_spider_task:{index}",
                 target=spider.spider_task,
                 args=(
@@ -218,7 +194,7 @@ def launcher(task):
 
         # 启动存储器
         for storer in storer_list:
-            Thread(
+            threading.Thread(
                 # name=f"xxxx_store_task:{storer.table}",
                 target=storer.store_task,
                 args=(
@@ -228,7 +204,7 @@ def launcher(task):
                 )
             ).start()
 
-        Thread(
+        threading.Thread(
             # name="check_spider",
             target=check,
             args=(

{cobweb-launcher-0.1.8/cobweb → cobweb-launcher-0.1.9/cobweb/equip}/distributed/models.py

@@ -1,8 +1,9 @@
 import time
 from hashlib import md5
-from cobweb import log, Queue, Seed
-from cobweb.utils import issubclass_cobweb_inf
+from inspect import isgenerator
 
+from .. import log, ici
+from .. import DealModel, Queue, Seed
 # from pympler import asizeof
 
 
@@ -11,7 +12,7 @@ class Scheduler:
     def schedule_seed(self, ready_seed_length, get_scheduler_lock, add_seed):
 
         inf_name = "SchedulerInterface"
-        if not issubclass_cobweb_inf(self.__class__, inf_name):
+        if not ici(self.__class__, inf_name):
             raise Exception("not have schedule function!")
 
         if self.__class__.__name__ == "Default":
@@ -48,9 +49,10 @@ class Scheduler:
 
 class Spider:
 
-    def __init__(self, queue, max_retries=5):
+    def __init__(self, queue, storage, max_retries=5):
         self.spider_in_progress = Queue()
         self.max_retries = max_retries
+        self.storage = storage
         self.queue = queue
 
     def spider_task(self, stop, func, item, del_seed):
@@ -65,33 +67,42 @@ class Spider:
             try:
                 self.spider_in_progress.push(1, direct_insertion=True)
                 # log.info("spider seed: " + str(seed))
-                ret_count = 0
-                status = None
+
                 store_queue = None
                 store_data = list()
-                for it in func(item, seed):
-                    ret_count += 1
+
+                iterators = func(item, seed)
+
+                if not isgenerator(iterators):
+                    if not self.storage:
+                        del_seed(seed, spider_status=True)
+                        continue
+                    raise TypeError(f"{func.__name__} isn't a generator")
+
+                for it in iterators:
                     if getattr(it, "table_name", None):
                         if not store_queue:
                             store_queue = it.queue()
                         store_data.append(it.struct_data)
                     elif isinstance(it, Seed):
                         self.queue.push(it)
-                    elif any(isinstance(it, t) for t in (list, tuple)):
-                        self.queue.push([s if isinstance(s, Seed) else Seed(s) for s in it])
-                    elif isinstance(it, bool):
-                        status = it
+
+                    elif isinstance(it, str) and it == DealModel.polling:
+                        self.queue.push(seed)
+                        break
+                    elif isinstance(it, str) and it == DealModel.success:
+                        del_seed(seed, spider_status=True)
+                        break
+                    elif isinstance(it, str) and it == DealModel.failure:
+                        del_seed(seed, spider_status=False)
+                        break
+                    else:
+                        raise TypeError("yield value type error!")
 
                 if store_queue and store_data:
                     store_data.append(seed)
                     store_queue.push(store_data)
 
-                if status:
-                    del_seed(seed, spider_status=True)
-                elif not ret_count or status is False:
-                    seed._retry += 1
-                    self.queue.push(seed)
-
             except Exception as e:
                 seed._retry += 1
                 self.queue.push(seed)
@@ -106,7 +117,7 @@ class Storer:
     def store_task(self, stop, last, reset_seed, set_storer):
 
         inf_name = "StorerInterface"
-        if not issubclass_cobweb_inf(self.__class__, inf_name):
+        if not ici(self.__class__, inf_name):
             return None
 
         if not getattr(self, "store", None):
@@ -131,12 +142,10 @@ class Storer:
                         continue
                     data_list.append(data)
 
-                if data_list:
-                    if self.store(data_list):
-                        set_storer(store_key_id, seeds)
-                    else:
-                        reset_seed(seeds)
-                    continue
+                if self.store(data_list):
+                    set_storer(store_key_id, seeds)
+                else:
+                    reset_seed(seeds)
 
             time.sleep(3)
 

{cobweb-launcher-0.1.8/cobweb → cobweb-launcher-0.1.9/cobweb/equip}/single/launcher.py

@@ -1,15 +1,9 @@
 import time
 import threading
-from threading import Thread
 
+from .. import log, sqn, rtn, pim
+from .. import Queue, DBItem, RedisDB, Setting
 from .models import Scheduler, Spider, Storer
-from cobweb import log, Queue, DBItem, RedisDB
-from cobweb.setting import MODEL, RESET_SCORE, CHECK_LOCK_TIME
-from cobweb.utils import (
-    struct_queue_name as sqn,
-    restore_table_name as rtn,
-    parse_import_model as pim,
-)
 
 
 def check(stop, last, spider, scheduler, storer, ready_seed_length, spider_queue_length):
@@ -29,27 +23,26 @@ def check(stop, last, spider, scheduler, storer, ready_seed_length, spider_queue
         redis_ready_seed_length = ready_seed_length()
         redis_spider_seed_length = spider_queue_length()
         memory_seed_queue_length = scheduler.queue.length
-        storer_upload_queue_length = storer.queue.length
+        storer_upload_queue_length = storer.queue.length if storer else None
         if (
                 scheduler.stop and
-                # not redis_ready_seed_length and
                 not memory_seed_queue_length and
                 not running_spider_thread_num
         ):
-            if not MODEL:
+            if not Setting.LAUNCHER_MODEL:
                 log.info("spider is done?")
             last.set()
             time.sleep(3)
             storer_queue_empty = True
-            if storer.queue.length:
+            if storer and storer.queue.length:
                 storer_queue_empty = False
-            storer_upload_queue_length = storer.queue.length
+            storer_upload_queue_length = storer.queue.length if storer else None
             if (
                     storer_queue_empty and
                     not redis_ready_seed_length and
                     not redis_spider_seed_length
             ):
-                if MODEL:
+                if Setting.LAUNCHER_MODEL:
                     log.info("waiting for push seeds...")
                     status = "waiting"
                     time.sleep(30)
@@ -78,32 +71,13 @@ def launcher(task):
     :param task: 任务配置信息
     """
     def decorator(func):
-        """
-        Item:
-            Textfile()
-            Loghub()
-            Console()
-        e.g.
-        task.fields = "a,b"
-        func(item, seed)
-            a = "a"
-            b = "b"
-            data = {"a": "a", "b": "b"}
-            yield item.Loghub(**data)
-            yield item.Loghub(a=a, b=b)
-        """
-        storer_list = []
-
         # 程序结束事件
         last = threading.Event()
         # 停止采集事件
         stop = threading.Event()
 
         # 初始化redis信息
-        redis_db = RedisDB(
-            task.project, task.task_name, task.redis_info,
-            model=MODEL, cs_lct=CHECK_LOCK_TIME, rs_time=RESET_SCORE
-        )
+        redis_db = RedisDB(task.project, task.task_name, task.redis_info)
 
         # new item
         item = type("Item", (object,), {"redis_client": redis_db.client})()
@@ -113,7 +87,6 @@ def launcher(task):
         seed_queue = Queue()
 
         scheduler_info = task.scheduler_info or dict()
-
         # 调度器动态继承
         sql = scheduler_info.get("sql")
         table = scheduler_info.get("table")
@@ -123,22 +96,15 @@ def launcher(task):
         DB, class_name = pim(scheduler_db, "scheduler")
         # SchedulerDB, table, sql, length, size, config = task.scheduler_info
         SchedulerTmp = type(class_name, (Scheduler, DB), {})
-
         # 初始化调度器
         scheduler = SchedulerTmp(
             table=table, sql=sql, size=size, queue=seed_queue,
             length=task.scheduler_queue_length, config=scheduler_config
         )
 
-        # 初始化采集器
-        spider = Spider(seed_queue, task.max_retries)
-
         storer = None
-
-        # 解析存储器信息
         storer_info = task.storer_info or dict()
 
-        # for storer_info in storer_info_list:
         if storer_info:
             storer_db = storer_info["db"]
             fields = storer_info["fields"]
@@ -166,14 +132,17 @@ def launcher(task):
                 queue=queue, config=storer_config
             )
 
-        Thread(target=redis_db.check_spider_queue, args=(stop, len(storer_list))).start()
-        Thread(target=redis_db.set_heartbeat, args=(stop,)).start()
+        # 初始化采集器
+        spider = Spider(seed_queue, storer and True, task.max_retries)
+
+        threading.Thread(target=redis_db.check_spider_queue, args=(stop, 0)).start()
+        threading.Thread(target=redis_db.set_heartbeat, args=(stop,)).start()
 
         # 推送初始种子
         # seeds = start_seeds(task.start_seed)
         redis_db.add_seed(task.seeds)
         # 启动调度器, 调度至redis队列
-        Thread(
+        threading.Thread(
             # name="xxxx_schedule_seeds",
             target=scheduler.schedule_seed,
             args=(
@@ -184,7 +153,7 @@ def launcher(task):
         ).start()
 
         # 启动调度器, 调度任务队列
-        Thread(
+        threading.Thread(
             # name="xxxx_schedule_task",
             target=scheduler.schedule_task,
             args=(
@@ -195,7 +164,7 @@ def launcher(task):
 
         # 启动采集器
         for index in range(task.spider_num):
-            Thread(
+            threading.Thread(
                 # name=f"xxxx_spider_task:{index}",
                 target=spider.spider_task,
                 args=(
@@ -206,7 +175,7 @@ def launcher(task):
 
         # 启动存储器
         if storer:
-            Thread(
+            threading.Thread(
                 # name=f"xxxx_store_task:{storer.table}",
                 target=storer.store_task,
                 args=(
@@ -216,7 +185,7 @@ def launcher(task):
                 )
             ).start()
 
-        Thread(
+        threading.Thread(
             # name="check_spider",
             target=check,
             args=(

{cobweb-launcher-0.1.8/cobweb → cobweb-launcher-0.1.9/cobweb/equip}/single/models.py

@@ -1,7 +1,8 @@
 import time
-from cobweb import log, Queue, Seed
-from cobweb.utils import issubclass_cobweb_inf
+from inspect import isgenerator
 # from pympler import asizeof
+from .. import log, ici
+from .. import DealModel, Queue, Seed
 
 
 class Scheduler:
@@ -9,7 +10,7 @@ class Scheduler:
     def schedule_seed(self, ready_seed_length, get_scheduler_lock, add_seed):
 
         inf_name = "SchedulerInterface"
-        if not issubclass_cobweb_inf(self.__class__, inf_name):
+        if not ici(self.__class__, inf_name):
             raise Exception("not have schedule function!")
 
         if self.__class__.__name__ == "Default":
@@ -46,27 +47,37 @@ class Scheduler:
 
 class Spider:
 
-    def __init__(self, queue, max_retries=5):
+    def __init__(self, queue, storage, max_retries=5):
         self.spider_in_progress = Queue()
         self.max_retries = max_retries
+        self.storage = storage
         self.queue = queue
 
     def spider_task(self, stop, func, item, del_seed):
         while not stop.is_set():
+
             seed = self.queue.pop()
+
             if not seed:
                 time.sleep(3)
                 continue
+
             elif seed._retry >= self.max_retries:
                 del_seed(seed, spider_status=False)
                 continue
+
             try:
                 self.spider_in_progress.push(1, direct_insertion=True)
                 # log.info("spider seed: " + str(seed))
-                ret_count = 0
-                status = None
-                for it in func(item, seed):
-                    ret_count += 1
+                iterators = func(item, seed)
+
+                if not isgenerator(iterators):
+                    if not self.storage:
+                        del_seed(seed, spider_status=True)
+                        continue
+                    raise TypeError(f"{func.__name__} isn't a generator")
+
+                for it in iterators:
                     if getattr(it, "table_name", None):
                         store_queue = it.queue()
                         store_queue.push(
@@ -75,16 +86,18 @@ class Spider:
                         )
                     elif isinstance(it, Seed):
                         self.queue.push(it)
-                    elif any(isinstance(it, t) for t in (list, tuple)):
-                        self.queue.push([s if isinstance(s, Seed) else Seed(s) for s in it])
-                    elif isinstance(it, bool):
-                        status = it
 
-                if status:
-                    del_seed(seed, spider_status=True)
-                elif not ret_count or status is False:
-                    seed._retry += 1
-                    self.queue.push(seed)
+                    elif isinstance(it, str) and it == DealModel.polling:
+                        self.queue.push(seed)
+                        break
+                    elif isinstance(it, str) and it == DealModel.success:
+                        del_seed(seed, spider_status=True)
+                        break
+                    elif isinstance(it, str) and it == DealModel.failure:
+                        del_seed(seed, spider_status=False)
+                        break
+                    else:
+                        raise TypeError("yield value type error!")
 
             except Exception as e:
                 seed._retry += 1
@@ -100,7 +113,7 @@ class Storer:
     def store_task(self, stop, last, reset_seed, del_seed):
 
         inf_name = "StorerInterface"
-        if not issubclass_cobweb_inf(self.__class__, inf_name):
+        if not ici(self.__class__, inf_name):
             return None
 
         if not getattr(self, "store", None):
@@ -121,13 +134,10 @@ class Storer:
                     seeds.append(seed)
                     data_list.append(data)
 
-                if data_list:
-                    if self.store(data_list):
-                        del_seed(seeds)
-                    else:
-                        reset_seed(seeds)
-                        log.info("reset seeds!")
-                    continue
+                if self.store(data_list):
+                    del_seed(seeds)
+                else:
+                    reset_seed(seeds)
 
             time.sleep(3)
 

{cobweb-launcher-0.1.8 → cobweb-launcher-0.1.9}/cobweb/task.py

@@ -1,11 +1,19 @@
+import os
+from .constant import *
 from .utils import parse_info, struct_start_seeds
 
 
+def init_task_env():
+    Setting.RESET_SCORE = int(os.getenv("RESET_SCORE", 600))
+    Setting.CHECK_LOCK_TIME = int(os.getenv("CHECK_LOCK_TIME", 30))
+    Setting.DEAL_MODEL = os.getenv("DEAL_MODEL", DealModel.failure)
+    Setting.LAUNCHER_MODEL = os.getenv("LAUNCHER_MODEL", LauncherModel.task)
+
+
 class Task:
 
     def __init__(
             self,
-            # model=None,
             seeds=None,
             project=None,
             task_name=None,
@@ -31,8 +39,7 @@ class Task:
         :param storer_queue_length:
         :param scheduler_queue_length:
         """
-        # self.model = model
-
+        init_task_env()
         self.seeds = struct_start_seeds(seeds)
         self.project = project or "test"
         self.task_name = task_name or "spider"

{cobweb-launcher-0.1.8 → cobweb-launcher-0.1.9}/cobweb_launcher.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 0.1.8
+Version: 0.1.9
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP
@@ -11,6 +11,10 @@ Classifier: Programming Language :: Python :: 3
 Requires-Python: >=3.7
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: requests>=2.19.1
+Requires-Dist: oss2>=2.18.1
+Requires-Dist: redis>=4.4.4
+Requires-Dist: aliyun-log-python-sdk
 
 # cobweb
 

{cobweb-launcher-0.1.8 → cobweb-launcher-0.1.9}/cobweb_launcher.egg-info/SOURCES.txt

@@ -3,10 +3,10 @@ README.md
 setup.py
 cobweb/__init__.py
 cobweb/bbb.py
+cobweb/constant.py
 cobweb/decorators.py
 cobweb/interface.py
 cobweb/log.py
-cobweb/setting.py
 cobweb/task.py
 cobweb/utils.py
 cobweb/db/__init__.py
@@ -18,14 +18,14 @@ cobweb/db/scheduler/textfile.py
 cobweb/db/storer/__init__.py
 cobweb/db/storer/console.py
 cobweb/db/storer/loghub.py
-cobweb/db/storer/redis.py
 cobweb/db/storer/textfile.py
-cobweb/distributed/__init__.py
-cobweb/distributed/launcher.py
-cobweb/distributed/models.py
-cobweb/single/__init__.py
-cobweb/single/launcher.py
-cobweb/single/models.py
+cobweb/equip/__init__.py
+cobweb/equip/distributed/__init__.py
+cobweb/equip/distributed/launcher.py
+cobweb/equip/distributed/models.py
+cobweb/equip/single/__init__.py
+cobweb/equip/single/launcher.py
+cobweb/equip/single/models.py
 cobweb_launcher.egg-info/PKG-INFO
 cobweb_launcher.egg-info/SOURCES.txt
 cobweb_launcher.egg-info/dependency_links.txt

{cobweb-launcher-0.1.8 → cobweb-launcher-0.1.9}/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
 
 setup(
     name="cobweb-launcher",
-    version="0.1.8",
+    version="0.1.9",
     packages=find_packages(),
     url="https://github.com/Juannie-PP/cobweb",
     license="MIT",

cobweb-launcher-0.1.8/cobweb/__init__.py

@@ -1,11 +0,0 @@
-from .bbb import Seed, Queue, DBItem
-from .task import Task
-from .log import log
-from .interface import SchedulerInterface, StorerInterface
-from .db.redis_db import RedisDB
-from .db.oss_db import OssDB
-from .distributed.launcher import launcher
-from .single.launcher import launcher as single_launcher
-from . import setting
-
-

cobweb-launcher-0.1.8/cobweb/db/__init__.py

@@ -1,2 +0,0 @@
-from . import oss_db, redis_db
-from . import scheduler, storer

cobweb-launcher-0.1.8/cobweb/db/scheduler/default.py

@@ -1,8 +0,0 @@
-from cobweb import SchedulerInterface
-
-
-class Default(SchedulerInterface):
-
-    def schedule(self):
-        pass
-

cobweb-launcher-0.1.8/cobweb/db/storer/textfile.py

@@ -1,15 +0,0 @@
-from cobweb import log, StorerInterface
-
-
-class Textfile(StorerInterface):
-
-    def store(self, data_list):
-        try:
-            data_str = "\n".join(str(data) for data in data_list)
-            with open(self.table, "a") as fp:
-                fp.write(data_str)
-            log.info(f"save data, data length: {len(data_list)}")
-            return True
-        except Exception as e:
-            return False
-

cobweb-launcher-0.1.8/cobweb/setting.py

@@ -1,13 +0,0 @@
-import os
-
-
-# model: 0, 1, 2
-MODEL = int(os.getenv("MODEL", "0"))
-
-# 重制score值的等待时间, 默认10分钟
-RESET_SCORE = int(os.getenv("RESET_SCORE", "600"))
-
-# 默认设置检查spider queue队列锁的存活时间为30s
-CHECK_LOCK_TIME = int(os.getenv("CHECK_LOCK_TIME", 30))
-
-