cobweb-launcher 1.2.13__py3-none-any.whl → 1.2.15__py3-none-any.whl

cobweb/crawlers/crawler.py

@@ -23,28 +23,34 @@ class Crawler(threading.Thread):
             self,
             stop: threading.Event,
             pause: threading.Event,
-            launcher_queue: Union[Mapping[str, Queue]],
+            # launcher_queue: Union[Mapping[str, Queue]],
+            get_seed: Callable,
+            set_seed: Callable,
+            add_seed: Callable,
+            delete_seed: Callable,
+            upload_data: Callable,
             custom_func: Union[Mapping[str, Callable]],
             thread_num: int,
             max_retries: int,
-            time_sleep: int
+            time_sleep: int,
     ):
         super().__init__()
 
         self._stop = stop
         self._pause = pause
-        self._new = launcher_queue["new"]
-        self._todo = launcher_queue["todo"]
-        self._done = launcher_queue["done"]
-        self._upload = launcher_queue["upload"]
+        self._get_seed = get_seed
+        self._set_seed = set_seed
+        self._add_seed = add_seed
+        self._delete_seed = delete_seed
+        self._upload_data = upload_data
 
         for func_name, _callable in custom_func.items():
             if isinstance(_callable, Callable):
                 self.__setattr__(func_name, _callable)
 
         self.thread_num = thread_num
-        self.max_retries = max_retries
         self.time_sleep = time_sleep
+        self.max_retries = max_retries
 
     @staticmethod
     def request(seed: Seed) -> Union[Request, BaseItem]:
@@ -66,23 +72,23 @@ class Crawler(threading.Thread):
 
     def distribute(self, item, seed):
         if isinstance(item, BaseItem):
-            self._upload.push(item)
+            self._upload_data(item)
         elif isinstance(item, Seed):
-            self._new.push(item)
+            self._add_seed(item)
         elif isinstance(item, str) and item == DealModel.poll:
-            self._todo.push(seed)
+            self._set_seed(seed)
         elif isinstance(item, str) and item == DealModel.done:
-            self._done.push(seed)
+            self._delete_seed(seed)
         elif isinstance(item, str) and item == DealModel.fail:
             seed.params.seed_status = DealModel.fail
-            self._done.push(seed)
+            self._delete_seed(seed)
         else:
             raise TypeError("yield value type error!")
 
     def spider(self):
         while not self._stop.is_set():
 
-            seed = self._todo.pop()
+            seed = self._get_seed()
 
             if not seed:
                 time.sleep(1)
@@ -90,7 +96,7 @@ class Crawler(threading.Thread):
 
             elif seed.params.retry > self.max_retries:
                 seed.params.seed_status = DealModel.fail
-                self._done.push(seed)
+                self._delete_seed(seed)
                 continue
 
             seed_detail_log_info = LogTemplate.log_info(seed.to_dict)
@@ -151,7 +157,8 @@ class Crawler(threading.Thread):
                     exception=''.join(traceback.format_exception(type(e), e, e.__traceback__))
                 ))
                 seed.params.retry += 1
-                self._todo.push(seed)
+                # self._todo.push(seed)
+                self._set_seed(seed)
                 time.sleep(self.time_sleep * seed.params.retry)
             finally:
                 time.sleep(0.1)

cobweb/db/redis_db.py

@@ -63,7 +63,7 @@ class RedisDB:
 
     def lock(self, key, t=15) -> bool:
         lua_script = """
-        local status = redis.call('setnx', KEYS[1], ARGV[1])
+        local status = redis.call('setnx', KEYS[1], 1)
         if ( status == 1 ) then
             redis.call('expire', KEYS[1], ARGV[1])
         end 

cobweb/launchers/launcher.py

@@ -17,6 +17,8 @@ def check_pause(func):
                 func(self, *args, **kwargs)
             except Exception as e:
                 logger.info(f"{func.__name__}: " + str(e))
+            finally:
+                time.sleep(0.1)
 
     return wrapper
 
@@ -95,6 +97,7 @@ class Launcher(threading.Thread):
         self._spider_max_retries = setting.SPIDER_MAX_RETRIES
         self._spider_thread_num = setting.SPIDER_THREAD_NUM
         self._spider_time_sleep = setting.SPIDER_TIME_SLEEP
+        self._spider_max_speed = setting.SPIDER_MAX_SPEED
 
         self._done_model = setting.DONE_MODEL
         self._task_model = setting.TASK_MODEL
@@ -156,6 +159,21 @@ class Launcher(threading.Thread):
             self.__DOING__.pop(seed, None)
         # logger.info("remove %s seeds from __DOING__" % len(seeds))
 
+    def _get_seed(self) -> Seed:
+        return self.__LAUNCHER_QUEUE__["todo"].pop()
+
+    def _set_seed(self, seed, **kwargs):
+        self.__LAUNCHER_QUEUE__["todo"].push(seed, **kwargs)
+
+    def _upload_data(self, data, **kwargs):
+        self.__LAUNCHER_QUEUE__["upload"].push(data, **kwargs)
+
+    def _add_seed(self, seed, **kwargs):
+        self.__LAUNCHER_QUEUE__["new"].push(seed, **kwargs)
+
+    def _delete_seed(self, seed, **kwargs):
+        self.__LAUNCHER_QUEUE__["done"].push(seed, **kwargs)
+
     def _execute(self):
         for func_name in self.__LAUNCHER_FUNC__:
             threading.Thread(name=func_name, target=getattr(self, func_name)).start()
@@ -168,7 +186,12 @@ class Launcher(threading.Thread):
 
         self._Crawler(
             stop=self._stop, pause=self._pause,
-            launcher_queue=self.__LAUNCHER_QUEUE__,
+            # launcher_queue=self.__LAUNCHER_QUEUE__,
+            get_seed=self._get_seed,
+            set_seed=self._set_seed,
+            add_seed=self._add_seed,
+            delete_seed=self._delete_seed,
+            upload_data=self._upload_data,
             custom_func=self.__CUSTOM_FUNC__,
             thread_num = self._spider_thread_num,
             max_retries = self._spider_max_retries,

cobweb/launchers/launcher_pro.py

@@ -16,10 +16,15 @@ class LauncherPro(Launcher):
         self._done_key = "{%s:%s}:done" % (project, task)
         self._fail_key = "{%s:%s}:fail" % (project, task)
         self._heartbeat_key = "heartbeat:%s_%s" % (project, task)
-        self._reset_lock_key = "lock:reset:%s_%s" % (project, task)
+
         self._statistics_done_key = "statistics:%s:%s:done" % (project, task)
         self._statistics_fail_key = "statistics:%s:%s:fail" % (project, task)
+        self._speed_control_key = "speed_control:%s_%s" % (project, task)
+
+        self._reset_lock_key = "lock:reset:%s_%s" % (project, task)
+
         self._bf_key = "bloom_%s_%s" % (project, task)
+
         self._db = RedisDB()
 
         self._bf = BloomFilter(self._bf_key)
@@ -37,6 +42,21 @@ class LauncherPro(Launcher):
         else:
             self._db._client.incrby(key, count)
 
+    def _get_seed(self) -> Seed:
+        spider_speed = self._db._client.get(self._speed_control_key)
+        if int(spider_speed or 0) > self._spider_max_speed:
+            expire_time = self._db.ttl(self._speed_control_key)
+            if expire_time == -1:
+                self._db.delete(self._speed_control_key)
+            else:
+                logger.info(f"Too fast! Please wait {expire_time} seconds...")
+                time.sleep(expire_time / 2)
+            return None
+        seed = self.__LAUNCHER_QUEUE__["todo"].pop()
+        if seed and not self._db.lock(self._speed_control_key, t=60):
+            self._db._client.incrby(self._speed_control_key, 1)
+        return seed
+
     @check_pause
     def _execute_heartbeat(self):
         if self._heartbeat_start_event.is_set():
@@ -114,7 +134,7 @@ class LauncherPro(Launcher):
         """
         删除队列种子，根据状态添加至成功或失败队列，移除doing字典种子索引
         """
-        seeds, s_seeds, f_seeds = [], [], []
+        seed_info = {"count": 0, "failed": [], "succeed": [], "common": []}
         status = self.__LAUNCHER_QUEUE__['done'].length < self._done_queue_max_size
 
         for _ in range(self._done_queue_max_size):
@@ -122,26 +142,25 @@ class LauncherPro(Launcher):
             if not seed:
                 break
             if seed.params.seed_status == DealModel.fail:
-                f_seeds.append(seed.to_string)
+                seed_info["failed"].append(seed.to_string)
             elif self._done_model == 1:
-                s_seeds.append(seed.to_string)
+                seed_info["succeed"].append(seed.to_string)
             else:
-                seeds.append(seed.to_string)
-        if seeds:
-            count = self._db.zrem(self._todo_key, *seeds)
-            if count:
-                self.statistics(self._statistics_done_key, count)
-            self._remove_doing_seeds(seeds)
-        if s_seeds:
-            count = self._db.done([self._todo_key, self._done_key], *s_seeds)
-            if count:
-                self.statistics(self._statistics_done_key, count)
-            self._remove_doing_seeds(s_seeds)
-        if f_seeds:
-            count = self._db.done([self._todo_key, self._fail_key], *f_seeds)
-            if count:
-                self.statistics(self._statistics_fail_key, count)
-            self._remove_doing_seeds(f_seeds)
+                seed_info["common"].append(seed.to_string)
+            seed_info['count'] += 1
+
+        if seed_info["count"]:
+
+            succeed_count = self._db.zrem(self._todo_key, *seed_info["common"])
+            succeed_count += self._db.done([self._todo_key, self._done_key], *seed_info["succeed"])
+            failed_count = self._db.done([self._todo_key, self._fail_key], *seed_info["failed"])
+
+            if failed_count:
+                self.statistics(self._statistics_fail_key, failed_count)
+            if succeed_count:
+                self.statistics(self._statistics_done_key, succeed_count)
+
+            self._remove_doing_seeds(seed_info["common"] + seed_info["succeed"] + seed_info["failed"])
 
         if status:
             time.sleep(self._done_queue_wait_seconds)

cobweb/setting.py

@@ -58,6 +58,7 @@ DONE_MODEL = 0   # 0:种子消费成功直接从队列移除，失败则添加
 SPIDER_THREAD_NUM = 10
 SPIDER_MAX_RETRIES = 5
 SPIDER_TIME_SLEEP = 10
+SPIDER_MAX_SPEED = 1000   # 一分钟最大采集数
 
 # 任务模式
 TASK_MODEL = 0  # 0:单次，1:常驻

cobweb/utils/bloom.py

@@ -1,56 +1,10 @@
+import math
 import time
 
-
-from redis import Redis
+import mmh3
+import redis
 from cobweb import setting
 
-# class BloomFilter:
-#
-#     def __init__(self, key, redis_config=None, capacity=None, error_rate=None):
-#         redis_config = redis_config or setting.REDIS_CONFIG
-#         capacity = capacity or setting.CAPACITY
-#         error_rate = error_rate or setting.ERROR_RATE
-#         redis_config['db'] = 3
-#
-#         self.key = key
-#
-#         pool = redis.ConnectionPool(**redis_config)
-#         self.bit_size = self.get_bit_size(capacity, error_rate)
-#         self.hash_count = self.get_hash_count(self.bit_size, capacity)
-#         self._init_bloom_key()
-#
-#     def add(self, value):
-#         for seed in range(self.hash_count):
-#             result = mmh3.hash(value, seed) % self.bit_size
-#             self._client.setbit(self.key, result, 1)
-#         return True
-#
-#     def exists(self, value):
-#         if not self._client.exists(self.key):
-#             return False
-#         for seed in range(self.hash_count):
-#             result = mmh3.hash(value, seed) % self.bit_size
-#             if not self._client.getbit(self.key, result):
-#                 return False
-#         return True
-#
-#     def _init_bloom_key(self):
-#         lua_script = """
-#         redis.call("SETBIT", KEYS[1], ARGV[1], ARGV[2])
-#         redis.call("EXPIRE", KEYS[1], 604800)
-#         """
-#         if self._client.exists(self.key):
-#             return True
-#         execute = self._client.register_script(lua_script)
-#         execute(keys=[self.key], args=[self.bit_size-1, 1])
-#
-#     @classmethod
-#     def get_bit_size(cls, n, p):
-#         return int(-(n * math.log(p)) / (math.log(2) ** 2))
-#
-#     @classmethod
-#     def get_hash_count(cls, m, n):
-#         return int((m / n) * math.log(2))
 
 class BloomFilter:
 
@@ -62,37 +16,43 @@ class BloomFilter:
 
         self.key = key
 
-        self._client = Redis(**redis_config).bf()
-        self._client.create(key=self.key, capacity=capacity, errorRate=error_rate)
+        pool = redis.ConnectionPool(**redis_config)
+        self._client = redis.Redis(connection_pool=pool)
+        self.bit_size = self.get_bit_size(capacity, error_rate)
+        self.hash_count = self.get_hash_count(self.bit_size, capacity)
+        self._init_bloom_key()
 
     def add(self, value):
-        return self._client.add(self.key, value)
-
-    def madd(self, items: list):
-        return self._client.madd(self.key, *items)
+        for seed in range(self.hash_count):
+            result = mmh3.hash(value, seed) % self.bit_size
+            self._client.setbit(self.key, result, 1)
+        return True
 
     def exists(self, value):
-        return self._client.exists(self.key, value)
-
-    def mexists(self, items: list):
-        return self._client.mexists(self.key, *items)
-
-
-if __name__ == '__main__':
-    testBLF = BloomFilter("test", {
-        "host": "r-j6c1t3etiefpmz7cwdpd.redis.rds.aliyuncs.com",
-        "password": "SpiderLinux666",
-    })
+        if not self._client.exists(self.key):
+            return False
+        for seed in range(self.hash_count):
+            result = mmh3.hash(value, seed) % self.bit_size
+            if not self._client.getbit(self.key, result):
+                return False
+        return True
+
+    def _init_bloom_key(self):
+        lua_script = """
+        redis.call("SETBIT", KEYS[1], ARGV[1], ARGV[2])
+        redis.call("EXPIRE", KEYS[1], 604800)
+        """
+        if self._client.exists(self.key):
+            return True
+        execute = self._client.register_script(lua_script)
+        execute(keys=[self.key], args=[self.bit_size-1, 1])
+
+    @classmethod
+    def get_bit_size(cls, n, p):
+        return int(-(n * math.log(p)) / (math.log(2) ** 2))
+
+    @classmethod
+    def get_hash_count(cls, m, n):
+        return int((m / n) * math.log(2))
 
-    print("start")
-    start_time = time.time()
-    testBLF.add("test")
-    add_time = time.time()
-    print("add time::: ")
-    print(add_time - start_time)
-    print("get::: ")
-    print(testBLF.exists("test"))
-    exist_time = time.time()
-    print("get time::: ")
-    print(exist_time - add_time)
 

{cobweb_launcher-1.2.13.dist-info → cobweb_launcher-1.2.15.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.2.13
+Version: 1.2.15
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb_launcher-1.2.13.dist-info → cobweb_launcher-1.2.15.dist-info}/RECORD

@@ -1,6 +1,6 @@
 cobweb/__init__.py,sha256=uMHyf4Fekbyw2xBCbkA8R0LwCpBJf5p_7pWbh60ZWYk,83
 cobweb/constant.py,sha256=zy3XYsc1qp2B76_Fn_hVQ8eGHlPBd3OFlZK2cryE6FY,2839
-cobweb/setting.py,sha256=S33EYK5hrrXn8GFkOmwt9Qn7OdNv8E4h8a0ZyeNydCM,2092
+cobweb/setting.py,sha256=MGe4QGnE5XOTh9Z7NhakaTFK7f-lZtzlA9PFcuc1qoY,2145
 cobweb/base/__init__.py,sha256=4gwWWQ0Q8cYG9cD7Lwf4XMqRGc5M_mapS3IczR6zeCE,222
 cobweb/base/common_queue.py,sha256=W7PPZZFl52j3Mc916T0imHj7oAUelA6aKJwW-FecDPE,872
 cobweb/base/decorators.py,sha256=wDCaQ94aAZGxks9Ljc0aXq6omDXT1_yzFy83ZW6VbVI,930
@@ -11,16 +11,16 @@ cobweb/base/response.py,sha256=eB1DWMXFCpn3cJ3yzgCRU1WeZAdayGDohRgdjdMUFN4,406
 cobweb/base/seed.py,sha256=Uz_VBRlAxNYQcFHk3tsZFMlU96yPOedHaWGTvk-zKd8,2908
 cobweb/crawlers/__init__.py,sha256=msvkB9mTpsgyj8JfNMsmwAcpy5kWk_2NrO1Adw2Hkw0,29
 cobweb/crawlers/base_crawler.py,sha256=ee_WSDnPQpPTk6wlFuY2UEx5L3hcsAZFcr6i3GLSry8,5751
-cobweb/crawlers/crawler.py,sha256=5mex-ENuzZSME0EIdwS9fnWkAX6LQuEyoDNcFm0emqs,6132
+cobweb/crawlers/crawler.py,sha256=xiFNM0t69f5xlm59hPbO2MpqtdirVAUhD84-CLpyHPM,6349
 cobweb/crawlers/file_crawler.py,sha256=2Sjbdgxzqd41WykKUQE3QQlGai3T8k-pmHNmPlTchjQ,4454
 cobweb/db/__init__.py,sha256=ut0iEyBLjcJL06WNG_5_d4hO5PJWvDrKWMkDOdmgh2M,30
-cobweb/db/redis_db.py,sha256=NNI2QkRV1hEZI-z-COEncXt88z3pZN6wusKlcQzc8V4,4304
+cobweb/db/redis_db.py,sha256=fumNZJiio-uQqRcSrymx8eJ1PqsdOwITe_Y-9JOXxrQ,4298
 cobweb/exceptions/__init__.py,sha256=E9SHnJBbhD7fOgPFMswqyOf8SKRDrI_i25L0bSpohvk,32
 cobweb/exceptions/oss_db_exception.py,sha256=iP_AImjNHT3-Iv49zCFQ3rdLnlvuHa3h2BXApgrOYpA,636
 cobweb/launchers/__init__.py,sha256=af0Y6wrGX8SQZ7w7XL2sOtREjCT3dwad-uCc3nIontY,76
-cobweb/launchers/launcher.py,sha256=sH9bj5TIPX5RjpNFvHmqTtlhyBxQqbTDhuvYSd2LspI,6114
+cobweb/launchers/launcher.py,sha256=AbkrytfJEyj8FhTbLgjmOOIuvOYV3cpVknE9yt31WbM,6930
 cobweb/launchers/launcher_air.py,sha256=KAk_M8F3029cXYe7m4nn3Nzyi89lbxJ2cqZjqW8iZ0E,2832
-cobweb/launchers/launcher_pro.py,sha256=4hi8xviIJr8HqTERShfdZjp767KXBeIyqyHaW9rYOhE,7815
+cobweb/launchers/launcher_pro.py,sha256=8QKhToKoD2WonIaqRQAhUWRhbNOIgYXzGFRK1id_3yM,8638
 cobweb/pipelines/__init__.py,sha256=zSUsGtx6smbs2iXBXvYynReKSgky-3gjqaAtKVnA_OU,105
 cobweb/pipelines/base_pipeline.py,sha256=fYnWf79GmhufXpcnMa3te18SbmnVeYLwxfyo-zLd9CY,1577
 cobweb/pipelines/loghub_pipeline.py,sha256=cjPO6w6UJ0jNw2fVvdX0BCdlm58T7dmYXlxzXOBpvfY,1027
@@ -28,11 +28,11 @@ cobweb/pipelines/pipeline.py,sha256=4TJLX0sUHRxYndF5A4Vs5btUGI-wigkOcFvhTW1hLXI,
 cobweb/pipelines/pipeline_console.py,sha256=NEh-4zhuVAQOqwXLsqeb-rcNZ9_KXFUpL3otUTL5qBs,754
 cobweb/pipelines/pipeline_loghub.py,sha256=xZ6D55BGdiM71WUv83jyLGbEyUwhBHLJRZoXthBxxTs,1019
 cobweb/utils/__init__.py,sha256=vBtZTy3EfRE0MmH43URhmr7nw6_oOWTEbGOM9xR_9o8,78
-cobweb/utils/bloom.py,sha256=G0PlaMVTz6KCmhcNToi28bAHj1YJjDVqwJAQ_DUBWGk,3030
+cobweb/utils/bloom.py,sha256=vng-YbKgh9HbtpAWYf_nkUSbfVTOj40aqUUejRYlsCU,1752
 cobweb/utils/oss.py,sha256=gyt8-UB07tVphZLQXMOf-JTJwU-mWq8KZkOXKkAf3uk,3513
 cobweb/utils/tools.py,sha256=5JEaaAwYoV9Sdla2UBIJn6faUBuXmxUMagm9ck6FVqs,1253
-cobweb_launcher-1.2.13.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
-cobweb_launcher-1.2.13.dist-info/METADATA,sha256=-HRZ-uipPBajhDyuDF2TTtQQAectd1gsjk3KwvWtHpo,6510
-cobweb_launcher-1.2.13.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
-cobweb_launcher-1.2.13.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
-cobweb_launcher-1.2.13.dist-info/RECORD,,
+cobweb_launcher-1.2.15.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
+cobweb_launcher-1.2.15.dist-info/METADATA,sha256=rP73tkfQPQB8MCEYg31ZOdEbBYUUoyXzAcUevfnwqec,6510
+cobweb_launcher-1.2.15.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
+cobweb_launcher-1.2.15.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
+cobweb_launcher-1.2.15.dist-info/RECORD,,