cobweb-launcher 1.2.10__py3-none-any.whl → 1.2.13__py3-none-any.whl

cobweb/crawlers/crawler.py

@@ -26,7 +26,8 @@ class Crawler(threading.Thread):
             launcher_queue: Union[Mapping[str, Queue]],
             custom_func: Union[Mapping[str, Callable]],
             thread_num: int,
-            max_retries: int
+            max_retries: int,
+            time_sleep: int
     ):
         super().__init__()
 
@@ -43,6 +44,7 @@ class Crawler(threading.Thread):
 
         self.thread_num = thread_num
         self.max_retries = max_retries
+        self.time_sleep = time_sleep
 
     @staticmethod
     def request(seed: Seed) -> Union[Request, BaseItem]:
@@ -150,6 +152,7 @@ class Crawler(threading.Thread):
                 ))
                 seed.params.retry += 1
                 self._todo.push(seed)
+                time.sleep(self.time_sleep * seed.params.retry)
             finally:
                 time.sleep(0.1)
         logger.info("spider thread close")

cobweb/launchers/launcher.py

@@ -94,10 +94,13 @@ class Launcher(threading.Thread):
 
         self._spider_max_retries = setting.SPIDER_MAX_RETRIES
         self._spider_thread_num = setting.SPIDER_THREAD_NUM
+        self._spider_time_sleep = setting.SPIDER_TIME_SLEEP
 
         self._done_model = setting.DONE_MODEL
         self._task_model = setting.TASK_MODEL
 
+        self._filter_field = setting.FILTER_FIELD
+
     @property
     def request(self):
         """
@@ -151,7 +154,7 @@ class Launcher(threading.Thread):
     def _remove_doing_seeds(self, seeds):
         for seed in seeds:
             self.__DOING__.pop(seed, None)
-        logger.info("remove %s seeds from __DOING__" % len(seeds))
+        # logger.info("remove %s seeds from __DOING__" % len(seeds))
 
     def _execute(self):
         for func_name in self.__LAUNCHER_FUNC__:
@@ -168,7 +171,8 @@ class Launcher(threading.Thread):
             launcher_queue=self.__LAUNCHER_QUEUE__,
             custom_func=self.__CUSTOM_FUNC__,
             thread_num = self._spider_thread_num,
-            max_retries = self._spider_max_retries
+            max_retries = self._spider_max_retries,
+            time_sleep=self._spider_time_sleep
         ).start()
 
         self._Pipeline(

cobweb/launchers/launcher_pro.py

@@ -3,6 +3,7 @@ import threading
 
 from cobweb.db import RedisDB
 from cobweb.base import Seed, logger
+from cobweb.utils import BloomFilter
 from cobweb.constant import DealModel, LogTemplate
 from .launcher import Launcher, check_pause
 
@@ -18,8 +19,11 @@ class LauncherPro(Launcher):
         self._reset_lock_key = "lock:reset:%s_%s" % (project, task)
         self._statistics_done_key = "statistics:%s:%s:done" % (project, task)
         self._statistics_fail_key = "statistics:%s:%s:fail" % (project, task)
+        self._bf_key = "bloom_%s_%s" % (project, task)
         self._db = RedisDB()
 
+        self._bf = BloomFilter(self._bf_key)
+
         self._heartbeat_start_event = threading.Event()
         self._redis_queue_empty_event = threading.Event()
 
@@ -125,15 +129,18 @@ class LauncherPro(Launcher):
                 seeds.append(seed.to_string)
         if seeds:
             count = self._db.zrem(self._todo_key, *seeds)
-            self.statistics(self._statistics_done_key, count)
+            if count:
+                self.statistics(self._statistics_done_key, count)
             self._remove_doing_seeds(seeds)
         if s_seeds:
             count = self._db.done([self._todo_key, self._done_key], *s_seeds)
-            self.statistics(self._statistics_done_key, count)
+            if count:
+                self.statistics(self._statistics_done_key, count)
             self._remove_doing_seeds(s_seeds)
         if f_seeds:
             count = self._db.done([self._todo_key, self._fail_key], *f_seeds)
-            self.statistics(self._statistics_fail_key, count)
+            if count:
+                self.statistics(self._statistics_fail_key, count)
             self._remove_doing_seeds(f_seeds)
 
         if status:

cobweb/pipelines/pipeline.py

@@ -61,7 +61,8 @@ class Pipeline(threading.Thread, ABC):
             if not status:
                 for seed in seeds:
                     seed.params.seed_status = "deal model: fail"
-            self._done.push(seeds)
+            if seeds:
+                self._done.push(seeds)
 
         logger.info("upload pipeline close!")
 

cobweb/setting.py

@@ -57,9 +57,15 @@ DONE_MODEL = 0   # 0:种子消费成功直接从队列移除，失败则添加
 # spider
 SPIDER_THREAD_NUM = 10
 SPIDER_MAX_RETRIES = 5
+SPIDER_TIME_SLEEP = 10
 
 # 任务模式
 TASK_MODEL = 0  # 0:单次，1:常驻
 
+
+# bloom过滤器
+CAPACITY = 100000000
+ERROR_RATE = 0.001
+FILTER_FIELD = "url"
 # 文件下载响应类型过滤
 # FILE_FILTER_CONTENT_TYPE = ["text/html", "application/xhtml+xml"]

cobweb/utils/__init__.py

@@ -1,3 +1,4 @@
 from .oss import OssUtil
 from .tools import *
+from .bloom import BloomFilter
 

cobweb/utils/bloom.py

@@ -0,0 +1,98 @@
+import time
+
+
+from redis import Redis
+from cobweb import setting
+
+# class BloomFilter:
+#
+#     def __init__(self, key, redis_config=None, capacity=None, error_rate=None):
+#         redis_config = redis_config or setting.REDIS_CONFIG
+#         capacity = capacity or setting.CAPACITY
+#         error_rate = error_rate or setting.ERROR_RATE
+#         redis_config['db'] = 3
+#
+#         self.key = key
+#
+#         pool = redis.ConnectionPool(**redis_config)
+#         self.bit_size = self.get_bit_size(capacity, error_rate)
+#         self.hash_count = self.get_hash_count(self.bit_size, capacity)
+#         self._init_bloom_key()
+#
+#     def add(self, value):
+#         for seed in range(self.hash_count):
+#             result = mmh3.hash(value, seed) % self.bit_size
+#             self._client.setbit(self.key, result, 1)
+#         return True
+#
+#     def exists(self, value):
+#         if not self._client.exists(self.key):
+#             return False
+#         for seed in range(self.hash_count):
+#             result = mmh3.hash(value, seed) % self.bit_size
+#             if not self._client.getbit(self.key, result):
+#                 return False
+#         return True
+#
+#     def _init_bloom_key(self):
+#         lua_script = """
+#         redis.call("SETBIT", KEYS[1], ARGV[1], ARGV[2])
+#         redis.call("EXPIRE", KEYS[1], 604800)
+#         """
+#         if self._client.exists(self.key):
+#             return True
+#         execute = self._client.register_script(lua_script)
+#         execute(keys=[self.key], args=[self.bit_size-1, 1])
+#
+#     @classmethod
+#     def get_bit_size(cls, n, p):
+#         return int(-(n * math.log(p)) / (math.log(2) ** 2))
+#
+#     @classmethod
+#     def get_hash_count(cls, m, n):
+#         return int((m / n) * math.log(2))
+
+class BloomFilter:
+
+    def __init__(self, key, redis_config=None, capacity=None, error_rate=None):
+        redis_config = redis_config or setting.REDIS_CONFIG
+        capacity = capacity or setting.CAPACITY
+        error_rate = error_rate or setting.ERROR_RATE
+        redis_config['db'] = 3
+
+        self.key = key
+
+        self._client = Redis(**redis_config).bf()
+        self._client.create(key=self.key, capacity=capacity, errorRate=error_rate)
+
+    def add(self, value):
+        return self._client.add(self.key, value)
+
+    def madd(self, items: list):
+        return self._client.madd(self.key, *items)
+
+    def exists(self, value):
+        return self._client.exists(self.key, value)
+
+    def mexists(self, items: list):
+        return self._client.mexists(self.key, *items)
+
+
+if __name__ == '__main__':
+    testBLF = BloomFilter("test", {
+        "host": "r-j6c1t3etiefpmz7cwdpd.redis.rds.aliyuncs.com",
+        "password": "SpiderLinux666",
+    })
+
+    print("start")
+    start_time = time.time()
+    testBLF.add("test")
+    add_time = time.time()
+    print("add time::: ")
+    print(add_time - start_time)
+    print("get::: ")
+    print(testBLF.exists("test"))
+    exist_time = time.time()
+    print("get time::: ")
+    print(exist_time - add_time)
+

{cobweb_launcher-1.2.10.dist-info → cobweb_launcher-1.2.13.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.2.10
+Version: 1.2.13
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP
@@ -16,6 +16,7 @@ Requires-Dist: requests (>=2.19.1)
 Requires-Dist: oss2 (>=2.18.1)
 Requires-Dist: redis (>=4.4.4)
 Requires-Dist: aliyun-log-python-sdk
+Requires-Dist: mmh3
 
 # cobweb 
 cobweb是一个基于python的分布式爬虫调度框架，目前支持分布式爬虫，单机爬虫，支持自定义数据库，支持自定义数据存储，支持自定义数据处理等操作。  

{cobweb_launcher-1.2.10.dist-info → cobweb_launcher-1.2.13.dist-info}/RECORD

@@ -1,6 +1,6 @@
 cobweb/__init__.py,sha256=uMHyf4Fekbyw2xBCbkA8R0LwCpBJf5p_7pWbh60ZWYk,83
 cobweb/constant.py,sha256=zy3XYsc1qp2B76_Fn_hVQ8eGHlPBd3OFlZK2cryE6FY,2839
-cobweb/setting.py,sha256=Wev0clo4ZETI5cRvBnzTWnJWo0Nowv_uvNCqlzYPSiE,1990
+cobweb/setting.py,sha256=S33EYK5hrrXn8GFkOmwt9Qn7OdNv8E4h8a0ZyeNydCM,2092
 cobweb/base/__init__.py,sha256=4gwWWQ0Q8cYG9cD7Lwf4XMqRGc5M_mapS3IczR6zeCE,222
 cobweb/base/common_queue.py,sha256=W7PPZZFl52j3Mc916T0imHj7oAUelA6aKJwW-FecDPE,872
 cobweb/base/decorators.py,sha256=wDCaQ94aAZGxks9Ljc0aXq6omDXT1_yzFy83ZW6VbVI,930
@@ -11,27 +11,28 @@ cobweb/base/response.py,sha256=eB1DWMXFCpn3cJ3yzgCRU1WeZAdayGDohRgdjdMUFN4,406
 cobweb/base/seed.py,sha256=Uz_VBRlAxNYQcFHk3tsZFMlU96yPOedHaWGTvk-zKd8,2908
 cobweb/crawlers/__init__.py,sha256=msvkB9mTpsgyj8JfNMsmwAcpy5kWk_2NrO1Adw2Hkw0,29
 cobweb/crawlers/base_crawler.py,sha256=ee_WSDnPQpPTk6wlFuY2UEx5L3hcsAZFcr6i3GLSry8,5751
-cobweb/crawlers/crawler.py,sha256=MJ2rFTrVjQ4Q9TmX0dpbOvT4tdokQwh7dKeFY_W7pPE,6002
+cobweb/crawlers/crawler.py,sha256=5mex-ENuzZSME0EIdwS9fnWkAX6LQuEyoDNcFm0emqs,6132
 cobweb/crawlers/file_crawler.py,sha256=2Sjbdgxzqd41WykKUQE3QQlGai3T8k-pmHNmPlTchjQ,4454
 cobweb/db/__init__.py,sha256=ut0iEyBLjcJL06WNG_5_d4hO5PJWvDrKWMkDOdmgh2M,30
 cobweb/db/redis_db.py,sha256=NNI2QkRV1hEZI-z-COEncXt88z3pZN6wusKlcQzc8V4,4304
 cobweb/exceptions/__init__.py,sha256=E9SHnJBbhD7fOgPFMswqyOf8SKRDrI_i25L0bSpohvk,32
 cobweb/exceptions/oss_db_exception.py,sha256=iP_AImjNHT3-Iv49zCFQ3rdLnlvuHa3h2BXApgrOYpA,636
 cobweb/launchers/__init__.py,sha256=af0Y6wrGX8SQZ7w7XL2sOtREjCT3dwad-uCc3nIontY,76
-cobweb/launchers/launcher.py,sha256=mBpq0CmxXXv-KdXQ2x7vwOkkAvaKECLkZLGhraafkQA,5953
+cobweb/launchers/launcher.py,sha256=sH9bj5TIPX5RjpNFvHmqTtlhyBxQqbTDhuvYSd2LspI,6114
 cobweb/launchers/launcher_air.py,sha256=KAk_M8F3029cXYe7m4nn3Nzyi89lbxJ2cqZjqW8iZ0E,2832
-cobweb/launchers/launcher_pro.py,sha256=IpckN0Xl5NAPVjuq-pWbCx0RerDmYG9gZ182EDURixo,7599
+cobweb/launchers/launcher_pro.py,sha256=4hi8xviIJr8HqTERShfdZjp767KXBeIyqyHaW9rYOhE,7815
 cobweb/pipelines/__init__.py,sha256=zSUsGtx6smbs2iXBXvYynReKSgky-3gjqaAtKVnA_OU,105
 cobweb/pipelines/base_pipeline.py,sha256=fYnWf79GmhufXpcnMa3te18SbmnVeYLwxfyo-zLd9CY,1577
 cobweb/pipelines/loghub_pipeline.py,sha256=cjPO6w6UJ0jNw2fVvdX0BCdlm58T7dmYXlxzXOBpvfY,1027
-cobweb/pipelines/pipeline.py,sha256=WBJnyLfSvyw0XmBMthgx3y_zwZBZS94L7FXpXPaKl8E,1983
+cobweb/pipelines/pipeline.py,sha256=4TJLX0sUHRxYndF5A4Vs5btUGI-wigkOcFvhTW1hLXI,2009
 cobweb/pipelines/pipeline_console.py,sha256=NEh-4zhuVAQOqwXLsqeb-rcNZ9_KXFUpL3otUTL5qBs,754
 cobweb/pipelines/pipeline_loghub.py,sha256=xZ6D55BGdiM71WUv83jyLGbEyUwhBHLJRZoXthBxxTs,1019
-cobweb/utils/__init__.py,sha256=JTE4sBfHnKHhD6w9Auk0MIT7O9BMOamCeryhlHNx3Zg,47
+cobweb/utils/__init__.py,sha256=vBtZTy3EfRE0MmH43URhmr7nw6_oOWTEbGOM9xR_9o8,78
+cobweb/utils/bloom.py,sha256=G0PlaMVTz6KCmhcNToi28bAHj1YJjDVqwJAQ_DUBWGk,3030
 cobweb/utils/oss.py,sha256=gyt8-UB07tVphZLQXMOf-JTJwU-mWq8KZkOXKkAf3uk,3513
 cobweb/utils/tools.py,sha256=5JEaaAwYoV9Sdla2UBIJn6faUBuXmxUMagm9ck6FVqs,1253
-cobweb_launcher-1.2.10.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
-cobweb_launcher-1.2.10.dist-info/METADATA,sha256=m1Gvgyfd8I-tPMoazMiUw46Aox58Nb0KHaD7Zu-nyts,6490
-cobweb_launcher-1.2.10.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
-cobweb_launcher-1.2.10.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
-cobweb_launcher-1.2.10.dist-info/RECORD,,
+cobweb_launcher-1.2.13.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
+cobweb_launcher-1.2.13.dist-info/METADATA,sha256=-HRZ-uipPBajhDyuDF2TTtQQAectd1gsjk3KwvWtHpo,6510
+cobweb_launcher-1.2.13.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
+cobweb_launcher-1.2.13.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
+cobweb_launcher-1.2.13.dist-info/RECORD,,