cobweb-launcher 1.2.49__py3-none-any.whl → 1.2.51__py3-none-any.whl

cobweb/base/seed.py

@@ -5,11 +5,13 @@ import hashlib
 
 class SeedParams:
 
-    def __init__(self, retry, priority, seed_version, seed_status=None):
+    def __init__(self, retry, priority, seed_version, seed_status=None, proxy_type=None, proxy=None):
         self.retry = retry or 0
         self.priority = priority or 300
         self.seed_version = seed_version or int(time.time())
         self.seed_status = seed_status
+        self.proxy_type = proxy_type
+        self.proxy = proxy
 
 
 class Seed:
@@ -18,7 +20,9 @@ class Seed:
         "retry",
         "priority",
         "seed_version",
-        "seed_status"
+        "seed_status",
+        "proxy_type",
+        "proxy"
     ]
 
     def __init__(
@@ -29,6 +33,8 @@ class Seed:
             priority=None,
             seed_version=None,
             seed_status=None,
+            proxy_type=None,
+            proxy=None,
             **kwargs
     ):
         if any(isinstance(seed, t) for t in (str, bytes)):
@@ -51,6 +57,8 @@ class Seed:
             "priority": priority,
             "seed_version": seed_version,
             "seed_status": seed_status,
+            "proxy_type": proxy_type,
+            "proxy": proxy
         }
 
         if kwargs:

cobweb/crawlers/crawler.py

@@ -19,7 +19,6 @@ from cobweb.base import (
     logger
 )
 from cobweb.utils import LoghubDot
-proxy_type = os.getenv("PROXY_TYPE", "")
 
 
 class Crawler(threading.Thread):
@@ -149,7 +148,9 @@ class Crawler(threading.Thread):
                                         topic=urlparse(download_item.response.request.url).netloc,
                                         data_size=int(download_item.response.headers.get("content-length", 0)),
                                         cost_time=end_time - start_time, status = 200,
-                                        url=download_item.response.url, proxy_type=proxy_type,
+                                        url=download_item.response.url,
+                                        proxy_type=seed.params.proxy_type,
+                                        proxy=seed.params.proxy,
                                         project=self.project, task=self.task,
                                     )
                                 parse_iterators = self.parse(download_item)
@@ -178,7 +179,8 @@ class Crawler(threading.Thread):
                     topic=urlparse(url).netloc,
                     data_size=-1, cost_time=-1,
                     status=status, url=url,
-                    proxy_type=proxy_type,
+                    proxy_type=seed.params.proxy_type,
+                    proxy=seed.params.proxy,
                     project=self.project,
                     task=self.task,
                     msg=exception_msg,

cobweb/launchers/launcher.py

@@ -100,6 +100,7 @@ class Launcher(threading.Thread):
         self._spider_time_sleep = setting.SPIDER_TIME_SLEEP
         self._spider_max_count = setting.SPIDER_MAX_COUNT
         self._time_window = setting.TIME_WINDOW
+        self._speed_control = setting.SPEED_CONTROL
 
         self._done_model = setting.DONE_MODEL
         self._task_model = setting.TASK_MODEL

cobweb/launchers/launcher_api.py

@@ -46,7 +46,7 @@ class LauncherApi(Launcher):
         设置时间窗口为self._time_window（秒），判断在该窗口内的采集量是否满足阈值（self._spider_max_speed）
         :return: True -> 种子, False -> None
         """
-        if (self.__LAUNCHER_QUEUE__["todo"].length and
+        if (self._speed_control and self.__LAUNCHER_QUEUE__["todo"].length and
                 not self._db.auto_incr(self._speed_control_key, t=self._time_window, limit=self._spider_max_count)):
             expire_time = self._db.ttl(self._speed_control_key)
             logger.info(f"Too fast! Please wait {expire_time} seconds...")

cobweb/launchers/launcher_pro.py

@@ -3,7 +3,7 @@ import threading
 
 from cobweb.db import RedisDB
 from cobweb.base import Seed, logger
-from cobweb.utils import BloomFilter
+# from cobweb.utils import BloomFilter
 from cobweb.constant import DealModel, LogTemplate
 from .launcher import Launcher, check_pause
 
@@ -43,17 +43,18 @@ class LauncherPro(Launcher):
             self._db._client.incrby(key, count)
 
     def _get_seed(self) -> Seed:
-        spider_speed = self._db._client.get(self._speed_control_key)
-        if int(spider_speed or 0) > self._spider_max_count:
-            expire_time = self._db.ttl(self._speed_control_key)
-            if expire_time == -1:
-                self._db.delete(self._speed_control_key)
-            else:
-                logger.info(f"Too fast! Please wait {expire_time} seconds...")
-                time.sleep(expire_time / 2)
-            return None
+        if self._speed_control:
+            spider_speed = self._db._client.get(self._speed_control_key)
+            if int(spider_speed or 0) > self._spider_max_count:
+                expire_time = self._db.ttl(self._speed_control_key)
+                if expire_time == -1:
+                    self._db.delete(self._speed_control_key)
+                else:
+                    logger.info(f"Too fast! Please wait {expire_time} seconds...")
+                    time.sleep(expire_time / 2)
+                return None
         seed = self.__LAUNCHER_QUEUE__["todo"].pop()
-        if seed and not self._db.lock(self._speed_control_key, t=self._time_window):
+        if self._speed_control and seed and not self._db.lock(self._speed_control_key, t=self._time_window):
             self._db._client.incrby(self._speed_control_key, 1)
         return seed
 

cobweb/setting.py

@@ -65,6 +65,8 @@ TIME_WINDOW = 60   # 频控固定时间窗口（秒）
 # 任务模式
 TASK_MODEL = 0  # 0:单次，1:常驻
 
+# 流控措施
+SPEED_CONTROL = 1  # 0:关闭，1:开启
 
 # bloom过滤器
 CAPACITY = 100000000

cobweb/utils/dotting.py

@@ -24,16 +24,7 @@ class LoghubDot:
                 temp[key] = value
         contents = sorted(temp.items())
         log_item.set_contents(contents)
-        # log_items.append(log_item)
-        # request = PutLogsRequest(
-        #     project="databee-download-log",
-        #     logstore="log",
-        #     topic=topic,
-        #     logitems=log_items,
-        #     compress=True
-        # )
         self.queue.push((topic, log_item), direct_insertion=True)
-        # self.client.put_logs(request=request)
 
     def build_run(self):
         while True:

{cobweb_launcher-1.2.49.dist-info → cobweb_launcher-1.2.51.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.2.49
+Version: 1.2.51
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb_launcher-1.2.49.dist-info → cobweb_launcher-1.2.51.dist-info}/RECORD

@@ -1,6 +1,6 @@
 cobweb/__init__.py,sha256=CBd2oByCfc5EmH2dCZYVHkxXYZG-oWrLyTtZU5sEoP0,96
 cobweb/constant.py,sha256=zy3XYsc1qp2B76_Fn_hVQ8eGHlPBd3OFlZK2cryE6FY,2839
-cobweb/setting.py,sha256=47HZsw40HLpsmOmvij1lyQALPQQCN_tWlKZ0wbn2MtM,2216
+cobweb/setting.py,sha256=JHRrVQL22iidkNe8seQTuxK3hlTdtnvb6VBVygpMs5I,2272
 cobweb/base/__init__.py,sha256=4gwWWQ0Q8cYG9cD7Lwf4XMqRGc5M_mapS3IczR6zeCE,222
 cobweb/base/basic.py,sha256=Z56SSLB3I2IGHWCCcSy0Qbfzj8Qbg_po3gP32q1jh4k,7741
 cobweb/base/common_queue.py,sha256=W7PPZZFl52j3Mc916T0imHj7oAUelA6aKJwW-FecDPE,872
@@ -10,10 +10,10 @@ cobweb/base/item.py,sha256=hYheVTV2Bozp4iciJpE2ZwBIXkaqBg4QQkRccP8yoVk,1049
 cobweb/base/log.py,sha256=L01hXdk3L2qEm9X1FOXQ9VmWIoHSELe0cyZvrdAN61A,2003
 cobweb/base/request.py,sha256=tEkgMVUfdQI-kZuzWuiit9P_q4Q9-_RZh9aXXpc0314,2352
 cobweb/base/response.py,sha256=eB1DWMXFCpn3cJ3yzgCRU1WeZAdayGDohRgdjdMUFN4,406
-cobweb/base/seed.py,sha256=Uz_VBRlAxNYQcFHk3tsZFMlU96yPOedHaWGTvk-zKd8,2908
+cobweb/base/seed.py,sha256=A-F1urjbE5hYNWTCwq3sUV4nrxlK_RGMoCmjBmIwYsI,3158
 cobweb/crawlers/__init__.py,sha256=msvkB9mTpsgyj8JfNMsmwAcpy5kWk_2NrO1Adw2Hkw0,29
 cobweb/crawlers/base_crawler.py,sha256=ee_WSDnPQpPTk6wlFuY2UEx5L3hcsAZFcr6i3GLSry8,5751
-cobweb/crawlers/crawler.py,sha256=dKyQJOVh1cnuazSryb9vA-IGJJnhw-gxAAlIqsEn6-M,8832
+cobweb/crawlers/crawler.py,sha256=pePDGroD6JJAht5QTU51L7MkFIY4ob9TKpznZ2wUmsw,8965
 cobweb/crawlers/file_crawler.py,sha256=2Sjbdgxzqd41WykKUQE3QQlGai3T8k-pmHNmPlTchjQ,4454
 cobweb/db/__init__.py,sha256=uZwSkd105EAwYo95oZQXAfofUKHVIAZZIPpNMy-hm2Q,56
 cobweb/db/api_db.py,sha256=bDc5dJQxq4z04h70KUTHd0OqUOEY7Cm3wcNJZtTvJIM,3015
@@ -21,10 +21,10 @@ cobweb/db/redis_db.py,sha256=fumNZJiio-uQqRcSrymx8eJ1PqsdOwITe_Y-9JOXxrQ,4298
 cobweb/exceptions/__init__.py,sha256=E9SHnJBbhD7fOgPFMswqyOf8SKRDrI_i25L0bSpohvk,32
 cobweb/exceptions/oss_db_exception.py,sha256=iP_AImjNHT3-Iv49zCFQ3rdLnlvuHa3h2BXApgrOYpA,636
 cobweb/launchers/__init__.py,sha256=qMuVlQcjErVK67HyKFZEsXf_rfZD5ODjx1QucSCKMOM,114
-cobweb/launchers/launcher.py,sha256=sPts-xlgxoeIfl1fn1XR2XVZxLzt7He9xrYDfTHRAGo,7029
+cobweb/launchers/launcher.py,sha256=_qC3EnD_QxQRP_RBNHHJ8HLfn_0FED1jUF_lyas5Hss,7081
 cobweb/launchers/launcher_air.py,sha256=KAk_M8F3029cXYe7m4nn3Nzyi89lbxJ2cqZjqW8iZ0E,2832
-cobweb/launchers/launcher_api.py,sha256=YFqCTRvKn6icBLWTR1VxkU0WEIte2F7fv_LgPkifqdo,7885
-cobweb/launchers/launcher_pro.py,sha256=B5FdxvuENRL3XrMl74ENdP1uNgnZOaYCUUfBfM0t3io,7842
+cobweb/launchers/launcher_api.py,sha256=7K7Kl3dk7Ung9iRBwhiMrALEJywcR66ie5RIkLQEM-Y,7909
+cobweb/launchers/launcher_pro.py,sha256=QMumHUTi6IudO5mw9SnwlmP8k_BJ50cpNjYjoQv-_YY,7936
 cobweb/pipelines/__init__.py,sha256=zSUsGtx6smbs2iXBXvYynReKSgky-3gjqaAtKVnA_OU,105
 cobweb/pipelines/pipeline.py,sha256=4TJLX0sUHRxYndF5A4Vs5btUGI-wigkOcFvhTW1hLXI,2009
 cobweb/pipelines/pipeline_console.py,sha256=NEh-4zhuVAQOqwXLsqeb-rcNZ9_KXFUpL3otUTL5qBs,754
@@ -34,11 +34,11 @@ cobweb/schedulers/scheduler_api.py,sha256=pFEdS1H4zuzxwMhCV-G7CoLz-rEOPv4EVo3xZU
 cobweb/schedulers/scheduler_redis.py,sha256=E5fjc3nNld8GbUhUGT7uY4smRejj2J2ZIzp2g6lhxFM,2205
 cobweb/utils/__init__.py,sha256=Ev2LZZ1-S56iQYDqFZrqadizEv4Gk8Of-DraH-_WnKY,109
 cobweb/utils/bloom.py,sha256=vng-YbKgh9HbtpAWYf_nkUSbfVTOj40aqUUejRYlsCU,1752
-cobweb/utils/dotting.py,sha256=_QsC-LCSh1QYk9XmYIWBRlRTs-E9RfqKSkkRkedV2Eo,2082
+cobweb/utils/dotting.py,sha256=mVICaa26R-dQ4JGmPK-kkR6QjX38QiRewXZnGb2DCIc,1784
 cobweb/utils/oss.py,sha256=gyt8-UB07tVphZLQXMOf-JTJwU-mWq8KZkOXKkAf3uk,3513
 cobweb/utils/tools.py,sha256=5JEaaAwYoV9Sdla2UBIJn6faUBuXmxUMagm9ck6FVqs,1253
-cobweb_launcher-1.2.49.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
-cobweb_launcher-1.2.49.dist-info/METADATA,sha256=VmuY5V3Hhjw9bZ5JqQfwJi8GfPWq5g6VDb6G39q36IM,6510
-cobweb_launcher-1.2.49.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
-cobweb_launcher-1.2.49.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
-cobweb_launcher-1.2.49.dist-info/RECORD,,
+cobweb_launcher-1.2.51.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
+cobweb_launcher-1.2.51.dist-info/METADATA,sha256=xzLg7vLTsWurDo9L8jjmnlq-8B6UE1bT2itXPHt4a8o,6510
+cobweb_launcher-1.2.51.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
+cobweb_launcher-1.2.51.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
+cobweb_launcher-1.2.51.dist-info/RECORD,,