cobweb-launcher 1.2.19__py3-none-any.whl → 1.2.21__py3-none-any.whl

cobweb/db/api_db.py

@@ -62,8 +62,8 @@ class ApiDB:
     def lock(self, name, t=15) -> bool:
         return self._get_response(api="/lock", params=dict(name=name, t=t))
 
-    def auto_incrby(self, name, t=15) -> bool:
-        return self._get_response(api="/auto_incrby", params=dict(name=name, t=t))
+    def auto_incr(self, name, t=15, limit=1000) -> bool:
+        return self._get_response(api="/auto_incr", params=dict(name=name, t=t, limit=limit))
 
     def members(self, name, score, start=0, count=5000, _min="-inf", _max="+inf"):
         return self._get_response(api="/members", params=dict(name=name, score=score, start=start, count=count, min=_min, max=_max))

cobweb/launchers/launcher_api.py

@@ -46,18 +46,13 @@ class LauncherApi(Launcher):
         设置时间窗口为self._time_window（秒），判断在该窗口内的采集量是否满足阈值（self._spider_max_speed）
         :return: True -> 种子, False -> None
         """
-        spider_speed = self._db.get(self._speed_control_key)
-        if int(spider_speed or 0) > self._spider_max_count:
+        if (self.__LAUNCHER_QUEUE__["todo"].length and
+                not self._db.auto_incr(self._speed_control_key, t=self._time_window, limit=self._spider_max_count)):
             expire_time = self._db.ttl(self._speed_control_key)
-            if expire_time == -1:
-                self._db.delete(self._speed_control_key)
-            elif expire_time and expire_time > 0:
-                logger.info(f"Too fast! Please wait {expire_time} seconds...")
-                time.sleep(expire_time / 2)
+            logger.info(f"Too fast! Please wait {expire_time} seconds...")
+            time.sleep(expire_time / 2)
             return None
         seed = self.__LAUNCHER_QUEUE__["todo"].pop()
-        if seed:
-            self._db.auto_incrby(self._speed_control_key, t=self._time_window)
         return seed
 
     @check_pause

{cobweb_launcher-1.2.19.dist-info → cobweb_launcher-1.2.21.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.2.19
+Version: 1.2.21
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb_launcher-1.2.19.dist-info → cobweb_launcher-1.2.21.dist-info}/RECORD

@@ -14,14 +14,14 @@ cobweb/crawlers/base_crawler.py,sha256=ee_WSDnPQpPTk6wlFuY2UEx5L3hcsAZFcr6i3GLSr
 cobweb/crawlers/crawler.py,sha256=xiFNM0t69f5xlm59hPbO2MpqtdirVAUhD84-CLpyHPM,6349
 cobweb/crawlers/file_crawler.py,sha256=2Sjbdgxzqd41WykKUQE3QQlGai3T8k-pmHNmPlTchjQ,4454
 cobweb/db/__init__.py,sha256=uZwSkd105EAwYo95oZQXAfofUKHVIAZZIPpNMy-hm2Q,56
-cobweb/db/api_db.py,sha256=V0FUacy7orrrwEpJWbSxFI_2I_OuhIMpVp6__cOy4tw,2844
+cobweb/db/api_db.py,sha256=646diIcZxdO0lfev958PZ-hTm5BmtJ0ns50nZOjlXOc,2865
 cobweb/db/redis_db.py,sha256=fumNZJiio-uQqRcSrymx8eJ1PqsdOwITe_Y-9JOXxrQ,4298
 cobweb/exceptions/__init__.py,sha256=E9SHnJBbhD7fOgPFMswqyOf8SKRDrI_i25L0bSpohvk,32
 cobweb/exceptions/oss_db_exception.py,sha256=iP_AImjNHT3-Iv49zCFQ3rdLnlvuHa3h2BXApgrOYpA,636
 cobweb/launchers/__init__.py,sha256=qMuVlQcjErVK67HyKFZEsXf_rfZD5ODjx1QucSCKMOM,114
 cobweb/launchers/launcher.py,sha256=SK4f3Fpuv-QMMriHruXGQ1sh1lxT1DZ2PdG0p2wAzNw,6978
 cobweb/launchers/launcher_air.py,sha256=KAk_M8F3029cXYe7m4nn3Nzyi89lbxJ2cqZjqW8iZ0E,2832
-cobweb/launchers/launcher_api.py,sha256=Ddx4cuD1nsdHRdY8BeyUS_pzVDEWP59QLcgZ9-LXh0M,8832
+cobweb/launchers/launcher_api.py,sha256=t-hPYhyD_Lq-75YxOXsPqTovSEuwMirCq0Tu1RKAILE,8638
 cobweb/launchers/launcher_pro.py,sha256=2JdN4khAFGBWLXDKaWknjG72XBhfTAnwtrgVmkF0K3M,8653
 cobweb/pipelines/__init__.py,sha256=zSUsGtx6smbs2iXBXvYynReKSgky-3gjqaAtKVnA_OU,105
 cobweb/pipelines/base_pipeline.py,sha256=fYnWf79GmhufXpcnMa3te18SbmnVeYLwxfyo-zLd9CY,1577
@@ -33,8 +33,8 @@ cobweb/utils/__init__.py,sha256=vBtZTy3EfRE0MmH43URhmr7nw6_oOWTEbGOM9xR_9o8,78
 cobweb/utils/bloom.py,sha256=vng-YbKgh9HbtpAWYf_nkUSbfVTOj40aqUUejRYlsCU,1752
 cobweb/utils/oss.py,sha256=gyt8-UB07tVphZLQXMOf-JTJwU-mWq8KZkOXKkAf3uk,3513
 cobweb/utils/tools.py,sha256=5JEaaAwYoV9Sdla2UBIJn6faUBuXmxUMagm9ck6FVqs,1253
-cobweb_launcher-1.2.19.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
-cobweb_launcher-1.2.19.dist-info/METADATA,sha256=0e2ggPHlbGt6nna7QasBSceYp8i9n3QffwWgnAWKoIY,6510
-cobweb_launcher-1.2.19.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
-cobweb_launcher-1.2.19.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
-cobweb_launcher-1.2.19.dist-info/RECORD,,
+cobweb_launcher-1.2.21.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
+cobweb_launcher-1.2.21.dist-info/METADATA,sha256=--vQihg3yNZyTL9B02LGWLl0gKNYB0uu1ktKC_PmFXI,6510
+cobweb_launcher-1.2.21.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
+cobweb_launcher-1.2.21.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
+cobweb_launcher-1.2.21.dist-info/RECORD,,