cobweb-launcher 3.1.5__py3-none-any.whl → 3.1.7__py3-none-any.whl

cobweb/__init__.py

@@ -1,2 +1,4 @@
 from .launchers import Launcher
 from .constant import CrawlerModel
+from .pipelines import Pipeline
+from .crawlers import Crawler

cobweb/db/api_db.py

@@ -71,7 +71,7 @@ class ApiDB:
     def auto_incr(self, name, t=15, limit=1000) -> bool:
         return self._get_response(api="/auto_incr", params=dict(name=name, t=t, limit=limit))
 
-    def members(self, name, score, start=0, count=5000, _min="-inf", _max="+inf"):
+    def members(self, name, score, start=0, count=1000, _min="-inf", _max="+inf"):
         return self._get_response(api="/members", params=dict(name=name, score=score, start=start, count=count, min=_min, max=_max))
 
     def done(self, name: list, *values):

cobweb/launchers/__init__.py

@@ -1,8 +1,3 @@
-# from .launcher_air import LauncherAir
-# from .launcher_pro import LauncherPro
-# from .launcher_api import LauncherApi
-
-
 from .launcher import Launcher
 from .uploader import Uploader
 from .distributor import Distributor

cobweb/launchers/distributor.py

@@ -1,22 +1,16 @@
 import time
 import threading
 import traceback
-from inspect import isgenerator
+
 from typing import Callable
+from inspect import isgenerator
 from urllib.parse import urlparse
 from requests import Response as Res
 
 from cobweb import setting
 from cobweb.constant import DealModel, LogTemplate
-from cobweb.base import (
-    Seed,
-    Queue,
-    BaseItem,
-    Request,
-    Response,
-    logger
-)
 from cobweb.utils import LoghubDot, check_pause
+from cobweb.base import Seed, Queue, BaseItem, Request, Response, logger
 
 
 class Distributor(threading.Thread):

cobweb/schedulers/scheduler_with_redis.py

@@ -7,7 +7,7 @@ from cobweb.utils import check_pause
 from cobweb.base import Queue, Seed, logger
 from cobweb.constant import LogTemplate
 from .scheduler import Scheduler
-use_api = bool(int(os.getenv("REDIS_API", 0)))
+use_api = bool(int(os.getenv("REDIS_API_HOST", 0)))
 
 
 class RedisScheduler(Scheduler):
@@ -153,10 +153,10 @@ class RedisScheduler(Scheduler):
                 else:
                     if all_count:
                         logger.info(f"todo seeds count: {todo_count}, queue length: {all_count}")
-                        self.pause.is_set()
+                        self.pause.clear()
                     else:
                         logger.info("Done! pause set...")
-                        self.pause.clear()
+                        self.pause.set()
             else:
                 if self.pause.is_set():
                     self.pause.clear()

cobweb/setting.py

@@ -18,7 +18,7 @@ LOGHUB_CONFIG = {
     "accessKey": os.getenv("LOGHUB_SECRET_KEY")
 }
 
-# oss util config
+# # oss util config
 OSS_BUCKET = os.getenv("OSS_BUCKET")
 OSS_ENDPOINT = os.getenv("OSS_ENDPOINT")
 OSS_ACCESS_KEY = os.getenv("OSS_ACCESS_KEY")
@@ -72,9 +72,9 @@ TASK_MODEL = 0  # 0:单次，1:常驻
 # 流控措施
 SPEED_CONTROL = 1  # 0:关闭，1:开启
 
-# bloom过滤器
-CAPACITY = 100000000
-ERROR_RATE = 0.001
-FILTER_FIELD = "url"
+# # bloom过滤器
+# CAPACITY = 100000000
+# ERROR_RATE = 0.001
+# FILTER_FIELD = "url"
 # 文件下载响应类型过滤
 # FILE_FILTER_CONTENT_TYPE = ["text/html", "application/xhtml+xml"]

cobweb/utils/dotting.py

@@ -9,7 +9,8 @@ from cobweb import setting
 
 class LoghubDot:
 
-    def __init__(self):
+    def __init__(self, stop):
+        self.stop = stop
         self.client = LogClient(**setting.LOGHUB_CONFIG)
         self.queue = Queue()
 
@@ -27,7 +28,7 @@ class LoghubDot:
         self.queue.push((topic, log_item), direct_insertion=True)
 
     def build_run(self):
-        while True:
+        while not self.stop.is_set():
             start_time = int(time.time())
             while True:
                 cost_time = int(time.time()) - start_time

{cobweb_launcher-3.1.5.dist-info → cobweb_launcher-3.1.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 3.1.5
+Version: 3.1.7
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb_launcher-3.1.5.dist-info → cobweb_launcher-3.1.7.dist-info}/RECORD

@@ -1,6 +1,6 @@
-cobweb/__init__.py,sha256=UfNq1asNq7_a8IKf5WCbk0ju5fxT4wElAyaItf_a-d8,67
+cobweb/__init__.py,sha256=YdBi3uytEFRXan155xU1kKMpiUKUupO2RGeJyXmH0zk,129
 cobweb/constant.py,sha256=zy3XYsc1qp2B76_Fn_hVQ8eGHlPBd3OFlZK2cryE6FY,2839
-cobweb/setting.py,sha256=yP9ZkVShTY4sCZ4DpzI_hO-FMSf0vGrQvkp7UCmZaa0,2338
+cobweb/setting.py,sha256=njtcRjtfUrAc4LwEsSQgBa2sRVpcVd_He3NARnDE_n0,2348
 cobweb/base/__init__.py,sha256=epVQttTHQcux9kAtrGuO9HB_wP74L-pi74vhzc4QOCw,224
 cobweb/base/common_queue.py,sha256=W7PPZZFl52j3Mc916T0imHj7oAUelA6aKJwW-FecDPE,872
 cobweb/base/item.py,sha256=hYheVTV2Bozp4iciJpE2ZwBIXkaqBg4QQkRccP8yoVk,1049
@@ -11,12 +11,12 @@ cobweb/base/seed.py,sha256=A-F1urjbE5hYNWTCwq3sUV4nrxlK_RGMoCmjBmIwYsI,3158
 cobweb/crawlers/__init__.py,sha256=msvkB9mTpsgyj8JfNMsmwAcpy5kWk_2NrO1Adw2Hkw0,29
 cobweb/crawlers/crawler.py,sha256=73WFGVNvIFvH8cP5RfDhAhM-WaFL_mdwnAhNohDnBO0,696
 cobweb/db/__init__.py,sha256=uZwSkd105EAwYo95oZQXAfofUKHVIAZZIPpNMy-hm2Q,56
-cobweb/db/api_db.py,sha256=bDc5dJQxq4z04h70KUTHd0OqUOEY7Cm3wcNJZtTvJIM,3015
+cobweb/db/api_db.py,sha256=kMJhQnhdmCewldG5It0Go8Ge9LBqw0eGwyoynVGeCrQ,3015
 cobweb/db/redis_db.py,sha256=rK6PnKLLVTeg6HMGVtlYl4_b5og3aO2J5jZrqU2Aoso,7721
 cobweb/exceptions/__init__.py,sha256=E9SHnJBbhD7fOgPFMswqyOf8SKRDrI_i25L0bSpohvk,32
 cobweb/exceptions/oss_db_exception.py,sha256=iP_AImjNHT3-Iv49zCFQ3rdLnlvuHa3h2BXApgrOYpA,636
-cobweb/launchers/__init__.py,sha256=XEW4hmBMPKp2bRkHBxxlplXwvKdlLVYBFg3etjtbRXo,222
-cobweb/launchers/distributor.py,sha256=WWLyDaSGMWQWKfZX9guUzIZEwAX9FM5pqOUP3WDfEak,6447
+cobweb/launchers/__init__.py,sha256=6_v2jd2sgj6YnOB1nPKiYBskuXVb5xpQnq2YaDGJgQ8,100
+cobweb/launchers/distributor.py,sha256=-tr3c3O3QBV9UYH1optdBfq76_o83RjII5YKiIYl33w,6420
 cobweb/launchers/launcher.py,sha256=rDT7OQX3wzaS_MsbrkeWDH5IHpN5GE_Oy4tUkvnovKM,5302
 cobweb/launchers/uploader.py,sha256=gJiKMVeoxxwrLsYPaMuXJN7wc5LFxXfjC4VVIlqqlqk,1818
 cobweb/pipelines/__init__.py,sha256=zSUsGtx6smbs2iXBXvYynReKSgky-3gjqaAtKVnA_OU,105
@@ -27,15 +27,15 @@ cobweb/schedulers/__init__.py,sha256=LEya11fdAv0X28YzbQTeC1LQZ156Fj4cyEMGqQHUWW0
 cobweb/schedulers/launcher_air.py,sha256=qdcxq41I9zN5snEpMUUNEQNTtUiZM1Hw_3N9zu4PuAs,3058
 cobweb/schedulers/launcher_api.py,sha256=3-A6k3Igvi-xnvP9M_3NkJKUweDJ_pY10ZrHClteD-g,8628
 cobweb/schedulers/scheduler.py,sha256=mN9XvaOCzNnBWQfzslTIM860ZGq2gyLtxpbVUd0Slqs,2240
-cobweb/schedulers/scheduler_with_redis.py,sha256=LmfQ_uwpXxwvG_i52pDl93T_fxqx3eWg4M2XgDgvmGQ,6397
+cobweb/schedulers/scheduler_with_redis.py,sha256=Qq8iMxbDWJpRml76NfcZyyB8q7tM2Et0YrOcF7nwcP0,6399
 cobweb/utils/__init__.py,sha256=8Bu5iZrIOUMS4jv4hi0inRPtscf6MK0ZFa7gQ7ZFoqw,145
 cobweb/utils/bloom.py,sha256=vng-YbKgh9HbtpAWYf_nkUSbfVTOj40aqUUejRYlsCU,1752
 cobweb/utils/decorators.py,sha256=066JCY_RNMr2mXkhEv8XTtOOKkv9CFiBm0ZNCcC-2ag,1131
-cobweb/utils/dotting.py,sha256=mVICaa26R-dQ4JGmPK-kkR6QjX38QiRewXZnGb2DCIc,1784
+cobweb/utils/dotting.py,sha256=x34WryNKu_o54EzVwAZByagjMTXF0ZP0VYKdjfYEdSM,1833
 cobweb/utils/oss.py,sha256=ktfwMGnq5FMzOkUUS3nNXI7pTdPNinudH3YRJ0nMhoU,3985
 cobweb/utils/tools.py,sha256=5JEaaAwYoV9Sdla2UBIJn6faUBuXmxUMagm9ck6FVqs,1253
-cobweb_launcher-3.1.5.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
-cobweb_launcher-3.1.5.dist-info/METADATA,sha256=qLeaG7MEqh9mQa-XozLLvrdiaKzKyM3XyCnsfQx8fcM,6509
-cobweb_launcher-3.1.5.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
-cobweb_launcher-3.1.5.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
-cobweb_launcher-3.1.5.dist-info/RECORD,,
+cobweb_launcher-3.1.7.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
+cobweb_launcher-3.1.7.dist-info/METADATA,sha256=1cXLkGwLN2Ar4CJGUHNjYOwkx2nAPq5Nz3_H3MK8hGE,6509
+cobweb_launcher-3.1.7.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
+cobweb_launcher-3.1.7.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
+cobweb_launcher-3.1.7.dist-info/RECORD,,