cobweb-launcher 3.1.4__py3-none-any.whl → 3.1.6__py3-none-any.whl

cobweb/__init__.py

@@ -1,2 +1,4 @@
 from .launchers import Launcher
 from .constant import CrawlerModel
+from .pipelines import Pipeline
+from .crawlers import Crawler

cobweb/db/api_db.py

@@ -71,7 +71,7 @@ class ApiDB:
     def auto_incr(self, name, t=15, limit=1000) -> bool:
         return self._get_response(api="/auto_incr", params=dict(name=name, t=t, limit=limit))
 
-    def members(self, name, score, start=0, count=5000, _min="-inf", _max="+inf"):
+    def members(self, name, score, start=0, count=1000, _min="-inf", _max="+inf"):
         return self._get_response(api="/members", params=dict(name=name, score=score, start=start, count=count, min=_min, max=_max))
 
     def done(self, name: list, *values):

cobweb/launchers/__init__.py

@@ -1,8 +1,3 @@
-# from .launcher_air import LauncherAir
-# from .launcher_pro import LauncherPro
-# from .launcher_api import LauncherApi
-
-
 from .launcher import Launcher
 from .uploader import Uploader
 from .distributor import Distributor

cobweb/launchers/distributor.py

@@ -1,22 +1,16 @@
 import time
 import threading
 import traceback
-from inspect import isgenerator
+
 from typing import Callable
+from inspect import isgenerator
 from urllib.parse import urlparse
 from requests import Response as Res
 
 from cobweb import setting
 from cobweb.constant import DealModel, LogTemplate
-from cobweb.base import (
-    Seed,
-    Queue,
-    BaseItem,
-    Request,
-    Response,
-    logger
-)
 from cobweb.utils import LoghubDot, check_pause
+from cobweb.base import Seed, Queue, BaseItem, Request, Response, logger
 
 
 class Distributor(threading.Thread):

cobweb/launchers/uploader.py

@@ -27,7 +27,7 @@ class Uploader(threading.Thread):
         self.upload_size = setting.UPLOAD_QUEUE_MAX_SIZE
         self.wait_seconds = setting.UPLOAD_QUEUE_WAIT_SECONDS
 
-        self.Pipeline = SpiderPipeline
+        self.pipeline = SpiderPipeline()
 
         logger.debug(f"Uploader instance attrs: {self.__dict__}")
 
@@ -45,11 +45,11 @@ class Uploader(threading.Thread):
                 if not item:
                     break
                 seeds.append(item.seed)
-                data = self.Pipeline.build(item)
+                data = self.pipeline.build(item)
                 data_info.setdefault(item.table, []).append(data)
             for table, datas in data_info.items():
                 try:
-                    self.Pipeline.upload(table, datas)
+                    self.pipeline.upload(table, datas)
                 except Exception as e:
                     logger.info(e)
         except Exception as e:

cobweb/schedulers/scheduler_with_redis.py

@@ -7,7 +7,7 @@ from cobweb.utils import check_pause
 from cobweb.base import Queue, Seed, logger
 from cobweb.constant import LogTemplate
 from .scheduler import Scheduler
-use_api = bool(int(os.getenv("REDIS_API", 0)))
+use_api = bool(int(os.getenv("REDIS_API_HOST", 0)))
 
 
 class RedisScheduler(Scheduler):
@@ -153,10 +153,10 @@ class RedisScheduler(Scheduler):
                 else:
                     if all_count:
                         logger.info(f"todo seeds count: {todo_count}, queue length: {all_count}")
-                        self.pause.is_set()
+                        self.pause.clear()
                     else:
                         logger.info("Done! pause set...")
-                        self.pause.clear()
+                        self.pause.set()
             else:
                 if self.pause.is_set():
                     self.pause.clear()

cobweb/setting.py

@@ -18,7 +18,7 @@ LOGHUB_CONFIG = {
     "accessKey": os.getenv("LOGHUB_SECRET_KEY")
 }
 
-# oss util config
+# # oss util config
 OSS_BUCKET = os.getenv("OSS_BUCKET")
 OSS_ENDPOINT = os.getenv("OSS_ENDPOINT")
 OSS_ACCESS_KEY = os.getenv("OSS_ACCESS_KEY")
@@ -72,9 +72,9 @@ TASK_MODEL = 0  # 0:单次，1:常驻
 # 流控措施
 SPEED_CONTROL = 1  # 0:关闭，1:开启
 
-# bloom过滤器
-CAPACITY = 100000000
-ERROR_RATE = 0.001
-FILTER_FIELD = "url"
+# # bloom过滤器
+# CAPACITY = 100000000
+# ERROR_RATE = 0.001
+# FILTER_FIELD = "url"
 # 文件下载响应类型过滤
 # FILE_FILTER_CONTENT_TYPE = ["text/html", "application/xhtml+xml"]

{cobweb_launcher-3.1.4.dist-info → cobweb_launcher-3.1.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 3.1.4
+Version: 3.1.6
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb_launcher-3.1.4.dist-info → cobweb_launcher-3.1.6.dist-info}/RECORD

@@ -1,6 +1,6 @@
-cobweb/__init__.py,sha256=UfNq1asNq7_a8IKf5WCbk0ju5fxT4wElAyaItf_a-d8,67
+cobweb/__init__.py,sha256=YdBi3uytEFRXan155xU1kKMpiUKUupO2RGeJyXmH0zk,129
 cobweb/constant.py,sha256=zy3XYsc1qp2B76_Fn_hVQ8eGHlPBd3OFlZK2cryE6FY,2839
-cobweb/setting.py,sha256=yP9ZkVShTY4sCZ4DpzI_hO-FMSf0vGrQvkp7UCmZaa0,2338
+cobweb/setting.py,sha256=njtcRjtfUrAc4LwEsSQgBa2sRVpcVd_He3NARnDE_n0,2348
 cobweb/base/__init__.py,sha256=epVQttTHQcux9kAtrGuO9HB_wP74L-pi74vhzc4QOCw,224
 cobweb/base/common_queue.py,sha256=W7PPZZFl52j3Mc916T0imHj7oAUelA6aKJwW-FecDPE,872
 cobweb/base/item.py,sha256=hYheVTV2Bozp4iciJpE2ZwBIXkaqBg4QQkRccP8yoVk,1049
@@ -11,14 +11,14 @@ cobweb/base/seed.py,sha256=A-F1urjbE5hYNWTCwq3sUV4nrxlK_RGMoCmjBmIwYsI,3158
 cobweb/crawlers/__init__.py,sha256=msvkB9mTpsgyj8JfNMsmwAcpy5kWk_2NrO1Adw2Hkw0,29
 cobweb/crawlers/crawler.py,sha256=73WFGVNvIFvH8cP5RfDhAhM-WaFL_mdwnAhNohDnBO0,696
 cobweb/db/__init__.py,sha256=uZwSkd105EAwYo95oZQXAfofUKHVIAZZIPpNMy-hm2Q,56
-cobweb/db/api_db.py,sha256=bDc5dJQxq4z04h70KUTHd0OqUOEY7Cm3wcNJZtTvJIM,3015
+cobweb/db/api_db.py,sha256=kMJhQnhdmCewldG5It0Go8Ge9LBqw0eGwyoynVGeCrQ,3015
 cobweb/db/redis_db.py,sha256=rK6PnKLLVTeg6HMGVtlYl4_b5og3aO2J5jZrqU2Aoso,7721
 cobweb/exceptions/__init__.py,sha256=E9SHnJBbhD7fOgPFMswqyOf8SKRDrI_i25L0bSpohvk,32
 cobweb/exceptions/oss_db_exception.py,sha256=iP_AImjNHT3-Iv49zCFQ3rdLnlvuHa3h2BXApgrOYpA,636
-cobweb/launchers/__init__.py,sha256=XEW4hmBMPKp2bRkHBxxlplXwvKdlLVYBFg3etjtbRXo,222
-cobweb/launchers/distributor.py,sha256=WWLyDaSGMWQWKfZX9guUzIZEwAX9FM5pqOUP3WDfEak,6447
+cobweb/launchers/__init__.py,sha256=6_v2jd2sgj6YnOB1nPKiYBskuXVb5xpQnq2YaDGJgQ8,100
+cobweb/launchers/distributor.py,sha256=-tr3c3O3QBV9UYH1optdBfq76_o83RjII5YKiIYl33w,6420
 cobweb/launchers/launcher.py,sha256=rDT7OQX3wzaS_MsbrkeWDH5IHpN5GE_Oy4tUkvnovKM,5302
-cobweb/launchers/uploader.py,sha256=8zKbUio7aPn63-aC1v7PASzRzc2JX7d59RqeiFw4zMY,1816
+cobweb/launchers/uploader.py,sha256=gJiKMVeoxxwrLsYPaMuXJN7wc5LFxXfjC4VVIlqqlqk,1818
 cobweb/pipelines/__init__.py,sha256=zSUsGtx6smbs2iXBXvYynReKSgky-3gjqaAtKVnA_OU,105
 cobweb/pipelines/pipeline.py,sha256=qwoOYMhlAB-MnEmMNpNeauTHoRTOr2wyBDYS4MF6B1c,261
 cobweb/pipelines/pipeline_console.py,sha256=NEh-4zhuVAQOqwXLsqeb-rcNZ9_KXFUpL3otUTL5qBs,754
@@ -27,15 +27,15 @@ cobweb/schedulers/__init__.py,sha256=LEya11fdAv0X28YzbQTeC1LQZ156Fj4cyEMGqQHUWW0
 cobweb/schedulers/launcher_air.py,sha256=qdcxq41I9zN5snEpMUUNEQNTtUiZM1Hw_3N9zu4PuAs,3058
 cobweb/schedulers/launcher_api.py,sha256=3-A6k3Igvi-xnvP9M_3NkJKUweDJ_pY10ZrHClteD-g,8628
 cobweb/schedulers/scheduler.py,sha256=mN9XvaOCzNnBWQfzslTIM860ZGq2gyLtxpbVUd0Slqs,2240
-cobweb/schedulers/scheduler_with_redis.py,sha256=LmfQ_uwpXxwvG_i52pDl93T_fxqx3eWg4M2XgDgvmGQ,6397
+cobweb/schedulers/scheduler_with_redis.py,sha256=Qq8iMxbDWJpRml76NfcZyyB8q7tM2Et0YrOcF7nwcP0,6399
 cobweb/utils/__init__.py,sha256=8Bu5iZrIOUMS4jv4hi0inRPtscf6MK0ZFa7gQ7ZFoqw,145
 cobweb/utils/bloom.py,sha256=vng-YbKgh9HbtpAWYf_nkUSbfVTOj40aqUUejRYlsCU,1752
 cobweb/utils/decorators.py,sha256=066JCY_RNMr2mXkhEv8XTtOOKkv9CFiBm0ZNCcC-2ag,1131
 cobweb/utils/dotting.py,sha256=mVICaa26R-dQ4JGmPK-kkR6QjX38QiRewXZnGb2DCIc,1784
 cobweb/utils/oss.py,sha256=ktfwMGnq5FMzOkUUS3nNXI7pTdPNinudH3YRJ0nMhoU,3985
 cobweb/utils/tools.py,sha256=5JEaaAwYoV9Sdla2UBIJn6faUBuXmxUMagm9ck6FVqs,1253
-cobweb_launcher-3.1.4.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
-cobweb_launcher-3.1.4.dist-info/METADATA,sha256=ZkU5297omwKNvX0bdLBCamER88JZ647bflowb9LWD70,6509
-cobweb_launcher-3.1.4.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
-cobweb_launcher-3.1.4.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
-cobweb_launcher-3.1.4.dist-info/RECORD,,
+cobweb_launcher-3.1.6.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
+cobweb_launcher-3.1.6.dist-info/METADATA,sha256=1hnno3Ujgp1nUQP2VTHAMy-VyUFMx09qRYjo4O4Iz3E,6509
+cobweb_launcher-3.1.6.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
+cobweb_launcher-3.1.6.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
+cobweb_launcher-3.1.6.dist-info/RECORD,,