cobweb-launcher 1.2.15__py3-none-any.whl → 1.2.17__py3-none-any.whl

cobweb/__init__.py

@@ -1,2 +1,2 @@
-from .launchers import LauncherAir, LauncherPro
+from .launchers import LauncherAir, LauncherPro, LauncherApi
 from .constant import CrawlerModel

cobweb/db/__init__.py

@@ -1 +1,2 @@
 from .redis_db import RedisDB
+from .api_db import ApiDB

cobweb/db/api_db.py

@@ -0,0 +1,71 @@
+import json
+import os
+import requests
+
+
+class ApiDB:
+
+    def __init__(self, host=None, **kwargs):
+        self.host = host or os.getenv("REDIS_API_HOST", "http://127.0.0.1:4396")
+
+    def _get_response(self, api, params: dict = None):
+        url = self.host + api
+        response = requests.get(url, params=params)
+        json_data = response.json()
+        return json_data["data"]
+
+    def _post_response(self, api, params: dict = None, data: dict = None):
+        url = self.host + api
+        headers = {"Content-Type": "application/json"}
+        response = requests.post(url, headers=headers, params=params, data=json.dumps(data))
+        json_data = response.json()
+        return json_data["data"]
+
+    def get(self, name):
+        return self._get_response(api="/get", params=dict(name=name))
+
+    def setnx(self, name, value=""):
+        return self._get_response(api="/setnx", params=dict(name=name, value=value))
+
+    def setex(self, name, t, value=""):
+        return self._get_response(api="/setex", params=dict(name=name, value=value, t=t))
+
+    def expire(self, name, t, nx: bool = False, xx: bool = False, gt: bool = False, lt: bool = False):
+        return self._get_response(api="/expire", params=dict(name=name, t=t, nx=nx, xx=xx, gt=gt, lt=lt))
+
+    def ttl(self, name):
+        return self._get_response(api="/ttl", params=dict(name=name))
+
+    def delete(self, name):
+        return self._get_response(api="/delete", params=dict(name=name))
+
+    def exists(self, name):
+        return self._get_response(api="/exists", params=dict(name=name))
+
+    def incrby(self, name, value):
+        return self._get_response(api="/incrby", params=dict(name=name, value=value))
+
+    def zcard(self, name) -> bool:
+        return self._get_response(api="/zcard", params=dict(name=name))
+
+    def zadd(self, name, item: dict, **kwargs):
+        return self._post_response(api="/zadd", data=dict(name=name, mapping=item, **kwargs))
+
+    def zrem(self, name, *values):
+        return self._post_response(api="/zrem", data=dict(name=name, values=values))
+
+    def zcount(self, name, _min, _max):
+        return self._get_response(api="/zcount", params=dict(name=name, min=_min, max=_max))
+
+    def lock(self, name, t=15) -> bool:
+        return self._get_response(api="/lock", params=dict(name=name, t=t))
+
+    def members(self, name, score, start=0, count=5000, _min="-inf", _max="+inf"):
+        return self._get_response(api="/members", params=dict(name=name, score=score, start=start, count=count, min=_min, max=_max))
+
+    def done(self, name: list, *values):
+        return self._post_response(api="/done", data=dict(name=name, values=values))
+
+
+
+

cobweb/launchers/__init__.py

@@ -1,2 +1,3 @@
 from .launcher_air import LauncherAir
 from .launcher_pro import LauncherPro
+from .launcher_api import LauncherApi

cobweb/launchers/launcher.py

@@ -97,7 +97,8 @@ class Launcher(threading.Thread):
         self._spider_max_retries = setting.SPIDER_MAX_RETRIES
         self._spider_thread_num = setting.SPIDER_THREAD_NUM
         self._spider_time_sleep = setting.SPIDER_TIME_SLEEP
-        self._spider_max_speed = setting.SPIDER_MAX_SPEED
+        self._spider_max_count = setting.SPIDER_MAX_COUNT
+        self._time_window = setting.TIME_WINDOW
 
         self._done_model = setting.DONE_MODEL
         self._task_model = setting.TASK_MODEL

cobweb/launchers/launcher_api.py

@@ -0,0 +1,225 @@
+import time
+import threading
+
+from cobweb.db import ApiDB
+from cobweb.base import Seed, logger
+from cobweb.constant import DealModel, LogTemplate
+from .launcher import Launcher, check_pause
+
+
+class LauncherApi(Launcher):
+
+    def __init__(self, task, project, custom_setting=None, **kwargs):
+        super().__init__(task, project, custom_setting, **kwargs)
+        self._db = ApiDB()
+
+        self._todo_key = "{%s:%s}:todo" % (project, task)
+        self._done_key = "{%s:%s}:done" % (project, task)
+        self._fail_key = "{%s:%s}:fail" % (project, task)
+        self._heartbeat_key = "heartbeat:%s_%s" % (project, task)
+
+        self._statistics_done_key = "statistics:%s:%s:done" % (project, task)
+        self._statistics_fail_key = "statistics:%s:%s:fail" % (project, task)
+        self._speed_control_key = "speed_control:%s_%s" % (project, task)
+
+        self._reset_lock_key = "lock:reset:%s_%s" % (project, task)
+
+        # self._bf_key = "bloom_%s_%s" % (project, task)
+        # self._bf = BloomFilter(self._bf_key)
+
+        self._heartbeat_start_event = threading.Event()
+        self._redis_queue_empty_event = threading.Event()
+
+    @property
+    def heartbeat(self):
+        return self._db.exists(self._heartbeat_key)
+
+    def statistics(self, key, count):
+        if not self._task_model and not self._db.exists(key):
+            self._db.setex(key, 86400 * 30, int(count))
+        else:
+            self._db.incrby(key, count)
+
+    def _get_seed(self) -> Seed:
+        """
+        从队列中获取种子（频控）
+        设置时间窗口为self._time_window（秒），判断在该窗口内的采集量是否满足阈值（self._spider_max_speed）
+        :return: True -> 种子, False -> None
+        """
+        spider_speed = self._db.get(self._speed_control_key)
+        if int(spider_speed or 0) > self._spider_max_count:
+            expire_time = self._db.ttl(self._speed_control_key)
+            if expire_time == -1:
+                self._db.delete(self._speed_control_key)
+            elif expire_time and expire_time > 0:
+                logger.info(f"Too fast! Please wait {expire_time} seconds...")
+                time.sleep(expire_time / 2)
+            return None
+        seed = self.__LAUNCHER_QUEUE__["todo"].pop()
+        if seed and not self._db.lock(self._speed_control_key, t=self._time_window):
+            self._db.incrby(self._speed_control_key, 1)
+        return seed
+
+    @check_pause
+    def _execute_heartbeat(self):
+        if self._heartbeat_start_event.is_set():
+            self._db.setex(self._heartbeat_key, 5)
+        time.sleep(3)
+
+    @check_pause
+    def _reset(self):
+        """
+        检查过期种子，重新添加到redis缓存中
+        """
+        reset_wait_seconds = 30
+        if self._db.lock(self._reset_lock_key, t=120):
+
+            _min = -int(time.time()) + self._seed_reset_seconds \
+                if self.heartbeat else "-inf"
+
+            self._db.members(self._todo_key, 0, _min=_min, _max="(0")
+            self._db.delete(self._reset_lock_key)
+
+            if not self.heartbeat:
+                self._heartbeat_start_event.set()
+
+        time.sleep(reset_wait_seconds)
+
+    @check_pause
+    def _scheduler(self):
+        """
+        调度任务，获取redis队列种子，同时添加到doing字典中
+        """
+        if not self._db.zcount(self._todo_key, 0, "(1000"):
+            time.sleep(self._scheduler_wait_seconds)
+        elif self.__LAUNCHER_QUEUE__['todo'].length >= self._todo_queue_size:
+            time.sleep(self._todo_queue_full_wait_seconds)
+        else:
+            members = self._db.members(
+                self._todo_key, int(time.time()),
+                count=self._todo_queue_size,
+                _min=0, _max="(1000"
+            )
+            for member, priority in members:
+                seed = Seed(member, priority=priority)
+                self.__LAUNCHER_QUEUE__['todo'].push(seed)
+                self.__DOING__[seed.to_string] = seed.params.priority
+
+    @check_pause
+    def _insert(self):
+        """
+        添加新种子到redis队列中
+        """
+        seeds = {}
+        status = self.__LAUNCHER_QUEUE__['new'].length < self._new_queue_max_size
+        for _ in range(self._new_queue_max_size):
+            seed = self.__LAUNCHER_QUEUE__['new'].pop()
+            if seed:
+                seeds[seed.to_string] = seed.params.priority
+        if seeds:
+            self._db.zadd(self._todo_key, seeds, nx=True)
+        if status:
+            time.sleep(self._new_queue_wait_seconds)
+
+    @check_pause
+    def _refresh(self):
+        """
+        刷新doing种子过期时间，防止reset重新消费
+        """
+        if self.__DOING__:
+            refresh_time = int(time.time())
+            seeds = {k:-refresh_time - v / 1000 for k, v in self.__DOING__.items()}
+            self._db.zadd(self._todo_key, item=seeds, xx=True)
+        time.sleep(15)
+
+    @check_pause
+    def _delete(self):
+        """
+        删除队列种子，根据状态添加至成功或失败队列，移除doing字典种子索引
+        """
+        seed_info = {"count": 0, "failed": [], "succeed": [], "common": []}
+        status = self.__LAUNCHER_QUEUE__['done'].length < self._done_queue_max_size
+
+        for _ in range(self._done_queue_max_size):
+            seed = self.__LAUNCHER_QUEUE__['done'].pop()
+            if not seed:
+                break
+            if seed.params.seed_status == DealModel.fail:
+                seed_info["failed"].append(seed.to_string)
+            elif self._done_model == 1:
+                seed_info["succeed"].append(seed.to_string)
+            else:
+                seed_info["common"].append(seed.to_string)
+            seed_info['count'] += 1
+
+        if seed_info["count"]:
+
+            succeed_count = self._db.zrem(self._todo_key, *seed_info["common"])
+            succeed_count += self._db.done([self._todo_key, self._done_key], *seed_info["succeed"])
+            failed_count = self._db.done([self._todo_key, self._fail_key], *seed_info["failed"])
+
+            if failed_count:
+                self.statistics(self._statistics_fail_key, failed_count)
+            if succeed_count:
+                self.statistics(self._statistics_done_key, succeed_count)
+
+            self._remove_doing_seeds(seed_info["common"] + seed_info["succeed"] + seed_info["failed"])
+
+        if status:
+            time.sleep(self._done_queue_wait_seconds)
+
+    def _polling(self):
+        wait_scheduler_execute = True
+        check_emtpy_times = 0
+        while not self._stop.is_set():
+            queue_not_empty_count = 0
+            pooling_wait_seconds = 30
+
+            for q in self.__LAUNCHER_QUEUE__.values():
+                if q.length != 0:
+                    queue_not_empty_count += 1
+                    wait_scheduler_execute = False
+
+            if queue_not_empty_count == 0:
+                pooling_wait_seconds = 3
+                if self._pause.is_set():
+                    check_emtpy_times = 0
+                    if not self._task_model and (
+                            not wait_scheduler_execute or
+                            int(time.time()) - self._app_time > self._before_scheduler_wait_seconds
+                    ):
+                        logger.info("Done! ready to close thread...")
+                        self._stop.set()
+
+                    elif self._db.zcount(self._todo_key, _min=0, _max="(1000"):
+                        logger.info(f"Recovery {self.task} task run！")
+                        self._pause.clear()
+                        self._execute()
+                    else:
+                        logger.info("pause! waiting for resume...")
+                elif check_emtpy_times > 2:
+                    self.__DOING__ = {}
+                    if not self._db.zcount(self._todo_key, _min="-inf", _max="(1000"):
+                        self._pause.set()
+                else:
+                    logger.info(
+                        "check whether the task is complete, "
+                        f"reset times {3 - check_emtpy_times}"
+                    )
+                    check_emtpy_times += 1
+            else:
+                logger.info(LogTemplate.launcher_pro_polling.format(
+                    task=self.task,
+                    doing_len=len(self.__DOING__.keys()),
+                    todo_len=self.__LAUNCHER_QUEUE__['todo'].length,
+                    done_len=self.__LAUNCHER_QUEUE__['done'].length,
+                    redis_seed_count=self._db.zcount(self._todo_key, "-inf", "+inf"),
+                    redis_todo_len=self._db.zcount(self._todo_key, 0, "(1000"),
+                    redis_doing_len=self._db.zcount(self._todo_key, "-inf", "(0"),
+                    upload_len=self.__LAUNCHER_QUEUE__['upload'].length,
+                ))
+
+            time.sleep(pooling_wait_seconds)
+
+        logger.info("Done! Ready to close thread...")
+

cobweb/launchers/launcher_pro.py

@@ -44,7 +44,7 @@ class LauncherPro(Launcher):
 
     def _get_seed(self) -> Seed:
         spider_speed = self._db._client.get(self._speed_control_key)
-        if int(spider_speed or 0) > self._spider_max_speed:
+        if int(spider_speed or 0) > self._spider_max_count:
             expire_time = self._db.ttl(self._speed_control_key)
             if expire_time == -1:
                 self._db.delete(self._speed_control_key)
@@ -53,7 +53,7 @@ class LauncherPro(Launcher):
                 time.sleep(expire_time / 2)
             return None
         seed = self.__LAUNCHER_QUEUE__["todo"].pop()
-        if seed and not self._db.lock(self._speed_control_key, t=60):
+        if seed and not self._db.lock(self._speed_control_key, t=self._time_window):
             self._db._client.incrby(self._speed_control_key, 1)
         return seed
 

cobweb/setting.py

@@ -58,7 +58,9 @@ DONE_MODEL = 0   # 0:种子消费成功直接从队列移除，失败则添加
 SPIDER_THREAD_NUM = 10
 SPIDER_MAX_RETRIES = 5
 SPIDER_TIME_SLEEP = 10
-SPIDER_MAX_SPEED = 1000   # 一分钟最大采集数
+
+SPIDER_MAX_COUNT = 1000   # 在规定时间窗口内最大采集数
+TIME_WINDOW = 60   # 频控固定时间窗口（秒）
 
 # 任务模式
 TASK_MODEL = 0  # 0:单次，1:常驻

{cobweb_launcher-1.2.15.dist-info → cobweb_launcher-1.2.17.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.2.15
+Version: 1.2.17
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb_launcher-1.2.15.dist-info → cobweb_launcher-1.2.17.dist-info}/RECORD

@@ -1,6 +1,6 @@
-cobweb/__init__.py,sha256=uMHyf4Fekbyw2xBCbkA8R0LwCpBJf5p_7pWbh60ZWYk,83
+cobweb/__init__.py,sha256=CBd2oByCfc5EmH2dCZYVHkxXYZG-oWrLyTtZU5sEoP0,96
 cobweb/constant.py,sha256=zy3XYsc1qp2B76_Fn_hVQ8eGHlPBd3OFlZK2cryE6FY,2839
-cobweb/setting.py,sha256=MGe4QGnE5XOTh9Z7NhakaTFK7f-lZtzlA9PFcuc1qoY,2145
+cobweb/setting.py,sha256=47HZsw40HLpsmOmvij1lyQALPQQCN_tWlKZ0wbn2MtM,2216
 cobweb/base/__init__.py,sha256=4gwWWQ0Q8cYG9cD7Lwf4XMqRGc5M_mapS3IczR6zeCE,222
 cobweb/base/common_queue.py,sha256=W7PPZZFl52j3Mc916T0imHj7oAUelA6aKJwW-FecDPE,872
 cobweb/base/decorators.py,sha256=wDCaQ94aAZGxks9Ljc0aXq6omDXT1_yzFy83ZW6VbVI,930
@@ -13,14 +13,16 @@ cobweb/crawlers/__init__.py,sha256=msvkB9mTpsgyj8JfNMsmwAcpy5kWk_2NrO1Adw2Hkw0,2
 cobweb/crawlers/base_crawler.py,sha256=ee_WSDnPQpPTk6wlFuY2UEx5L3hcsAZFcr6i3GLSry8,5751
 cobweb/crawlers/crawler.py,sha256=xiFNM0t69f5xlm59hPbO2MpqtdirVAUhD84-CLpyHPM,6349
 cobweb/crawlers/file_crawler.py,sha256=2Sjbdgxzqd41WykKUQE3QQlGai3T8k-pmHNmPlTchjQ,4454
-cobweb/db/__init__.py,sha256=ut0iEyBLjcJL06WNG_5_d4hO5PJWvDrKWMkDOdmgh2M,30
+cobweb/db/__init__.py,sha256=uZwSkd105EAwYo95oZQXAfofUKHVIAZZIPpNMy-hm2Q,56
+cobweb/db/api_db.py,sha256=53-avN4zkqWLxmRRils8K3RFx2lBG5jNOuNFOHDMCD8,2663
 cobweb/db/redis_db.py,sha256=fumNZJiio-uQqRcSrymx8eJ1PqsdOwITe_Y-9JOXxrQ,4298
 cobweb/exceptions/__init__.py,sha256=E9SHnJBbhD7fOgPFMswqyOf8SKRDrI_i25L0bSpohvk,32
 cobweb/exceptions/oss_db_exception.py,sha256=iP_AImjNHT3-Iv49zCFQ3rdLnlvuHa3h2BXApgrOYpA,636
-cobweb/launchers/__init__.py,sha256=af0Y6wrGX8SQZ7w7XL2sOtREjCT3dwad-uCc3nIontY,76
-cobweb/launchers/launcher.py,sha256=AbkrytfJEyj8FhTbLgjmOOIuvOYV3cpVknE9yt31WbM,6930
+cobweb/launchers/__init__.py,sha256=qMuVlQcjErVK67HyKFZEsXf_rfZD5ODjx1QucSCKMOM,114
+cobweb/launchers/launcher.py,sha256=SK4f3Fpuv-QMMriHruXGQ1sh1lxT1DZ2PdG0p2wAzNw,6978
 cobweb/launchers/launcher_air.py,sha256=KAk_M8F3029cXYe7m4nn3Nzyi89lbxJ2cqZjqW8iZ0E,2832
-cobweb/launchers/launcher_pro.py,sha256=8QKhToKoD2WonIaqRQAhUWRhbNOIgYXzGFRK1id_3yM,8638
+cobweb/launchers/launcher_api.py,sha256=Gcaj38_CdKN6dqzju9NtWBAwbm7KM4J2BG0STXnIh8Y,8877
+cobweb/launchers/launcher_pro.py,sha256=2JdN4khAFGBWLXDKaWknjG72XBhfTAnwtrgVmkF0K3M,8653
 cobweb/pipelines/__init__.py,sha256=zSUsGtx6smbs2iXBXvYynReKSgky-3gjqaAtKVnA_OU,105
 cobweb/pipelines/base_pipeline.py,sha256=fYnWf79GmhufXpcnMa3te18SbmnVeYLwxfyo-zLd9CY,1577
 cobweb/pipelines/loghub_pipeline.py,sha256=cjPO6w6UJ0jNw2fVvdX0BCdlm58T7dmYXlxzXOBpvfY,1027
@@ -31,8 +33,8 @@ cobweb/utils/__init__.py,sha256=vBtZTy3EfRE0MmH43URhmr7nw6_oOWTEbGOM9xR_9o8,78
 cobweb/utils/bloom.py,sha256=vng-YbKgh9HbtpAWYf_nkUSbfVTOj40aqUUejRYlsCU,1752
 cobweb/utils/oss.py,sha256=gyt8-UB07tVphZLQXMOf-JTJwU-mWq8KZkOXKkAf3uk,3513
 cobweb/utils/tools.py,sha256=5JEaaAwYoV9Sdla2UBIJn6faUBuXmxUMagm9ck6FVqs,1253
-cobweb_launcher-1.2.15.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
-cobweb_launcher-1.2.15.dist-info/METADATA,sha256=rP73tkfQPQB8MCEYg31ZOdEbBYUUoyXzAcUevfnwqec,6510
-cobweb_launcher-1.2.15.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
-cobweb_launcher-1.2.15.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
-cobweb_launcher-1.2.15.dist-info/RECORD,,
+cobweb_launcher-1.2.17.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
+cobweb_launcher-1.2.17.dist-info/METADATA,sha256=z1BLGBzB7XsJtv15rxaT_IZRdeGNevCtMXdpzzLKB1k,6510
+cobweb_launcher-1.2.17.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
+cobweb_launcher-1.2.17.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
+cobweb_launcher-1.2.17.dist-info/RECORD,,