cobweb-launcher 1.3.14__tar.gz → 3.1.0__tar.gz

{cobweb-launcher-1.3.14/cobweb_launcher.egg-info → cobweb-launcher-3.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.3.14
+Version: 3.1.0
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

cobweb-launcher-3.1.0/cobweb/__init__.py

@@ -0,0 +1,2 @@
+from .launchers import Launcher
+from .constant import CrawlerModel

cobweb-launcher-3.1.0/cobweb/base/__init__.py

@@ -0,0 +1,9 @@
+from .common_queue import Queue
+from .response import Response
+from .request import Request
+from .item import BaseItem, ConsoleItem
+from .seed import Seed
+
+from .log import logger
+# from .decorators import decorator_oss_db
+

{cobweb-launcher-1.3.14 → cobweb-launcher-3.1.0}/cobweb/base/common_queue.py

@@ -1,4 +1,3 @@
-import time
 from collections import deque
 
 
@@ -29,15 +28,3 @@ class Queue:
             return None
         except AttributeError:
             return None
-
-    def clear(self):
-        self._queue.clear()
-
-    def get(self):
-        try:
-            yield self._queue.popleft()
-        except IndexError:
-            time.sleep(1)
-            yield None
-        except AttributeError:
-            yield None

{cobweb-launcher-1.3.14 → cobweb-launcher-3.1.0}/cobweb/base/request.py

@@ -1,4 +1,3 @@
-import json
 import random
 import requests
 
@@ -31,6 +30,7 @@ class Request:
             **kwargs
     ):
         self.url = url
+        self.seed = seed
         self.check_status_code = check_status_code
         self.request_setting = {}
 
@@ -46,12 +46,6 @@ class Request:
         if random_ua:
             self._build_header()
 
-        if isinstance(seed, Seed):
-            self.seed = seed.to_string
-        else:
-            kwargs.update(**seed.to_dict)
-            self.seed = self.to_string
-
     @property
     def _random_ua(self) -> str:
         v1 = random.randint(4, 15)
@@ -79,16 +73,10 @@ class Request:
     @property
     def to_dict(self):
         _dict = self.__dict__.copy()
+        _dict.pop('url')
         _dict.pop('seed')
         _dict.pop('check_status_code')
         _dict.pop('request_setting')
         return _dict
 
-    @property
-    def to_string(self) -> str:
-        return json.dumps(
-            self.to_dict,
-            ensure_ascii=False,
-            separators=(",", ":")
-        )
 

{cobweb-launcher-1.3.14 → cobweb-launcher-3.1.0}/cobweb/base/seed.py

@@ -5,11 +5,13 @@ import hashlib
 
 class SeedParams:
 
-    def __init__(self, retry, priority, seed_version, seed_status=None):
+    def __init__(self, retry, priority, seed_version, seed_status=None, proxy_type=None, proxy=None):
         self.retry = retry or 0
         self.priority = priority or 300
         self.seed_version = seed_version or int(time.time())
         self.seed_status = seed_status
+        self.proxy_type = proxy_type
+        self.proxy = proxy
 
 
 class Seed:
@@ -18,7 +20,9 @@ class Seed:
         "retry",
         "priority",
         "seed_version",
-        "seed_status"
+        "seed_status",
+        "proxy_type",
+        "proxy"
     ]
 
     def __init__(
@@ -29,6 +33,8 @@ class Seed:
             priority=None,
             seed_version=None,
             seed_status=None,
+            proxy_type=None,
+            proxy=None,
             **kwargs
     ):
         if any(isinstance(seed, t) for t in (str, bytes)):
@@ -51,6 +57,8 @@ class Seed:
             "priority": priority,
             "seed_version": seed_version,
             "seed_status": seed_status,
+            "proxy_type": proxy_type,
+            "proxy": proxy
         }
 
         if kwargs:
@@ -104,15 +112,11 @@ class Seed:
             separators=(",", ":")
         )
 
-    # @property
-    # def get_all(self):
-    #     return json.dumps(
-    #         self.__dict__,
-    #         ensure_ascii=False,
-    #         separators=(",", ":")
-    #     )
-
     @property
-    def seed(self):
-        return self.to_string
+    def get_all(self):
+        return json.dumps(
+            self.__dict__,
+            ensure_ascii=False,
+            separators=(",", ":")
+        )
 

{cobweb-launcher-1.3.14 → cobweb-launcher-3.1.0}/cobweb/constant.py

@@ -37,22 +37,6 @@ class LogTemplate:
     ----------------------- end  - console pipeline ------------------
     """
 
-    launcher_polling = """
-    ----------------------- start - 轮训日志: {task} -----------------
-            正在运行任务
-                构造请求任务数: {memory_todo_count}
-                正在下载任务数: {memory_download_count}
-            任务内存队列 
-                待构造请求队列:  {todo_queue_len}
-                待删除请求队列:  {delete_queue_len}
-                待进行下载队列:  {request_queue_len}
-                待解析响应队列:  {response_queue_len}
-                待删除下载队列:  {done_queue_len}
-            存储队列 
-               待上传数据队列:  {upload_queue_len}
-    ----------------------- end  - 轮训日志: {task} ------------------
-    """
-
     launcher_air_polling = """
     ----------------------- start - 轮训日志: {task} -----------------
             内存队列 

cobweb-launcher-3.1.0/cobweb/crawlers/crawler.py

@@ -0,0 +1,28 @@
+import json
+from typing import Union
+from cobweb.base import (
+    Seed,
+    BaseItem,
+    Request,
+    Response,
+    ConsoleItem,
+)
+
+
+class Crawler:
+
+    @staticmethod
+    def request(seed: Seed) -> Union[Request, BaseItem]:
+        yield Request(seed.url, seed, timeout=5)
+
+    @staticmethod
+    def download(item: Request) -> Union[Seed, BaseItem, Response, str]:
+        response = item.download()
+        yield Response(item.seed, response, **item.to_dict)
+
+    @staticmethod
+    def parse(item: Response) -> BaseItem:
+        upload_item = item.to_dict
+        upload_item["text"] = item.response.text
+        yield ConsoleItem(item.seed, data=json.dumps(upload_item, ensure_ascii=False))
+

cobweb-launcher-3.1.0/cobweb/db/redis_db.py

@@ -0,0 +1,215 @@
+import redis
+import time
+from cobweb import setting
+from redis.exceptions import ConnectionError, TimeoutError
+
+
+class RedisDB:
+    def __init__(self, **kwargs):
+        redis_config = kwargs or setting.REDIS_CONFIG
+        self.host = redis_config['host']
+        self.password = redis_config['password']
+        self.port = redis_config['port']
+        self.db = redis_config['db']
+
+        self.max_retries = 5
+        self.retry_delay = 5
+        self.client = None
+        self.connect()
+
+    def connect(self):
+        """尝试连接 Redis"""
+        retries = 0
+        while retries < self.max_retries:
+            try:
+                self.client = redis.Redis(
+                    host=self.host,
+                    port=self.port,
+                    password=self.password,
+                    db=self.db,
+                    socket_timeout=5,  # 设置连接超时时间
+                    socket_connect_timeout=5  # 设置连接超时时间
+                )
+                # 测试连接是否成功
+                self.client.ping()
+                return
+            except (ConnectionError, TimeoutError) as e:
+                retries += 1
+                if retries < self.max_retries:
+                    time.sleep(self.retry_delay)
+                else:
+                    raise Exception("达到最大重试次数，无法连接 Redis")
+
+    def is_connected(self):
+        try:
+            self.client.ping()
+            return True
+        except (ConnectionError, TimeoutError):
+            return False
+
+    def reconnect(self):
+        self.connect()
+
+    def execute_command(self, command, *args, **kwargs):
+        retries = 0
+        while retries < self.max_retries:
+            try:
+                if not self.is_connected():
+                    self.reconnect()
+                return getattr(self.client, command)(*args, **kwargs)
+            except (ConnectionError, TimeoutError) as e:
+                retries += 1
+                if retries < self.max_retries:
+                    time.sleep(self.retry_delay)
+                else:
+                    raise Exception("达到最大重试次数，无法执行命令")
+
+    def get(self, name):
+        # with self.get_connection() as client:
+        #     return client.get(name)
+        return self.execute_command("get", name)
+
+    def incrby(self, name, value):
+        # with self.get_connection() as client:
+        #     client.incrby(name, value)
+        self.execute_command("incrby", name, value)
+
+    def setnx(self, name, value=""):
+        # with self.get_connection() as client:
+        #     client.setnx(name, value)
+        self.execute_command("setnx", name, value)
+
+    def setex(self, name, t, value=""):
+        # with self.get_connection() as client:
+        #     client.setex(name, t, value)
+        self.execute_command("setex", name, t, value)
+
+    def expire(self, name, t, nx: bool = False, xx: bool = False, gt: bool = False, lt: bool = False):
+        # with self.get_connection() as client:
+        # client.expire(name, t, nx, xx, gt, lt)
+        self.execute_command("expire", name, t, nx, xx, gt, lt)
+
+    def ttl(self, name):
+        # with self.get_connection() as client:
+        #     return client.ttl(name)
+        return self.execute_command("ttl", name)
+
+    def delete(self, name):
+        # with self.get_connection() as client:
+        #     return client.delete(name)
+        return self.execute_command("delete", name)
+
+    def exists(self, *name) -> bool:
+        # with self.get_connection() as client:
+        #     return client.exists(*name)
+        return self.execute_command("exists", *name)
+
+    def sadd(self, name, value):
+        # with self.get_connection() as client:
+        #     return client.sadd(name, value)
+        return self.execute_command("sadd", name, value)
+
+    def zcard(self, name) -> bool:
+        # with self.get_connection() as client:
+        #     return client.zcard(name)
+        return self.execute_command("zcard", name)
+
+    def zadd(self, name, item: dict, **kwargs):
+        # with self.get_connection() as client:
+        #     return client.zadd(name, item, **kwargs)
+        return self.execute_command("zadd", name, item, **kwargs)
+
+    def zrem(self, name, *value):
+        # with self.get_connection() as client:
+        #     return client.zrem(name, *value)
+        return self.execute_command("zrem", name, *value)
+
+    def zcount(self, name, _min, _max):
+        # with self.get_connection() as client:
+        #     return client.zcount(name, _min, _max)
+        return self.execute_command("zcount", name, _min, _max)
+
+    # def zrangebyscore(self, name, _min, _max, start, num, withscores: bool = False, *args):
+    #     with self.get_connection() as client:
+    #        return client.zrangebyscore(name, _min, _max, start, num, withscores, *args)
+
+    def lua(self, script: str, keys: list = None, args: list = None):
+        keys = keys or []
+        args = args or []
+        keys_count = len(keys)
+        return self.execute_command("eval", script, keys_count, *keys, *args)
+
+    def lua_sha(self, sha1: str, keys: list = None, args: list = None):
+        keys = keys or []
+        args = args or []
+        keys_count = len(keys)
+        return self.execute_command("evalsha", sha1, keys_count, *keys, *args)
+
+    def execute_lua(self, lua_script: str, keys: list, *args):
+        execute = self.execute_command("register_script", lua_script)
+        return execute(keys=keys, args=args)
+
+    def lock(self, key, t=15) -> bool:
+        lua_script = """
+        local status = redis.call('setnx', KEYS[1], 1)
+        if ( status == 1 ) then
+            redis.call('expire', KEYS[1], ARGV[1])
+        end 
+        return status 
+        """
+        status = self.execute_lua(lua_script, [key], t)
+        return bool(status)
+
+    def members(self, key, score, start=0, count=1000, _min="-inf", _max="+inf") -> list:
+        lua_script = """
+        local min = ARGV[1]
+        local max = ARGV[2]
+        local start = ARGV[3]
+        local count = ARGV[4]
+        local score = ARGV[5]
+        local members = nil
+
+        if ( type(count) == string ) then
+            members = redis.call('zrangebyscore', KEYS[1], min, max, 'WITHSCORES')
+        else
+            members = redis.call('zrangebyscore', KEYS[1], min, max, 'WITHSCORES', 'limit', start, count)
+        end
+
+        local result = {}
+
+        for i = 1, #members, 2 do
+            local priority = nil
+            local member = members[i]
+            local originPriority = nil
+            if ( members[i+1] + 0 < 0 ) then
+                originPriority = math.ceil(members[i+1]) * 1000 - members[i+1] * 1000
+            else
+                originPriority = math.floor(members[i+1])
+            end
+
+            if ( score + 0 >= 1000 ) then
+                priority = -score - originPriority / 1000
+            elseif ( score + 0 == 0 ) then
+                priority = originPriority
+            else
+                originPriority = score 
+                priority = score
+            end
+            redis.call('zadd', KEYS[1], priority, member)
+            table.insert(result, member)
+            table.insert(result, originPriority)
+        end
+
+        return result
+        """
+        members = self.execute_lua(lua_script, [key], _min, _max, start, count, score)
+        return [(members[i].decode(), int(members[i + 1])) for i in range(0, len(members), 2)]
+
+    def done(self, keys: list, *args) -> list:
+        lua_script = """
+        for i, member in ipairs(ARGV) do
+            redis.call("zrem", KEYS[1], member)
+            redis.call("sadd", KEYS[2], member)
+        end
+        """
+        self.execute_lua(lua_script, keys, *args)

cobweb-launcher-3.1.0/cobweb/launchers/__init__.py

@@ -0,0 +1,9 @@
+# from .launcher_air import LauncherAir
+# from .launcher_pro import LauncherPro
+# from .launcher_api import LauncherApi
+
+
+from .launcher import Launcher
+from .uploader import Uploader
+from .distributor import Distributor
+

cobweb-launcher-3.1.0/cobweb/launchers/distributor.py

@@ -0,0 +1,171 @@
+import time
+import threading
+import traceback
+from inspect import isgenerator
+from typing import Callable
+from urllib.parse import urlparse
+from requests import Response as Res
+
+from cobweb import setting
+from cobweb.constant import DealModel, LogTemplate
+from cobweb.base import (
+    Seed,
+    Queue,
+    BaseItem,
+    Request,
+    Response,
+    logger
+)
+from cobweb.utils import LoghubDot, check_pause
+
+
+class Distributor(threading.Thread):
+
+    def __init__(
+            self,
+            task: str,
+            project: str,
+            new: Queue,
+            todo: Queue,
+            done: Queue,
+            upload: Queue,
+            register: Callable,
+            stop: threading.Event,
+            pause: threading.Event,
+            SpiderCrawler
+    ):
+        super().__init__()
+        self.task = task
+        self.project = project
+        self.stop = stop
+        self.pause = pause
+
+        self.new = new
+        self.todo = todo
+        self.done = done
+        self.upload = upload
+        self.register = register
+
+        self.time_sleep = setting.SPIDER_TIME_SLEEP
+        self.thread_num = setting.SPIDER_THREAD_NUM
+        self.max_retries = setting.SPIDER_MAX_RETRIES
+        self.record_failed = setting.RECORD_FAILED_SPIDER
+        self.loghub_dot = LoghubDot()  # todo: 解偶
+
+        self.Crawler = SpiderCrawler
+
+        logger.debug(f"Distribute instance attrs: {self.__dict__}")
+
+    def distribute(self, item, seed, _id: int):
+        if isinstance(item, Request):
+            seed.params.start_time = time.time()
+            self.process(item=seed, seed=seed, callback=self.Crawler.download, _id=1)
+        elif isinstance(item, Response):
+            if _id == 2:
+                raise TypeError("parse function can't yield a Response instance")
+            dot = isinstance(item.response, Res)
+            self.spider_logging(seed, item, dot=dot)
+            self.process(item=seed, seed=seed, callback=self.Crawler.parse, _id=2)
+        elif isinstance(item, BaseItem):
+            self.upload.push(item)
+        elif isinstance(item, Seed):
+            self.new.push((seed, item), direct_insertion=True)
+        elif isinstance(item, str) and item == DealModel.poll:
+            self.todo.push(seed)
+        elif isinstance(item, str) and item == DealModel.done:
+            self.done.push(seed)
+        elif isinstance(item, str) and item == DealModel.fail:
+            seed.params.retry += 1
+            if seed.params.retry < self.max_retries:
+                self.todo.push(seed)
+            else:
+                if record_failed := self.record_failed:
+                    try:
+                        response = Response(seed, "failed", max_retries=True)
+                        self.process(response, seed, self.Crawler.parse, _id=2)
+                    except:
+                        record_failed = False
+                if not record_failed:
+                    self.done.push(seed)
+        else:
+            raise TypeError("yield value type error!")
+
+    def process(self, item, seed, callback, _id: int):
+        result_iterators = callback(item)
+        if not isgenerator(result_iterators):
+            raise TypeError(f"{callback.__name__} function isn't a generator!")
+        for result_item in result_iterators:
+            self.distribute(result_item, seed, _id)
+
+    @check_pause
+    def spider(self):
+        if seed := self.todo.pop():
+            try:
+                self.process(item=seed, seed=seed, callback=self.Crawler.request, _id=0)
+            except Exception as e:
+                url, status = seed.url, e.__class__.__name__
+                msg = ''.join(traceback.format_exception(type(e), e, e.__traceback__))
+                if getattr(e, "response", None) and isinstance(e.response, Res):
+                    url = e.response.request.url
+                    status = e.response.status_code
+                self.spider_logging(seed, None, error=True, url=url, status=status, msg=msg)
+                self.distribute(DealModel.fail, seed, _id=-1)
+
+    def spider_logging(
+            self, seed,
+            item: Response = None,
+            error: bool = False,
+            dot: bool = True,
+            **kwargs
+    ):
+        detail_log_info = LogTemplate.log_info(seed.to_dict)
+        if error:
+            url = kwargs.get("url")
+            msg = kwargs.get("msg")
+            status = kwargs.get("status")
+            if dot:
+                self.loghub_dot.build(
+                    topic=urlparse(url).netloc,
+                    data_size=-1, cost_time=-1,
+                    status=status, url=url,
+                    seed=seed.to_string,
+                    proxy_type=seed.params.proxy_type,
+                    proxy=seed.params.proxy,
+                    project=self.project,
+                    task=self.task, msg=msg,
+                )
+            logger.info(LogTemplate.download_exception.format(
+                detail=detail_log_info,
+                retry=seed.params.retry,
+                priority=seed.params.priority,
+                seed_version=seed.params.seed_version,
+                identifier=seed.identifier or "",
+                exception=msg
+            ))
+        else:
+            logger.info(LogTemplate.download_info.format(
+                detail=detail_log_info,
+                retry=seed.params.retry,
+                priority=seed.params.priority,
+                seed_version=seed.params.seed_version,
+                identifier=seed.identifier or "",
+                status=item.response,
+                response=LogTemplate.log_info(item.to_dict)
+            ))
+            if dot:
+                end_time = time.time()
+                stime = seed.params.start_time
+                cost_time = end_time - stime if stime else -1
+                topic = urlparse(item.response.request.url).netloc
+                data_size = int(item.response.headers.get("content-length", 0))
+                self.loghub_dot.build(
+                    topic=topic, data_size=data_size, cost_time=cost_time,
+                    status=200, seed=seed.to_string, url=item.response.url,
+                    proxy=seed.params.proxy, proxy_type=seed.params.proxy_type,
+                    project=self.project, task=self.task,
+                )
+
+    def run(self):
+        self.register(self.loghub_dot.build_run, tag="LoghubDot")
+        for _ in range(self.thread_num):
+            self.register(self.spider, tag="Distributor")