cobweb-launcher 1.3.14__py3-none-any.whl → 3.1.0__py3-none-any.whl

cobweb/__init__.py

@@ -1,2 +1,2 @@
-from .launchers import LauncherPro, LauncherApi
+from .launchers import Launcher
 from .constant import CrawlerModel

cobweb/base/__init__.py

@@ -1,154 +1,9 @@
-import os
-import time
-import traceback
-import threading
-
-from functools import wraps
-from inspect import isgenerator
-from typing import Callable, Union
-
 from .common_queue import Queue
 from .response import Response
-from .basic import Seed, Request, Response
+from .request import Request
 from .item import BaseItem, ConsoleItem
-# from .seed import Seed
-from .log import logger
-from .dotting import LoghubDot
-
-
-class TaskQueue:
-    TODO = Queue()          # 任务种子队列
-    DOWNLOAD = Queue()      # 下载任务队列
-
-    SEED = Queue()          # 添加任务种子队列
-    REQUEST = Queue()       # 请求队列
-    RESPONSE = Queue()      # 响应队列
-    DONE = Queue()          # 下载完成队列
-    UPLOAD = Queue()        # 任务上传队列
-    DELETE = Queue()        # 任务删除队列
-    DOT = LoghubDot()
-
-    @staticmethod
-    def is_empty():
-        total_length = TaskQueue.SEED.length
-        total_length += TaskQueue.TODO.length
-        total_length += TaskQueue.REQUEST.length
-        total_length += TaskQueue.DOWNLOAD.length
-        total_length += TaskQueue.RESPONSE.length
-        total_length += TaskQueue.UPLOAD.length
-        total_length += TaskQueue.DONE.length
-        total_length += TaskQueue.DELETE.length
-        return not bool(total_length)
-
-    @staticmethod
-    def process_task(it: Union[Seed, Request, Response, BaseItem], crawler_func: Callable):
-        try:
-            start_time = time.time()
-            iterators = crawler_func(it)
-            if not isgenerator(iterators):
-                raise TypeError(f"{crawler_func.__name__} function isn't a generator")
-            for tk in iterators:
-                if isinstance(tk, Request):
-                    TaskQueue.REQUEST.push(tk)
-                elif isinstance(tk, Response):
-                    TaskQueue.RESPONSE.push(tk)
-                elif isinstance(tk, BaseItem):
-                    TaskQueue.UPLOAD.push(tk)
-                elif isinstance(tk, Seed):
-                    TaskQueue.SEED.push(tk)
-                else:
-                    raise TypeError(f"{crawler_func.__name__} function return type isn't supported")
-                TaskQueue.DOT.build(
-                    topic=f"{os.getenv('PROJECT')}:{os.getenv('TASK')}",
-                    cost_time=round(time.time() - start_time, 2),
-                    process_task_type=tk.__class__.__name__,
-                    **tk.to_dict
-                )
-        except Exception as e:
-            it.params.retry += 1
-            if isinstance(it, Request):
-                TaskQueue.DOWNLOAD.push(it)
-            elif isinstance(it, Response):
-                TaskQueue.RESPONSE.push(it)
-            elif isinstance(it, Seed):
-                TaskQueue.TODO.push(it)
-            elif isinstance(it, BaseItem):
-                TaskQueue.UPLOAD.push(it)
-            logger.info(
-                f"{crawler_func.__name__} failed: "
-                f"{''.join(traceback.format_exception(type(e), e, e.__traceback__))}"
-            )
-            time.sleep(1)
-
-
-class Decorators:
+from .seed import Seed
 
-    @staticmethod
-    def add_thread(num=1):
-        def decorator(func):
-            @wraps(func)
-            def wrapper(self, *args):
-                for i in range(num):
-                    name = func.__name__ + "_" + str(i) if num > 1 else func.__name__
-                    self._threads.append(threading.Thread(name=name, target=func, args=(self,) + args))
-
-            return wrapper
-
-        return decorator
-
-    @staticmethod
-    def pause(func):
-        @wraps(func)
-        def wrapper(self, *args, **kwargs):
-            while not self.stop.is_set():
-                while not self.pause.is_set():
-                    try:
-                        func(self)
-                    except Exception as e:
-                        logger.info(f"{func.__name__}: " + str(e))
-                    finally:
-                        time.sleep(0.1)
-                # logger.info(f"{func.__name__}: close!")
-
-        return wrapper
-
-    @staticmethod
-    def stop(func):
-        @wraps(func)
-        def wrapper(self, *args, **kwargs):
-            while not self.stop.is_set():
-                try:
-                    func(self, *args, **kwargs)
-                except Exception as e:
-                    logger.info(
-                        f"{func.__name__} exception: \n" +
-                        ''.join(traceback.format_exception(type(e), e, e.__traceback__))
-                    )
-                finally:
-                    time.sleep(0.1)
-
-        return wrapper
-
-    @staticmethod
-    def decorator_oss_db(exception, retries=3):
-        def decorator(func):
-            @wraps(func)
-            def wrapper(callback_func, *args, **kwargs):
-                result = None
-                for i in range(retries):
-                    msg = None
-                    try:
-                        return func(callback_func, *args, **kwargs)
-                    except Exception as e:
-                        result = None
-                        msg = e
-                    finally:
-                        if result:
-                            return result
-
-                        if i >= 2 and msg:
-                            raise exception(msg)
-
-            return wrapper
+from .log import logger
+# from .decorators import decorator_oss_db
 
-        return decorator

cobweb/base/common_queue.py

@@ -1,4 +1,3 @@
-import time
 from collections import deque
 
 
@@ -29,15 +28,3 @@ class Queue:
             return None
         except AttributeError:
             return None
-
-    def clear(self):
-        self._queue.clear()
-
-    def get(self):
-        try:
-            yield self._queue.popleft()
-        except IndexError:
-            time.sleep(1)
-            yield None
-        except AttributeError:
-            yield None

cobweb/base/request.py

@@ -1,4 +1,3 @@
-import json
 import random
 import requests
 
@@ -31,6 +30,7 @@ class Request:
             **kwargs
     ):
         self.url = url
+        self.seed = seed
         self.check_status_code = check_status_code
         self.request_setting = {}
 
@@ -46,12 +46,6 @@ class Request:
         if random_ua:
             self._build_header()
 
-        if isinstance(seed, Seed):
-            self.seed = seed.to_string
-        else:
-            kwargs.update(**seed.to_dict)
-            self.seed = self.to_string
-
     @property
     def _random_ua(self) -> str:
         v1 = random.randint(4, 15)
@@ -79,16 +73,10 @@ class Request:
     @property
     def to_dict(self):
         _dict = self.__dict__.copy()
+        _dict.pop('url')
         _dict.pop('seed')
         _dict.pop('check_status_code')
         _dict.pop('request_setting')
         return _dict
 
-    @property
-    def to_string(self) -> str:
-        return json.dumps(
-            self.to_dict,
-            ensure_ascii=False,
-            separators=(",", ":")
-        )
 

cobweb/base/seed.py

@@ -5,11 +5,13 @@ import hashlib
 
 class SeedParams:
 
-    def __init__(self, retry, priority, seed_version, seed_status=None):
+    def __init__(self, retry, priority, seed_version, seed_status=None, proxy_type=None, proxy=None):
         self.retry = retry or 0
         self.priority = priority or 300
         self.seed_version = seed_version or int(time.time())
         self.seed_status = seed_status
+        self.proxy_type = proxy_type
+        self.proxy = proxy
 
 
 class Seed:
@@ -18,7 +20,9 @@ class Seed:
         "retry",
         "priority",
         "seed_version",
-        "seed_status"
+        "seed_status",
+        "proxy_type",
+        "proxy"
     ]
 
     def __init__(
@@ -29,6 +33,8 @@ class Seed:
             priority=None,
             seed_version=None,
             seed_status=None,
+            proxy_type=None,
+            proxy=None,
             **kwargs
     ):
         if any(isinstance(seed, t) for t in (str, bytes)):
@@ -51,6 +57,8 @@ class Seed:
             "priority": priority,
             "seed_version": seed_version,
             "seed_status": seed_status,
+            "proxy_type": proxy_type,
+            "proxy": proxy
         }
 
         if kwargs:
@@ -104,15 +112,11 @@ class Seed:
             separators=(",", ":")
         )
 
-    # @property
-    # def get_all(self):
-    #     return json.dumps(
-    #         self.__dict__,
-    #         ensure_ascii=False,
-    #         separators=(",", ":")
-    #     )
-
     @property
-    def seed(self):
-        return self.to_string
+    def get_all(self):
+        return json.dumps(
+            self.__dict__,
+            ensure_ascii=False,
+            separators=(",", ":")
+        )
 

cobweb/constant.py

@@ -37,22 +37,6 @@ class LogTemplate:
     ----------------------- end  - console pipeline ------------------
     """
 
-    launcher_polling = """
-    ----------------------- start - 轮训日志: {task} -----------------
-            正在运行任务
-                构造请求任务数: {memory_todo_count}
-                正在下载任务数: {memory_download_count}
-            任务内存队列 
-                待构造请求队列:  {todo_queue_len}
-                待删除请求队列:  {delete_queue_len}
-                待进行下载队列:  {request_queue_len}
-                待解析响应队列:  {response_queue_len}
-                待删除下载队列:  {done_queue_len}
-            存储队列 
-               待上传数据队列:  {upload_queue_len}
-    ----------------------- end  - 轮训日志: {task} ------------------
-    """
-
     launcher_air_polling = """
     ----------------------- start - 轮训日志: {task} -----------------
             内存队列 

cobweb/crawlers/crawler.py

@@ -1,40 +1,19 @@
 import json
-import os
-import time
-import threading
-from typing import Union, Callable, Mapping
-
-from cobweb import setting
+from typing import Union
 from cobweb.base import (
     Seed,
     BaseItem,
     Request,
     Response,
     ConsoleItem,
-    Decorators,
-    TaskQueue,
 )
-from cobweb.constant import DealModel
-
 
-class Crawler(threading.Thread):
 
-    def __init__(self, stop, pause, custom_func: Union[Mapping[str, Callable]]):
-        super().__init__()
-        self.stop = stop
-        self.pause = pause
-        for func_name, _callable in custom_func.items():
-            if isinstance(_callable, Callable):
-                self.__setattr__(func_name, _callable)
-
-        self.spider_max_retries = setting.SPIDER_MAX_RETRIES
-        self.request_queue_size = setting.REQUEST_QUEUE_SIZE
-        self.download_queue_size = setting.DOWNLOAD_QUEUE_SIZE
-        self.upload_queue_size = setting.UPLOAD_QUEUE_SIZE
+class Crawler:
 
     @staticmethod
     def request(seed: Seed) -> Union[Request, BaseItem]:
-        yield Request(seed, timeout=5)
+        yield Request(seed.url, seed, timeout=5)
 
     @staticmethod
     def download(item: Request) -> Union[Seed, BaseItem, Response, str]:
@@ -47,64 +26,3 @@ class Crawler(threading.Thread):
         upload_item["text"] = item.response.text
         yield ConsoleItem(item.seed, data=json.dumps(upload_item, ensure_ascii=False))
 
-    # @decorators.add_thread()
-    @Decorators.pause
-    def build_request_item(self):
-        thread_sleep = 0.1
-        if TaskQueue.REQUEST.length >= self.request_queue_size:
-            thread_sleep = 5
-        elif seed := TaskQueue.TODO.pop():
-            # member, priority = seed_info
-            # seed = Seed(member, priority=priority)
-            if seed.params.retry > self.spider_max_retries:
-                TaskQueue.DOT.build(
-                    topic=f"{os.getenv('PROJECT')}:{os.getenv('TASK')}",
-                    process_task_type=seed.__class__.__name__,
-                    seed_status=DealModel.fail,
-                    retries=seed.params.retry,
-                    **seed.to_dict
-                )
-            else:
-                TaskQueue.process_task(seed, self.request)
-            TaskQueue.DELETE.push(seed.seed)
-        time.sleep(thread_sleep)
-
-    @Decorators.pause
-    def build_download_item(self):
-        thread_sleep = 0.1
-        if TaskQueue.RESPONSE.length >= self.download_queue_size:
-            thread_sleep = 5
-        elif request_item := TaskQueue.DOWNLOAD.pop():
-            if request_item.params.retry > self.spider_max_retries:
-                TaskQueue.DOT.build(
-                    topic=f"{os.getenv('PROJECT')}:{os.getenv('TASK')}",
-                    process_task_type=request_item.__class__.__name__,
-                    retries=request_item.params.retry,
-                    seed_status=DealModel.fail,
-                    **request_item.to_dict
-                )
-                TaskQueue.DONE.push(request_item.seed)
-            else:
-                TaskQueue.process_task(request_item, self.download)
-        time.sleep(thread_sleep)
-
-    @Decorators.pause
-    def build_parse_item(self):
-        thread_sleep = 0.1
-        if TaskQueue.UPLOAD.length >= self.upload_queue_size:
-            thread_sleep = 5
-        elif response_item := TaskQueue.RESPONSE.pop():
-            if response_item.params.retry > self.spider_max_retries:
-                TaskQueue.DOT.build(
-                    topic=f"{os.getenv('PROJECT')}:{os.getenv('TASK')}",
-                    process_task_type=response_item.__class__.__name__,
-                    seed_status=DealModel.fail,
-                    retries=response_item.params.retry,
-                    **response_item.to_dict
-                )
-                TaskQueue.DONE.push(response_item.seed)
-            else:
-                TaskQueue.process_task(response_item, self.parse)
-        time.sleep(thread_sleep)
-
-

cobweb/db/redis_db.py

@@ -1,67 +1,152 @@
 import redis
+import time
 from cobweb import setting
+from redis.exceptions import ConnectionError, TimeoutError
 
 
 class RedisDB:
-
     def __init__(self, **kwargs):
         redis_config = kwargs or setting.REDIS_CONFIG
-        pool = redis.ConnectionPool(**redis_config)
-        self._client = redis.Redis(connection_pool=pool)
+        self.host = redis_config['host']
+        self.password = redis_config['password']
+        self.port = redis_config['port']
+        self.db = redis_config['db']
+
+        self.max_retries = 5
+        self.retry_delay = 5
+        self.client = None
+        self.connect()
+
+    def connect(self):
+        """尝试连接 Redis"""
+        retries = 0
+        while retries < self.max_retries:
+            try:
+                self.client = redis.Redis(
+                    host=self.host,
+                    port=self.port,
+                    password=self.password,
+                    db=self.db,
+                    socket_timeout=5,  # 设置连接超时时间
+                    socket_connect_timeout=5  # 设置连接超时时间
+                )
+                # 测试连接是否成功
+                self.client.ping()
+                return
+            except (ConnectionError, TimeoutError) as e:
+                retries += 1
+                if retries < self.max_retries:
+                    time.sleep(self.retry_delay)
+                else:
+                    raise Exception("达到最大重试次数，无法连接 Redis")
+
+    def is_connected(self):
+        try:
+            self.client.ping()
+            return True
+        except (ConnectionError, TimeoutError):
+            return False
+
+    def reconnect(self):
+        self.connect()
+
+    def execute_command(self, command, *args, **kwargs):
+        retries = 0
+        while retries < self.max_retries:
+            try:
+                if not self.is_connected():
+                    self.reconnect()
+                return getattr(self.client, command)(*args, **kwargs)
+            except (ConnectionError, TimeoutError) as e:
+                retries += 1
+                if retries < self.max_retries:
+                    time.sleep(self.retry_delay)
+                else:
+                    raise Exception("达到最大重试次数，无法执行命令")
+
+    def get(self, name):
+        # with self.get_connection() as client:
+        #     return client.get(name)
+        return self.execute_command("get", name)
+
+    def incrby(self, name, value):
+        # with self.get_connection() as client:
+        #     client.incrby(name, value)
+        self.execute_command("incrby", name, value)
 
     def setnx(self, name, value=""):
-        return self._client.setnx(name, value)
+        # with self.get_connection() as client:
+        #     client.setnx(name, value)
+        self.execute_command("setnx", name, value)
 
     def setex(self, name, t, value=""):
-        return self._client.setex(name, t, value)
+        # with self.get_connection() as client:
+        #     client.setex(name, t, value)
+        self.execute_command("setex", name, t, value)
 
     def expire(self, name, t, nx: bool = False, xx: bool = False, gt: bool = False, lt: bool = False):
-        return self._client.expire(name, t, nx, xx, gt, lt)
+        # with self.get_connection() as client:
+        # client.expire(name, t, nx, xx, gt, lt)
+        self.execute_command("expire", name, t, nx, xx, gt, lt)
 
     def ttl(self, name):
-        return self._client.ttl(name)
+        # with self.get_connection() as client:
+        #     return client.ttl(name)
+        return self.execute_command("ttl", name)
 
     def delete(self, name):
-        return self._client.delete(name)
+        # with self.get_connection() as client:
+        #     return client.delete(name)
+        return self.execute_command("delete", name)
 
     def exists(self, *name) -> bool:
-        return self._client.exists(*name)
-
-    def incrby(self, name, value):
-        return self._client.incrby(name, value)
+        # with self.get_connection() as client:
+        #     return client.exists(*name)
+        return self.execute_command("exists", *name)
 
     def sadd(self, name, value):
-        return self._client.sadd(name, value)
+        # with self.get_connection() as client:
+        #     return client.sadd(name, value)
+        return self.execute_command("sadd", name, value)
 
     def zcard(self, name) -> bool:
-        return self._client.zcard(name)
+        # with self.get_connection() as client:
+        #     return client.zcard(name)
+        return self.execute_command("zcard", name)
 
     def zadd(self, name, item: dict, **kwargs):
-        return self._client.zadd(name, item, **kwargs)
+        # with self.get_connection() as client:
+        #     return client.zadd(name, item, **kwargs)
+        return self.execute_command("zadd", name, item, **kwargs)
 
     def zrem(self, name, *value):
-        return self._client.zrem(name, *value)
+        # with self.get_connection() as client:
+        #     return client.zrem(name, *value)
+        return self.execute_command("zrem", name, *value)
 
     def zcount(self, name, _min, _max):
-        return self._client.zcount(name, _min, _max)
+        # with self.get_connection() as client:
+        #     return client.zcount(name, _min, _max)
+        return self.execute_command("zcount", name, _min, _max)
 
     # def zrangebyscore(self, name, _min, _max, start, num, withscores: bool = False, *args):
-    #     return self._client.zrangebyscore(name, _min, _max, start, num, withscores, *args)
+    #     with self.get_connection() as client:
+    #        return client.zrangebyscore(name, _min, _max, start, num, withscores, *args)
 
     def lua(self, script: str, keys: list = None, args: list = None):
         keys = keys or []
         args = args or []
         keys_count = len(keys)
-        return self._client.eval(script, keys_count, *keys, *args)
+        return self.execute_command("eval", script, keys_count, *keys, *args)
 
     def lua_sha(self, sha1: str, keys: list = None, args: list = None):
         keys = keys or []
         args = args or []
         keys_count = len(keys)
-        return self._client.evalsha(sha1, keys_count, *keys, *args)
+        return self.execute_command("evalsha", sha1, keys_count, *keys, *args)
 
     def execute_lua(self, lua_script: str, keys: list, *args):
-        execute = self._client.register_script(lua_script)
+        execute = self.execute_command("register_script", lua_script)
         return execute(keys=keys, args=args)
 
     def lock(self, key, t=15) -> bool:
@@ -75,32 +160,7 @@ class RedisDB:
         status = self.execute_lua(lua_script, [key], t)
         return bool(status)
 
-    def auto_incr(self, name, t=15, limit=1000):
-        lua_script = """
-            local count = 0
-            local status = false
-            local limit = ARGV[2]
-            local expire = redis.call('ttl', KEYS[1])
-
-            if ( expire == -2 ) then
-                redis.call('setnx', KEYS[1], 1)
-            elseif ( expire == -1) then
-                redis.call('expire', KEYS[1], ARGV[1])
-            else
-                count = redis.call('get', KEYS[1])
-            end
-
-            if ( count + 0 < limit + 0 ) then
-                status = true
-                redis.call('incr', KEYS[1])
-            end
-
-            return status
-        """
-        status = self.execute_lua(lua_script, [name], t, limit)
-        return bool(status)
-
-    def members(self, key, score, start=0, count=5000, _min="-inf", _max="+inf") -> list:
+    def members(self, key, score, start=0, count=1000, _min="-inf", _max="+inf") -> list:
         lua_script = """
         local min = ARGV[1]
         local max = ARGV[2]
@@ -114,7 +174,7 @@ class RedisDB:
         else
             members = redis.call('zrangebyscore', KEYS[1], min, max, 'WITHSCORES', 'limit', start, count)
         end
-        
+
         local result = {}
 
         for i = 1, #members, 2 do
@@ -126,7 +186,7 @@ class RedisDB:
             else
                 originPriority = math.floor(members[i+1])
             end
-            
+
             if ( score + 0 >= 1000 ) then
                 priority = -score - originPriority / 1000
             elseif ( score + 0 == 0 ) then
@@ -153,6 +213,3 @@ class RedisDB:
         end
         """
         self.execute_lua(lua_script, keys, *args)
-
-
-

cobweb/launchers/__init__.py

@@ -1,3 +1,9 @@
 # from .launcher_air import LauncherAir
-from .launcher_pro import LauncherPro
-from .launcher_api import LauncherApi
+# from .launcher_pro import LauncherPro
+# from .launcher_api import LauncherApi
+
+
+from .launcher import Launcher
+from .uploader import Uploader
+from .distributor import Distributor
+