cobweb-launcher 0.0.1__py3-none-any.whl

cobweb/base/queue_tmp.py

@@ -0,0 +1,60 @@
+from typing import Iterable
+
+# from pympler import asizeof
+from collections import deque
+
+
+class Queue:
+
+    def __init__(self):
+        self._seed_queue = deque()
+
+    @property
+    def queue_names(self):
+        return tuple(self.__dict__.keys())
+
+    @property
+    def used_memory(self):
+        return asizeof.asizeof(self)
+
+    def create_queue(self, queue_name: str):
+        self.__setattr__(queue_name, deque())
+
+    def push_seed(self, seed):
+        self.push("_seed_queue", seed)
+
+    def pop_seed(self):
+        return self.pop("_seed_queue")
+
+    def push(self, queue_name: str, data, left: bool = False):
+        try:
+            if not data:
+                return None
+            queue = self.__getattribute__(queue_name)
+            if isinstance(data, Iterable):
+                queue.extend(data) if left else queue.extendleft(data)
+            else:
+                queue.appendleft(data) if left else queue.append(data)
+        except AttributeError as e:
+            print(e)
+
+    def pop(self, queue_name: str, left: bool = True):
+        try:
+            queue = self.__getattribute__(queue_name)
+            return queue.pop() if left else queue.popleft()
+        except IndexError as e:
+            print(e)
+            return None
+        except AttributeError as e:
+            print(e)
+            return None
+
+
+# qqueue = Queue()
+# # qqueue.create_queue("test")
+# print(qqueue.queue_names)
+# qqueue.push("task_queue", "key")
+# print(qqueue.used_memory)
+# c = qqueue.pop("task_queue")
+# print(c)
+

cobweb/base/request.py

@@ -0,0 +1,62 @@
+import random
+from typing import Union
+
+import requests
+
+
+class Request:
+
+    def __init__(self):
+        pass
+
+
+def gen_user_agent(platform: str = 'android', redis_client=None):
+    user_agent = ''
+    if platform == 'android':
+        os_version = f'{random.randint(4, 10)}.{random.randint(0, 9)}.{random.randint(0, 9)}'
+        model = (redis_client and redis_client.srandmember('(md)set_android_model').decode()) or ''
+        webkit_version = f'{random.randint(450, 550)}.{random.randint(0, 100)}.{random.randint(0, 100)}'
+        version = f'{random.randint(3, 6)}.{random.randint(0, 9)}.{random.randint(0, 9)}'
+        chrome_version = f'{random.randint(50, 88)}.{random.randint(0, 9)}.{random.randint(1000, 5000)}.{random.randint(0, 1000)}'
+        user_agent = f'Mozilla/5.0 (Linux; U; Android {os_version}; zh-cn; {model} Build/{model}) AppleWebKit/{webkit_version} (KHTML, like Gecko) Version/{version} Chrome/{chrome_version} Mobile Safari/{webkit_version}'
+    elif platform == 'iphone':
+        os_version = f'{random.randint(5, 13)}_{random.randint(0, 9)}_{random.randint(0, 9)}'
+        webkit_version = f'{random.randint(550, 650)}.{random.randint(0, 100)}.{random.randint(0, 100)}'
+        version = f'{random.randint(4, 13)}.{random.randint(0, 9)}.{random.randint(0, 9)}'
+        user_agent = f'Mozilla/5.0 (iPhone; CPU iPhone OS {os_version} like Mac OS X) AppleWebKit/{webkit_version} (KHTML, like Gecko) Version/{version} Mobile Safari/{webkit_version}'
+
+    return user_agent
+
+
+def config(
+        url,
+        method: str = "GET",
+        headers: dict = None,
+        proxies: dict = None,
+        cookies: dict = None,
+        params: dict = None,
+        timeout: int = None,
+        stream: bool = False,
+        data: Union[dict, str, tuple] = None,
+) -> dict:
+    if not headers:
+        headers = {"accept": "*/*", "user-agent": gen_user_agent()}
+
+    elif "user-agent" not in [key.lower() for key in headers.keys()]:
+        headers["user-agent"] = gen_user_agent()
+
+    return {
+        "method": method,
+        "url": url,
+        "data": data,
+        "params": params,
+        "cookies": cookies,
+        "headers": headers,
+        "proxies": proxies,
+        "stream": stream,
+        "timeout": timeout or 3,
+    }
+
+
+def request(**kwargs):
+    return requests.request(**kwargs)

cobweb/base/task.py

@@ -0,0 +1,38 @@
+from config import info
+
+
+class Task:
+
+    def __init__(
+            self,
+            project=None,
+            task_name=None,
+            start_seed=None,
+            spider_num=None,
+            # queue_length=None,
+            max_retries=None,
+            scheduler_info=None,
+            storer_info=None,
+            redis_info=None
+    ):
+        """
+
+        :param project:
+        :param task_name:
+        :param start_seed:
+        :param spider_num:
+        # :param queue_length:
+        :param scheduler_info:
+        :param storer_info: Union(list, DataInfo/namedtuple), 单个元素构成必须有3个值(数据库类型，表名，字段名)
+        """
+        self.project = project or "test"
+        self.task_name = task_name or "spider"
+        self.start_seed = start_seed
+        self.spider_num = spider_num or 1
+        self.max_retries = max_retries or 5
+        # self.redis_info = RedisInfo(**(redis_info or dict()))
+        self.redis_info = info(redis_info, tag=0)
+        # self.scheduler_info = SchedulerDB.info(scheduler_info)
+        self.scheduler_info = info(scheduler_info, tag=1)
+        # self.storer_info = StorerDB.info(storer_info)
+        self.storer_info = info(storer_info, tag=2)

cobweb/base/utils.py

@@ -0,0 +1,15 @@
+import sys
+
+
+def struct_table_name(table_name):
+    return table_name.replace(".", "__p__").replace(":", "__c__")
+
+
+def restore_table_name(table_name):
+    return table_name.replace("__p__", ".").replace("__c__", ":")
+
+
+def struct_queue_name(db_name, table_name):
+    return sys.intern(f"__{db_name}_{table_name}_queue__")
+
+

cobweb/db/base/client_db.py

@@ -0,0 +1 @@
+

cobweb/db/base/oss_db.py

@@ -0,0 +1,116 @@
+import oss2
+from typing import Union
+from oss2.models import PartInfo
+from requests import Response
+from base.log import log
+
+
+class OssDB:
+
+    def __init__(
+            self,
+            bucket_name,
+            endpoint,
+            access_key,
+            secret_key,
+            chunk_size,
+            min_size
+    ):
+        self.auth = oss2.Auth(
+            access_key_id=access_key,
+            access_key_secret=secret_key
+        )
+        self.bucket = oss2.Bucket(
+            auth=self.auth,
+            endpoint=endpoint,
+            bucket_name=bucket_name
+        )
+        self.chunk_size = chunk_size or 1024 ** 2
+        self.min_size = min_size or 1024
+
+    @staticmethod
+    def format_upload_len(length):
+        if not length:
+            raise ValueError("Length cannot be None or 0")
+
+        units = ["KB", "MB", "GB", "TB"]
+        for i in range(3):
+            num = length / 1024 ** (i + 1)
+            if num >= 1:
+                return f"{round(num, 2)} {units[i]}"
+
+    def assemble(self, ready_data, part_data):
+        upload_data = None
+        ready_data = ready_data + part_data
+        if len(ready_data) >= self.chunk_size:
+            upload_data = ready_data[:self.chunk_size]
+            ready_data = ready_data[self.chunk_size:]
+
+        return ready_data, upload_data
+
+    def iter_data(self, data):
+        if isinstance(data, Response):
+            for part_data in data.iter_content(self.chunk_size):
+                yield part_data
+        if isinstance(data, bytes):
+            for i in range(0, len(data), self.chunk_size):
+                yield data[i:i + self.chunk_size]
+
+    def upload_split(
+            self, oss_path: str,
+            data: Union[bytes, Response],
+            timeout: int = 300,
+    ):
+        parts = []
+        upload_id = None
+        ready_data = b""
+        upload_data_len = 0
+        headers = {"Expires": str(timeout * 1000)}
+        try:
+            upload_id = self.bucket.init_multipart_upload(oss_path).upload_id
+            for part_data in self.iter_data(data):
+                upload_data_len += len(part_data)
+                ready_data, upload_data = self.assemble(ready_data, part_data)
+                if upload_data:
+                    part_index = len(parts) + 1
+                    upload_info = self.bucket.upload_part(
+                        oss_path, upload_id, part_index, upload_data
+                    )
+                    parts.append(PartInfo(part_index, upload_info.etag))
+
+            format_upload = self.format_upload_len(upload_data_len)
+
+            if parts and ready_data:
+                part_index = len(parts) + 1
+                upload_info = self.bucket.upload_part(
+                    oss_path, upload_id, part_index, ready_data
+                )
+                parts.append(PartInfo(part_index, upload_info.etag))
+                self.bucket.complete_multipart_upload(
+                    oss_path, upload_id, parts
+                )
+                log.info(
+                    f"split upload, file path: {oss_path}"
+                    f", file size: {format_upload}"
+                )
+
+            elif len(ready_data) > self.min_size:
+                self.bucket.put_object(oss_path, ready_data, headers)
+                log.info(
+                    f"upload file, file path: {oss_path}"
+                    f", file size: {format_upload}"
+                )
+
+            else:
+                log.info(
+                    f"file size smaller than min size! "
+                    f"file size: {format_upload}"
+                )
+            status = True
+        except Exception as e:
+            self.bucket.abort_multipart_upload(oss_path, upload_id, headers)
+            log.exception("upload file exception: " + str(e))
+            status = False
+
+        return status
+

cobweb/db/base/redis_db.py

@@ -0,0 +1,214 @@
+import time
+import redis
+from base.bbb import Seed
+
+
+class RedisDB:
+
+    def __init__(
+            self,
+            project: str,
+            task_name: str,
+            host=None,
+            port=None,
+            username=None,
+            password=None,
+            db=0
+    ):
+        pool = redis.ConnectionPool(
+            host=host,
+            port=port,
+            username=username,
+            password=password,
+            db=db
+        )
+        self.heartbeat_key = f"{project}:{task_name}:heartbeat"  # redis type string
+        self.spider_key = f"{project}:{task_name}:seed_info:spider"  # redis type zset, .format(priority)
+        self.storer_key = f"{project}:{task_name}:seed_info:storer:%s"  # redis type set,
+        self.failed_key = f"{project}:{task_name}:seed_info:failed"  # redis type set, .format(priority)
+        self.succeed_key = f"{project}:{task_name}:seed_info:succeed"  # redis type set, .format(priority)
+        self.update_lock = f"{project}:{task_name}:update_seed_lock"  # redis type string
+        self.check_lock = f"{project}:{task_name}:check_seed_lock"  # redis type string
+        self.scheduler_lock = f"{project}:{task_name}:scheduler_lock"  # redis type string
+        self.client = redis.Redis(connection_pool=pool)
+
+    # pass!
+    def _get_lock(self, key, t=15, timeout=3, sleep_time=0.1):
+        begin_time = int(time.time())
+        while True:
+            if self.client.setnx(key, ""):
+                self.client.expire(key, t)
+                return True
+            if int(time.time()) - begin_time > timeout:
+                break
+            time.sleep(sleep_time)
+
+        if self.client.ttl(key) == -1:
+            delete_status = True
+            for _ in range(3):
+                if self.client.ttl(key) != -1:
+                    delete_status = False
+                    break
+                time.sleep(0.5)
+            if delete_status:
+                self.client.expire(key, t)
+            return False
+        else:
+            ttl = self.client.ttl(key)
+            print("ttl: " + str(ttl))
+            return False
+
+    # pass!
+    def _deal_seed(self, seeds, is_add: bool):
+        if not seeds:
+            return None
+
+        if not isinstance(seeds, list):
+            seeds = [seeds]
+
+        item_info = dict()
+
+        for seed in seeds:
+            if not isinstance(seed, Seed):
+                seed = Seed(seed)
+            item_info[seed.format_seed] = seed._priority
+
+        if item_info:
+            self.client.zadd(self.spider_key, mapping=item_info, nx=is_add, xx=not is_add)
+
+    # pass!
+    def add_seed(self, seeds):
+        self._deal_seed(seeds, is_add=True)
+
+    def reset_seed(self, seeds):
+        self._deal_seed(seeds, is_add=False)
+
+    # pass!
+    def del_seed(self, seeds, spider_status: bool = True):
+        if not seeds:
+            return None
+
+        if not isinstance(seeds, list):
+            seeds = [seeds]
+
+        seeds = [seed if isinstance(seed, Seed) else Seed(seed) for seed in seeds]
+
+        if seeds:
+            redis_key = self.succeed_key if spider_status else self.failed_key
+            self.client.sadd(redis_key, *(str(seed) for seed in seeds))
+            self.client.zrem(self.spider_key, *(seed.format_seed for seed in seeds))
+
+    # pass!
+    def set_storer(self, key, seeds):
+        if not seeds:
+            return None
+
+        if not isinstance(seeds, list):
+            seeds = [seeds]
+
+        item_info = dict()
+        score = -int(time.time())
+        for seed in seeds:
+            if not isinstance(seed, Seed):
+                seed = Seed(seed)
+            item_info[seed.format_seed] = score
+
+        if item_info:
+            self.client.zadd(self.storer_key % key, mapping=item_info)
+            print("zadd storer key", len(item_info.keys()))
+
+    # pass!
+    def get_seed(self, length: int = 200):
+        cs = time.time()
+
+        if self._get_lock(key=self.update_lock):
+
+            update_item, result = {}, []
+
+            version = int(time.time())
+
+            items = self.client.zrangebyscore(self.spider_key, min=0, max="+inf", start=0, num=length, withscores=True)
+
+            for value, priority in items:
+                score = -(version + int(priority) / 1000)
+                seed = Seed(value, priority=priority, version=version)
+                update_item[value] = score
+                result.append(seed)
+
+            print("\nset seeds into queue time: " + str(time.time() - cs))
+            if result:
+                self.client.zadd(self.spider_key, mapping=update_item, xx=True)
+
+            self.client.delete(self.update_lock)
+            print("push seeds into queue time: " + str(time.time() - cs))
+            return result
+
+    # pass!
+    def check_spider_queue(self, stop, storer_num):
+        while not stop.is_set():
+            # 每15s获取check锁,等待600s后仍获取不到锁则重试;获取到锁后，设置锁的存活时间为15s
+            if self._get_lock(key=self.check_lock, t=5, timeout=600, sleep_time=3):
+                heartbeat = True if self.client.exists(self.heartbeat_key) else False
+                # 重启重制score值，否则获取10分钟前的分数值
+                score = -int(time.time()) + 600 if heartbeat else "-inf"
+
+                keys = self.client.keys(self.storer_key % "*")
+                if len(keys) == storer_num:
+                    intersection_key = self.storer_key % "intersection"
+                    self.client.delete(intersection_key)
+                    self.client.zinterstore(intersection_key, keys)
+                    while True:
+                        members = self.client.zrange(intersection_key, 0, 1999)
+                        if not members:
+                            break
+                        for key in keys:
+                            self.client.zrem(key, *members)
+                        self.client.sadd(self.succeed_key, *members)
+                        self.client.zrem(self.spider_key, *members)
+                        self.client.zrem(intersection_key, *members)
+                        print("succeed spider data ...")
+
+                for key in keys:
+                    self.client.zremrangebyscore(key, min=score, max="(0")
+
+                while True:
+                    items = self.client.zrangebyscore(self.spider_key, min=score, max="(0", start=0, num=5000, withscores=True)
+                    if not items:
+                        break
+                    reset_items = {}
+                    for value, priority in items:
+                        reset_score = "{:.3f}".format(priority).split(".")[1]
+                        reset_items[value] = int(reset_score)
+                    if reset_items:
+                        self.client.zadd(self.spider_key, mapping=reset_items, xx=True)
+
+                if not heartbeat:
+                    self.client.setex(self.heartbeat_key, 15, "")
+
+                self.client.delete(self.check_lock)
+                time.sleep(3)
+
+    # pass!
+    def set_heartbeat(self, stop):
+        time.sleep(5)
+        while not stop.is_set():
+            self.client.setex(self.heartbeat_key, 5, "")
+            time.sleep(3)
+
+    # # pass!
+    # def heartbeat(self):
+    #     """
+    #     返回心跳key剩余存活时间
+    #     """
+    #     return self.client.ttl(self.heartbeat_key)
+
+    # pass!
+    def spider_queue_length(self):
+        return self.client.zcard(self.spider_key)
+
+    # pass!
+    def ready_seed_length(self):
+        return self.client.zcount(self.spider_key, min=0, max="+inf")
+
+    def get_scheduler_lock(self):
+        return self._get_lock(self.scheduler_lock)