cobweb-launcher 0.1.23__py3-none-any.whl → 1.0.0__py3-none-any.whl

cobweb/crawlers/file_crawler.py

@@ -0,0 +1,182 @@
+import setting
+
+from typing import Union
+
+from cobweb.utils import OssUtil
+from cobweb.crawlers import Crawler
+from cobweb.base import Seed, BaseItem, Request, Response
+from cobweb.exceptions import OssDBPutPartError, OssDBMergeError
+
+
+class CrawlerAir(Crawler):
+    
+    oss_util = OssUtil()
+
+    @staticmethod
+    def download(item: Request) -> Union[Seed, BaseItem, Response, str]:
+        seed_dict = item.seed.to_dict
+        bucket_name = CrawlerAir.oss_util.bucket
+        try:
+            key = item.seed.oss_path
+            if CrawlerAir.oss_util.exists(key):
+                content_length = CrawlerAir.oss_util.head(key).content_length
+                yield Response(item.seed, "exists", bucket_name=bucket_name, data_size=content_length, **seed_dict)
+                # data, cols = download_meta(item.seed, bucket_name=bucket_name, data_size=content_length, **seed_dict)
+                # yield DownloadItem(item.seed, sid=item.seed.sid, cols=cols, data=data)
+
+            end = seed_dict.get("end", "")
+            start = seed_dict.get("start", "0")
+
+            if end or int(start):
+                item.request_setting["headers"]['Range'] = f'bytes={start}-{end}'
+
+            if not item.seed.params.identifier:
+                content = b""
+                chunk_size = CrawlerAir.oss_util.chunk_size
+                min_upload_size = CrawlerAir.oss_util.min_upload_size
+                position = seed_dict.get("position", 1)
+
+                response = item.download()
+
+                content_length = response.headers.get("content-length") or 0
+                content_type = response.headers.get("content-type", "").split(";")[0]
+                if content_type and content_type in setting.FILE_FILTER_CONTENT_TYPE:
+                    yield Response(
+                        item.seed, response, filter=True, msg=f"response content type is {content_type}",
+                        bucket_name=bucket_name, data_size=content_length, **seed_dict
+                    )
+                elif position == 1 and min_upload_size >= int(content_length) > 0:
+                    """过小文件标识返回"""
+                    yield Response(
+                        item.seed, response, filter=True, msg="file size is too small",
+                        bucket_name=bucket_name, data_size=content_length, **seed_dict
+                    )
+                elif position == 1 and chunk_size > int(content_length) > min_upload_size:
+                    """小文件直接下载"""
+                    for part_data in response.iter_content(chunk_size):
+                        content += part_data
+                    CrawlerAir.oss_util.put(key, content)
+                    yield Response(item.seed, response, bucket_name=bucket_name, data_size=content_length, **seed_dict)
+                    response.close()
+                else:
+                    """中大文件同步分片下载"""
+                    upload_content_length = 0
+                    if not seed_dict.get("upload_id"):
+                        seed_dict["upload_id"] = CrawlerAir.oss_util.init_part(key).upload_id
+                    upload_id = seed_dict["upload_id"]
+                    for part_data in response.iter_content(chunk_size):
+                        content += part_data
+                        if len(content) >= chunk_size:
+                            upload_data = content[:chunk_size]
+                            content = content[chunk_size:]
+                            CrawlerAir.oss_util.put_part(key, upload_id, position, upload_data)
+                            upload_content_length += len(upload_data)
+                            position += 1
+                            seed_dict['position'] = position
+                            seed_dict['start'] = upload_content_length
+
+                    response.close()
+                    if content:
+                        CrawlerAir.oss_util.put_part(key, upload_id, position, content)
+                        content_length += len(content)
+                    CrawlerAir.oss_util.merge(key, upload_id)
+                    yield Response(item.seed, response, bucket_name=bucket_name, data_size=content_length, **seed_dict)
+                    # data, cols = download_meta(item.seed, bucket_name, data_size=content_length, **seed_dict)
+                    # yield DownloadItem(item.seed, sid=item.seed.sid, cols=cols, data=data)
+
+            elif item.seed.params.identifier == "merge":
+                CrawlerAir.oss_util.merge(key, seed_dict["upload_id"])
+                content_length = CrawlerAir.oss_util.head(key).content_length
+                yield Response(item.seed, "merge", bucket_name=bucket_name, data_size=content_length, **seed_dict)
+                # data, cols = download_meta(item.seed, bucket_name, data_size=content_length, **seed_dict)
+                # yield DownloadItem(item.seed, sid=item.seed.sid, cols=cols, data=data)
+        except OssDBPutPartError:
+            yield Seed(seed_dict)
+        except OssDBMergeError:
+            yield Seed(seed_dict, identifier="merge")
+
+
+class CrawlerPro(Crawler):
+
+    oss_util = OssUtil()
+
+    @staticmethod
+    def download(item: Request) -> Union[Seed, BaseItem, Response, str]:
+        seed_dict = item.seed.to_dict
+        bucket_name = CrawlerAir.oss_util.bucket
+        try:
+            key = item.seed.oss_path
+            if CrawlerAir.oss_util.exists(key):
+                content_length = CrawlerAir.oss_util.head(key).content_length
+                yield Response(item.seed, "exists", bucket_name=bucket_name, data_size=content_length, **seed_dict)
+
+            end = seed_dict.get("end", "")
+            start = seed_dict.get("start", "0")
+
+            if end or int(start):
+                item.request_setting["headers"]['Range'] = f'bytes={start}-{end}'
+
+            if not item.seed.params.identifier:
+                content = b""
+                chunk_size = CrawlerAir.oss_util.chunk_size
+                min_upload_size = CrawlerAir.oss_util.min_upload_size
+                position = seed_dict.get("position", 1)
+
+                response = item.download()
+
+                content_length = response.headers.get("content-length") or 0
+                content_type = response.headers.get("content-type", "").split(";")[0]
+                if content_type and content_type in setting.FILE_FILTER_CONTENT_TYPE:
+                    yield Response(
+                        item.seed, response, filter=True, msg=f"response content type is {content_type}",
+                        bucket_name=bucket_name, data_size=content_length, **seed_dict
+                    )
+                elif position == 1 and min_upload_size >= int(content_length) > 0:
+                    """过小文件标识返回"""
+                    yield Response(
+                        item.seed, response, filter=True, msg="file size is too small",
+                        bucket_name=bucket_name, data_size=content_length, **seed_dict
+                    )
+                elif position == 1 and chunk_size > int(content_length) > min_upload_size:
+                    """小文件直接下载"""
+                    for part_data in response.iter_content(chunk_size):
+                        content += part_data
+                    CrawlerAir.oss_util.put(key, content)
+                    yield Response(item.seed, response, bucket_name=bucket_name, data_size=content_length, **seed_dict)
+                    response.close()
+                else:
+                    """中大文件同步分片下载"""
+                    upload_content_length = 0
+                    if not seed_dict.get("upload_id"):
+                        seed_dict["upload_id"] = CrawlerAir.oss_util.init_part(key).upload_id
+                    upload_id = seed_dict["upload_id"]
+                    for part_data in response.iter_content(chunk_size):
+                        content += part_data
+                        if len(content) >= chunk_size:
+                            upload_data = content[:chunk_size]
+                            content = content[chunk_size:]
+                            CrawlerAir.oss_util.put_part(key, upload_id, position, upload_data)
+                            upload_content_length += len(upload_data)
+                            position += 1
+                            seed_dict['position'] = position
+                            seed_dict['start'] = upload_content_length
+
+                    response.close()
+                    if content:
+                        CrawlerAir.oss_util.put_part(key, upload_id, position, content)
+                        content_length += len(content)
+                    CrawlerAir.oss_util.merge(key, upload_id)
+                    yield Response(item.seed, response, bucket_name=bucket_name, data_size=content_length, **seed_dict)
+                    # data, cols = download_meta(item.seed, bucket_name, data_size=content_length, **seed_dict)
+                    # yield DownloadItem(item.seed, sid=item.seed.sid, cols=cols, data=data)
+
+            elif item.seed.params.identifier == "merge":
+                CrawlerAir.oss_util.merge(key, seed_dict["upload_id"])
+                content_length = CrawlerAir.oss_util.head(key).content_length
+                yield Response(item.seed, "merge", bucket_name=bucket_name, data_size=content_length, **seed_dict)
+                # data, cols = download_meta(item.seed, bucket_name, data_size=content_length, **seed_dict)
+                # yield DownloadItem(item.seed, sid=item.seed.sid, cols=cols, data=data)
+        except OssDBPutPartError:
+            yield Seed(seed_dict)
+        except OssDBMergeError:
+            yield Seed(seed_dict, identifier="merge")

cobweb/db/__init__.py

@@ -1,3 +1 @@
-from .. import log, Seed, decorators
-from ..constant import Setting, DealModel
-from ..interface import SchedulerInterface, StorerInterface
+from .redis_db import RedisDB

cobweb/db/redis_db.py

@@ -1,211 +1,128 @@
-import time
 import redis
-from . import log, decorators, Seed, Setting, DealModel
-# from cobweb.decorators import decorators.check_redis_status
-# from cobweb.constant import Setting, DealModel
+import setting
 
 
 class RedisDB:
 
-    def __init__(
-            self,
-            project: str,
-            task_name: str,
-            config: dict,
-    ):
-        pool = redis.ConnectionPool(**config)
-        self.heartbeat_key = f"{project}:{task_name}:heartbeat"  # redis type string
-        self.spider_key = f"{project}:{task_name}:seed_info:spider"  # redis type zset, .format(priority)
-        self.storer_key = f"{project}:{task_name}:seed_info:storer:%s"  # redis type set,
-        self.failed_key = f"{project}:{task_name}:seed_info:failed"  # redis type set, .format(priority)
-        self.succeed_key = f"{project}:{task_name}:seed_info:succeed"  # redis type set, .format(priority)
-        self.update_lock = f"{project}:{task_name}:update_seed_lock"  # redis type string
-        self.check_lock = f"{project}:{task_name}:check_seed_lock"  # redis type string
-        self.scheduler_lock = f"{project}:{task_name}:scheduler_lock"  # redis type string
-        self.client = redis.Redis(connection_pool=pool)
-
-    @decorators.check_redis_status
-    def _get_lock(self, key, t=15, timeout=3, sleep_time=0.1):
-        begin_time = int(time.time())
-        while True:
-            if self.client.setnx(key, ""):
-                self.client.expire(key, t)
-                return True
-            if int(time.time()) - begin_time > timeout:
-                break
-            time.sleep(sleep_time)
-
-        if self.client.ttl(key) == -1:
-            delete_status = True
-            for _ in range(3):
-                if self.client.ttl(key) != -1:
-                    delete_status = False
-                    break
-                time.sleep(0.5)
-            if delete_status:
-                self.client.expire(key, t)
-            return False
-        else:
-            ttl = self.client.ttl(key)
-            log.info("ttl: " + str(ttl))
-            return False
-
-    @decorators.check_redis_status
-    def _deal_seed(self, seeds, is_add: bool):
-        if not seeds:
-            return None
-
-        if not isinstance(seeds, list):
-            seeds = [seeds]
-
-        item_info = dict()
-
-        for seed in seeds:
-            if not isinstance(seed, Seed):
-                seed = Seed(seed)
-            item_info[seed.format_seed] = seed._priority
-
-        if item_info:
-            self.client.zadd(self.spider_key, mapping=item_info, nx=is_add, xx=not is_add)
-
-    @decorators.check_redis_status
-    def add_seed(self, seeds):
-        self._deal_seed(seeds, is_add=True)
-
-    @decorators.check_redis_status
-    def reset_seed(self, seeds):
-        self._deal_seed(seeds, is_add=False)
-
-    @decorators.check_redis_status
-    def del_seed(self, seeds, spider_status: bool = True):
-        if not seeds:
-            return None
-
-        if not isinstance(seeds, list):
-            seeds = [seeds]
-
-        seeds = [seed if isinstance(seed, Seed) else Seed(seed) for seed in seeds]
-
-        if seeds:
-            redis_key = None
-            if spider_status and Setting.DEAL_MODEL in [DealModel.success, DealModel.polling]:
-                self.client.sadd(redis_key, *(seed.format_seed for seed in seeds))
-            elif not spider_status:
-                self.client.sadd(redis_key, *(str(seed) for seed in seeds))
-            self.client.zrem(self.spider_key, *(seed.format_seed for seed in seeds))
-
-    @decorators.check_redis_status
-    def set_storer(self, key, seeds):
-        if not seeds:
-            return None
-
-        if not isinstance(seeds, list):
-            seeds = [seeds]
-
-        item_info = dict()
-        score = -int(time.time())
-        for seed in seeds:
-            if not isinstance(seed, Seed):
-                seed = Seed(seed)
-            item_info[seed.format_seed] = score
-
-        if item_info:
-            self.client.zadd(self.storer_key % key, mapping=item_info)
-            log.info(f"zadd storer key: length {len(item_info.keys())}")
-
-    @decorators.check_redis_status
-    def get_seed(self, length: int = 200):
-        cs = time.time()
-
-        if self._get_lock(key=self.update_lock):
-
-            update_item, result = {}, []
-
-            version = int(time.time())
-
-            items = self.client.zrangebyscore(self.spider_key, min=0, max="+inf", start=0, num=length, withscores=True)
-
-            for value, priority in items:
-                score = -(version + int(priority) / 1000)
-                seed = Seed(value, priority=priority, version=version)
-                update_item[value] = score
-                result.append(seed)
-
-            log.info("set seeds into queue time: " + str(time.time() - cs))
-            if result:
-                self.client.zadd(self.spider_key, mapping=update_item, xx=True)
-
-            self.client.delete(self.update_lock)
-            log.info("push seeds into queue time: " + str(time.time() - cs))
-            return result
-
-    @decorators.check_redis_status
-    def check_spider_queue(self, stop, storer_num):
-        while not stop.is_set():
-            # 每15s获取check锁,等待600s后仍获取不到锁则重试;获取到锁后，设置锁的存活时间为${cs_lct}s
-            if self._get_lock(key=self.check_lock, t=Setting.CHECK_LOCK_TIME, timeout=600, sleep_time=3):
-                heartbeat = True if self.client.exists(self.heartbeat_key) else False
-                # 重启重制score值，否则获取${rs_time}分钟前的分数值
-                score = -int(time.time()) + Setting.RESET_SCORE if heartbeat else "-inf"
-
-                keys = self.client.keys(self.storer_key % "*")
-
-                if keys and len(keys) >= storer_num:
-                    intersection_key = self.storer_key % "intersection"
-                    self.client.delete(intersection_key)
-                    self.client.zinterstore(intersection_key, keys)
-
-                    while True:
-                        members = self.client.zrange(intersection_key, 0, 1999)
-                        if not members:
-                            break
-                        for key in keys:
-                            self.client.zrem(key, *members)
-                        if Setting.DEAL_MODEL in [DealModel.success, DealModel.polling]:
-                            self.client.sadd(self.succeed_key, *members)
-                        self.client.zrem(self.spider_key, *members)
-                        self.client.zrem(intersection_key, *members)
-                        log.info("succeed spider data ...")
-
-                for key in keys:
-                    self.client.zremrangebyscore(key, min=score, max="(0")
-
-                while True:
-                    items = self.client.zrangebyscore(self.spider_key, min=score, max="(0", start=0, num=5000, withscores=True)
-                    if not items:
-                        break
-                    reset_items = {}
-                    for value, priority in items:
-                        reset_score = "{:.3f}".format(priority).split(".")[1]
-                        reset_items[value] = int(reset_score)
-                    if reset_items:
-                        self.client.zadd(self.spider_key, mapping=reset_items, xx=True)
-
-                if not heartbeat:
-                    self.client.setex(self.heartbeat_key, 15, "")
-
-    @decorators.check_redis_status
-    def set_heartbeat(self, stop):
-        time.sleep(5)
-        while not stop.is_set():
-            self.client.setex(self.heartbeat_key, 5, "")
-            time.sleep(3)
-
-    # @decorators.check_redis_status
-    # def heartbeat(self):
-    #     """
-    #     返回心跳key剩余存活时间
-    #     """
-    #     return self.client.ttl(self.heartbeat_key)
-
-    @decorators.check_redis_status
-    def spider_queue_length(self):
-        return self.client.zcard(self.spider_key)
-
-    @decorators.check_redis_status
-    def ready_seed_length(self):
-        return self.client.zcount(self.spider_key, min=0, max="+inf")
-
-    @decorators.check_redis_status
-    def get_scheduler_lock(self):
-        return self._get_lock(self.scheduler_lock)
+    def __init__(self):
+        pool = redis.ConnectionPool(**setting.REDIS_CONFIG)
+        self._client = redis.Redis(connection_pool=pool)
+
+    def setnx(self, name, value=""):
+        return self._client.setnx(name, value)
+
+    def setex(self, name, t, value=""):
+        return self._client.setex(name, t, value)
+
+    def expire(self, name, t, nx: bool = False, xx: bool = False, gt: bool = False, lt: bool = False):
+        return self._client.expire(name, t, nx, xx, gt, lt)
+
+    def ttl(self, name):
+        return self._client.ttl(name)
+
+    def delete(self, name):
+        return self._client.delete(name)
+
+    def exists(self, *name) -> bool:
+        return self._client.exists(*name)
+
+    def sadd(self, name, value):
+        return self._client.sadd(name, value)
+
+    def zcard(self, name) -> bool:
+        return self._client.zcard(name)
+
+    def zadd(self, name, item: dict, **kwargs):
+        return self._client.zadd(name, item, **kwargs)
+
+    def zrem(self, name, *value):
+        return self._client.zrem(name, *value)
+
+    def zcount(self, name, _min, _max):
+        return self._client.zcount(name, _min, _max)
+
+    # def zrangebyscore(self, name, _min, _max, start, num, withscores: bool = False, *args):
+    #     return self._client.zrangebyscore(name, _min, _max, start, num, withscores, *args)
+
+    def lua(self, script: str, keys: list = None, args: list = None):
+        keys = keys or []
+        args = args or []
+        keys_count = len(keys)
+        return self._client.eval(script, keys_count, *keys, *args)
+
+    def lua_sha(self, sha1: str, keys: list = None, args: list = None):
+        keys = keys or []
+        args = args or []
+        keys_count = len(keys)
+        return self._client.evalsha(sha1, keys_count, *keys, *args)
+
+    def execute_lua(self, lua_script: str, keys: list, *args):
+        execute = self._client.register_script(lua_script)
+        return execute(keys=keys, args=args)
+
+    def lock(self, key, t=15) -> bool:
+        lua_script = """
+        local status = redis.call('setnx', KEYS[1], ARGV[1])
+        if ( status == 1 ) then
+            redis.call('expire', KEYS[1], ARGV[1])
+        end 
+        return status 
+        """
+        status = self.execute_lua(lua_script, [key], t)
+        return bool(status)
+
+    def members(self, key, score, start=0, count=5000, _min="-inf", _max="+inf") -> list:
+        lua_script = """
+        local min = ARGV[1]
+        local max = ARGV[2]
+        local start = ARGV[3]
+        local count = ARGV[4]
+        local score = ARGV[5]
+        local members = nil
+
+        if ( type(count) == string ) then
+            members = redis.call('zrangebyscore', KEYS[1], min, max, 'WITHSCORES')
+        else
+            members = redis.call('zrangebyscore', KEYS[1], min, max, 'WITHSCORES', 'limit', start, count)
+        end
+        
+        local result = {}
+
+        for i = 1, #members, 2 do
+            local priority = nil
+            local member = members[i]
+            local originPriority = nil
+            if ( members[i+1] + 0 < 0 ) then
+                originPriority = math.ceil(members[i+1]) * 1000 - members[i+1] * 1000
+            else
+                originPriority = math.floor(members[i+1])
+            end
+            
+            if ( score + 0 >= 1000 ) then
+                priority = -score - originPriority / 1000
+            elseif ( score + 0 == 0 ) then
+                priority = originPriority
+            else
+                originPriority = score 
+                priority = score
+            end
+            redis.call('zadd', KEYS[1], priority, member)
+            table.insert(result, member)
+            table.insert(result, originPriority)
+        end
+
+        return result
+        """
+        members = self.execute_lua(lua_script, [key], _min, _max, start, count, score)
+        return [(members[i].decode(), int(members[i + 1])) for i in range(0, len(members), 2)]
+
+    def done(self, keys: list, *args) -> list:
+        lua_script = """
+        for i, member in ipairs(ARGV) do
+            redis.call("zrem", KEYS[1], member)
+            redis.call("sadd", KEYS[2], member)
+        end
+        """
+        self.execute_lua(lua_script, keys, *args)
+
+

cobweb/exceptions/__init__.py

@@ -0,0 +1 @@
+from .oss_db_exception import *

cobweb/exceptions/oss_db_exception.py

@@ -0,0 +1,28 @@
+class OssDBException(Exception):
+    """Base oss client exception that all others inherit."""
+
+
+class OssDBMergeError(OssDBException):
+    """
+    Exception raised when execute merge operation fails.
+    """
+
+
+class OssDBPutPartError(OssDBException):
+    """
+    Exception raised when upload part operation fails.
+    """
+
+
+class OssDBPutObjError(OssDBException):
+    """
+    Exception raised when upload operation fails.
+    """
+
+
+class OssDBAppendObjError(OssDBException):
+    """Exception raised when upload operation fails."""
+
+
+class OssDBInitPartError(OssDBException):
+    """Exception raised when init upload operation fails."""

cobweb/launchers/__init__.py

@@ -0,0 +1,2 @@
+from .launcher import Launcher
+from .launcher_pro import LauncherPro