cobweb-launcher 1.0.5__py3-none-any.whl → 3.2.18__py3-none-any.whl

cobweb/crawlers/file_crawler.py

@@ -1,181 +0,0 @@
-
-from typing import Union
-from cobweb import setting
-from cobweb.utils import OssUtil
-from cobweb.crawlers import Crawler
-from cobweb.base import Seed, BaseItem, Request, Response
-from cobweb.exceptions import OssDBPutPartError, OssDBMergeError
-
-
-class CrawlerAir(Crawler):
-    
-    oss_util = OssUtil()
-
-    @staticmethod
-    def download(item: Request) -> Union[Seed, BaseItem, Response, str]:
-        seed_dict = item.seed.to_dict
-        bucket_name = CrawlerAir.oss_util.bucket
-        try:
-            key = item.seed.oss_path
-            if CrawlerAir.oss_util.exists(key):
-                content_length = CrawlerAir.oss_util.head(key).content_length
-                yield Response(item.seed, "exists", bucket_name=bucket_name, data_size=content_length, **seed_dict)
-                # data, cols = download_meta(item.seed, bucket_name=bucket_name, data_size=content_length, **seed_dict)
-                # yield DownloadItem(item.seed, sid=item.seed.sid, cols=cols, data=data)
-
-            end = seed_dict.get("end", "")
-            start = seed_dict.get("start", "0")
-
-            if end or int(start):
-                item.request_setting["headers"]['Range'] = f'bytes={start}-{end}'
-
-            if not item.seed.params.identifier:
-                content = b""
-                chunk_size = CrawlerAir.oss_util.chunk_size
-                min_upload_size = CrawlerAir.oss_util.min_upload_size
-                position = seed_dict.get("position", 1)
-
-                response = item.download()
-
-                content_length = response.headers.get("content-length") or 0
-                content_type = response.headers.get("content-type", "").split(";")[0]
-                if content_type and content_type in setting.FILE_FILTER_CONTENT_TYPE:
-                    yield Response(
-                        item.seed, response, filter=True, msg=f"response content type is {content_type}",
-                        bucket_name=bucket_name, data_size=content_length, **seed_dict
-                    )
-                elif position == 1 and min_upload_size >= int(content_length) > 0:
-                    """过小文件标识返回"""
-                    yield Response(
-                        item.seed, response, filter=True, msg="file size is too small",
-                        bucket_name=bucket_name, data_size=content_length, **seed_dict
-                    )
-                elif position == 1 and chunk_size > int(content_length) > min_upload_size:
-                    """小文件直接下载"""
-                    for part_data in response.iter_content(chunk_size):
-                        content += part_data
-                    CrawlerAir.oss_util.put(key, content)
-                    yield Response(item.seed, response, bucket_name=bucket_name, data_size=content_length, **seed_dict)
-                    response.close()
-                else:
-                    """中大文件同步分片下载"""
-                    upload_content_length = 0
-                    if not seed_dict.get("upload_id"):
-                        seed_dict["upload_id"] = CrawlerAir.oss_util.init_part(key).upload_id
-                    upload_id = seed_dict["upload_id"]
-                    for part_data in response.iter_content(chunk_size):
-                        content += part_data
-                        if len(content) >= chunk_size:
-                            upload_data = content[:chunk_size]
-                            content = content[chunk_size:]
-                            CrawlerAir.oss_util.put_part(key, upload_id, position, upload_data)
-                            upload_content_length += len(upload_data)
-                            position += 1
-                            seed_dict['position'] = position
-                            seed_dict['start'] = upload_content_length
-
-                    response.close()
-                    if content:
-                        CrawlerAir.oss_util.put_part(key, upload_id, position, content)
-                        content_length += len(content)
-                    CrawlerAir.oss_util.merge(key, upload_id)
-                    yield Response(item.seed, response, bucket_name=bucket_name, data_size=content_length, **seed_dict)
-                    # data, cols = download_meta(item.seed, bucket_name, data_size=content_length, **seed_dict)
-                    # yield DownloadItem(item.seed, sid=item.seed.sid, cols=cols, data=data)
-
-            elif item.seed.params.identifier == "merge":
-                CrawlerAir.oss_util.merge(key, seed_dict["upload_id"])
-                content_length = CrawlerAir.oss_util.head(key).content_length
-                yield Response(item.seed, "merge", bucket_name=bucket_name, data_size=content_length, **seed_dict)
-                # data, cols = download_meta(item.seed, bucket_name, data_size=content_length, **seed_dict)
-                # yield DownloadItem(item.seed, sid=item.seed.sid, cols=cols, data=data)
-        except OssDBPutPartError:
-            yield Seed(seed_dict)
-        except OssDBMergeError:
-            yield Seed(seed_dict, identifier="merge")
-
-
-class CrawlerPro(Crawler):
-
-    oss_util = OssUtil()
-
-    @staticmethod
-    def download(item: Request) -> Union[Seed, BaseItem, Response, str]:
-        seed_dict = item.seed.to_dict
-        bucket_name = CrawlerAir.oss_util.bucket
-        try:
-            key = item.seed.oss_path
-            if CrawlerAir.oss_util.exists(key):
-                content_length = CrawlerAir.oss_util.head(key).content_length
-                yield Response(item.seed, "exists", bucket_name=bucket_name, data_size=content_length, **seed_dict)
-
-            end = seed_dict.get("end", "")
-            start = seed_dict.get("start", "0")
-
-            if end or int(start):
-                item.request_setting["headers"]['Range'] = f'bytes={start}-{end}'
-
-            if not item.seed.params.identifier:
-                content = b""
-                chunk_size = CrawlerAir.oss_util.chunk_size
-                min_upload_size = CrawlerAir.oss_util.min_upload_size
-                position = seed_dict.get("position", 1)
-
-                response = item.download()
-
-                content_length = response.headers.get("content-length") or 0
-                content_type = response.headers.get("content-type", "").split(";")[0]
-                if content_type and content_type in setting.FILE_FILTER_CONTENT_TYPE:
-                    yield Response(
-                        item.seed, response, filter=True, msg=f"response content type is {content_type}",
-                        bucket_name=bucket_name, data_size=content_length, **seed_dict
-                    )
-                elif position == 1 and min_upload_size >= int(content_length) > 0:
-                    """过小文件标识返回"""
-                    yield Response(
-                        item.seed, response, filter=True, msg="file size is too small",
-                        bucket_name=bucket_name, data_size=content_length, **seed_dict
-                    )
-                elif position == 1 and chunk_size > int(content_length) > min_upload_size:
-                    """小文件直接下载"""
-                    for part_data in response.iter_content(chunk_size):
-                        content += part_data
-                    CrawlerAir.oss_util.put(key, content)
-                    yield Response(item.seed, response, bucket_name=bucket_name, data_size=content_length, **seed_dict)
-                    response.close()
-                else:
-                    """中大文件同步分片下载"""
-                    upload_content_length = 0
-                    if not seed_dict.get("upload_id"):
-                        seed_dict["upload_id"] = CrawlerAir.oss_util.init_part(key).upload_id
-                    upload_id = seed_dict["upload_id"]
-                    for part_data in response.iter_content(chunk_size):
-                        content += part_data
-                        if len(content) >= chunk_size:
-                            upload_data = content[:chunk_size]
-                            content = content[chunk_size:]
-                            CrawlerAir.oss_util.put_part(key, upload_id, position, upload_data)
-                            upload_content_length += len(upload_data)
-                            position += 1
-                            seed_dict['position'] = position
-                            seed_dict['start'] = upload_content_length
-
-                    response.close()
-                    if content:
-                        CrawlerAir.oss_util.put_part(key, upload_id, position, content)
-                        content_length += len(content)
-                    CrawlerAir.oss_util.merge(key, upload_id)
-                    yield Response(item.seed, response, bucket_name=bucket_name, data_size=content_length, **seed_dict)
-                    # data, cols = download_meta(item.seed, bucket_name, data_size=content_length, **seed_dict)
-                    # yield DownloadItem(item.seed, sid=item.seed.sid, cols=cols, data=data)
-
-            elif item.seed.params.identifier == "merge":
-                CrawlerAir.oss_util.merge(key, seed_dict["upload_id"])
-                content_length = CrawlerAir.oss_util.head(key).content_length
-                yield Response(item.seed, "merge", bucket_name=bucket_name, data_size=content_length, **seed_dict)
-                # data, cols = download_meta(item.seed, bucket_name, data_size=content_length, **seed_dict)
-                # yield DownloadItem(item.seed, sid=item.seed.sid, cols=cols, data=data)
-        except OssDBPutPartError:
-            yield Seed(seed_dict)
-        except OssDBMergeError:
-            yield Seed(seed_dict, identifier="merge")

cobweb/launchers/launcher_pro.py

@@ -1,174 +0,0 @@
-import time
-import threading
-
-from cobweb.db import RedisDB
-from cobweb.base import Seed, logger
-from cobweb.launchers import Launcher
-from cobweb.constant import DealModel, LogTemplate
-
-
-class LauncherPro(Launcher):
-
-    def __init__(self, task, project, custom_setting=None):
-        super().__init__(task, project, custom_setting)
-        self._todo = "{%s:%s}:todo" % (project, task)
-        self._done = "{%s:%s}:done" % (project, task)
-        self._fail = "{%s:%s}:fail" % (project, task)
-        self._heartbeat = "heartbeat:%s_%s" % (project, task)
-        self._reset_lock = "lock:reset:%s_%s" % (project, task)
-        self._heartbeat_lock = "lock:heartbeat:%s_%s" % (project, task)
-        self._db = RedisDB()
-
-        self._heartbeat_start_event = threading.Event()
-        self._redis_queue_empty_event = threading.Event()
-
-    @property
-    def heartbeat(self):
-        return self._db.exists(self._heartbeat)
-
-    def _execute_heartbeat(self):
-        while not self._stop.is_set():
-            if self._heartbeat_start_event.is_set():
-                if self._db.lock(self._heartbeat_lock, t=1):
-                    self._db.setex(self._heartbeat, 3)
-                time.sleep(1)
-            time.sleep(0.5)
-
-    def _reset(self):
-        """
-        检查过期种子，重新添加到redis缓存中
-        """
-        first = True
-        while not self._pause.is_set():
-            reset_wait_seconds = 15
-            if self._db.lock(self._reset_lock, t=120):
-                if not self.heartbeat:
-                    self._heartbeat_start_event.set()
-
-                _min = -int(time.time()) + self._seed_reset_seconds \
-                    if self.heartbeat or not first else "-inf"
-
-                self._db.members(
-                    self._todo, 0,
-                    _min=_min, _max="(0"
-                )
-                self._db.delete(self._reset_lock)
-                reset_wait_seconds = 60
-
-            time.sleep(reset_wait_seconds)
-            first = False
-
-    def _scheduler(self):
-        """
-        调度任务，获取redis队列种子，同时添加到doing字典中
-        """
-        if self.start_seeds:
-            self.__LAUNCHER_QUEUE__['todo'].push(self.start_seeds)
-        while not self._pause.is_set():
-            if not self._db.zcount(self._todo, 0, "(1000"):
-                time.sleep(self._scheduler_wait_seconds)
-                continue
-            if self.__LAUNCHER_QUEUE__['todo'].length >= self._todo_queue_size:
-                time.sleep(self._todo_queue_full_wait_seconds)
-                continue
-            members = self._db.members(
-                self._todo, int(time.time()),
-                count=self._todo_queue_size,
-                _min=0, _max="(1000"
-            )
-            for member, priority in members:
-                seed = Seed(member, priority=priority)
-                self.__LAUNCHER_QUEUE__['todo'].push(seed)
-                self.__DOING__[seed.to_string] = seed.params.priority
-
-    def _insert(self):
-        """
-        添加新种子到redis队列中
-        """
-        while not self._pause.is_set():
-            seeds = {}
-            status = self.__LAUNCHER_QUEUE__['new'].length < self._new_queue_max_size
-            for _ in range(self._new_queue_max_size):
-                seed = self.__LAUNCHER_QUEUE__['new'].pop()
-                if not seed:
-                    break
-                seeds[seed.to_string] = seed.params.priority
-            if seeds:
-                self._db.zadd(self._todo, seeds, nx=True)
-            if status:
-                time.sleep(self._new_queue_wait_seconds)
-
-    def _refresh(self):
-        """
-        刷新doing种子过期时间，防止reset重新消费
-        """
-        while not self._pause.is_set():
-            if self.__DOING__:
-                refresh_time = int(time.time())
-                seeds = {k:-refresh_time - v / 1000 for k, v in self.__DOING__.items()}
-                self._db.zadd(self._todo, item=seeds, xx=True)
-            time.sleep(30)
-
-    def _delete(self):
-        """
-        删除队列种子，根据状态添加至成功或失败队列，移除doing字典种子索引
-        """
-        while not self._pause.is_set():
-            seeds, s_seeds, f_seeds = [], [], []
-            status = self.__LAUNCHER_QUEUE__['done'].length < self._done_queue_max_size
-
-            for _ in range(self._done_queue_max_size):
-                seed = self.__LAUNCHER_QUEUE__['done'].pop()
-                if not seed:
-                    break
-                if seed.params.identifier == DealModel.fail:
-                    f_seeds.append(seed.to_string)
-                elif self._done_model == 1:
-                    s_seeds.append(seed.to_string)
-                else:
-                    seeds.append(seed.to_string)
-            if seeds:
-                self._db.zrem(self._todo, *seeds)
-            if s_seeds:
-                self._db.done([self._todo, self._done], *s_seeds)
-            if f_seeds:
-                self._db.done([self._todo, self._fail], *f_seeds)
-
-            self._remove_doing_seeds(seeds)
-
-            if status:
-                time.sleep(self._done_queue_wait_seconds)
-
-    def _polling(self):
-        check_emtpy_times = 0
-        while not self._stop.is_set():
-            queue_not_empty_count = 0
-            pooling_wait_seconds = 30
-            if not self._db.zcard(self._todo):
-                for q in self.__LAUNCHER_QUEUE__.values():
-                    if q.length != 0:
-                        queue_not_empty_count += 1
-                if self._pause.is_set() and queue_not_empty_count != 0:
-                    self._pause.clear()
-                    self._execute()
-                elif queue_not_empty_count == 0:
-                    pooling_wait_seconds = 3
-                    check_emtpy_times += 1
-                else:
-                    check_emtpy_times = 0
-                if check_emtpy_times > 2:
-                    check_emtpy_times = 0
-                    self.__DOING__ = {}
-                    self._pause.set()
-            if not self._pause.is_set():
-                logger.info(LogTemplate.launcher_pro_polling.format(
-                    task=self.task,
-                    doing_len=len(self.__DOING__.keys()),
-                    todo_len=self.__LAUNCHER_QUEUE__['todo'].length,
-                    done_len=self.__LAUNCHER_QUEUE__['done'].length,
-                    redis_seed_count=self._db.zcount(self._todo, "-inf", "+inf"),
-                    redis_todo_len=self._db.zcount(self._todo, 0, "(1000"),
-                    redis_doing_len=self._db.zcount(self._todo, "-inf", "(0"),
-                    upload_len=self._upload_queue.length
-                ))
-            time.sleep(pooling_wait_seconds)

cobweb/pipelines/base_pipeline.py

@@ -1,54 +0,0 @@
-import time
-import threading
-
-from abc import ABC, abstractmethod
-from cobweb.base import BaseItem, Queue, logger
-
-
-class Pipeline(threading.Thread, ABC):
-
-    def __init__(
-            self,
-            done_queue: Queue,
-            upload_queue: Queue,
-            upload_queue_size: int,
-            upload_wait_seconds: int
-    ):
-        super().__init__()
-        self.done_queue = done_queue
-        self.upload_queue = upload_queue
-        self.upload_queue_size = upload_queue_size
-        self.upload_wait_seconds = upload_wait_seconds
-
-    @abstractmethod
-    def build(self, item: BaseItem) -> dict:
-        pass
-
-    @abstractmethod
-    def upload(self, table: str, data: list) -> bool:
-        pass
-
-    def run(self):
-        while True:
-            status = self.upload_queue.length < self.upload_queue_size
-            if status:
-                time.sleep(self.upload_wait_seconds)
-            data_info, seeds = {}, []
-            for _ in range(self.upload_queue_size):
-                item = self.upload_queue.pop()
-                if not item:
-                    break
-                data = self.build(item)
-                seeds.append(item.seed)
-                data_info.setdefault(item.table, []).append(data)
-            for table, datas in data_info.items():
-                try:
-                    self.upload(table, datas)
-                    status = True
-                except Exception as e:
-                    logger.info(e)
-                    status = False
-                if status:
-                    self.done_queue.push(seeds)
-
-

cobweb/pipelines/loghub_pipeline.py

@@ -1,34 +0,0 @@
-import json
-
-from cobweb import setting
-from cobweb.base import BaseItem
-from cobweb.pipelines import Pipeline
-from aliyun.log import LogClient, LogItem, PutLogsRequest
-
-
-class LoghubPipeline(Pipeline):
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.client = LogClient(**setting.LOGHUB_CONFIG)
-
-    def build(self, item: BaseItem):
-        log_item = LogItem()
-        temp = item.to_dict
-        for key, value in temp.items():
-            if not isinstance(value, str):
-                temp[key] = json.dumps(value, ensure_ascii=False)
-        contents = sorted(temp.items())
-        log_item.set_contents(contents)
-        return log_item
-
-    def upload(self, table, datas):
-        request = PutLogsRequest(
-            project=setting.LOGHUB_PROJECT,
-            logstore=table,
-            topic=setting.LOGHUB_TOPIC,
-            source=setting.LOGHUB_SOURCE,
-            logitems=datas,
-            compress=True
-        )
-        self.client.put_logs(request=request)

cobweb_launcher-1.0.5.dist-info/METADATA

@@ -1,48 +0,0 @@
-Metadata-Version: 2.1
-Name: cobweb-launcher
-Version: 1.0.5
-Summary: spider_hole
-Home-page: https://github.com/Juannie-PP/cobweb
-Author: Juannie-PP
-Author-email: 2604868278@qq.com
-License: MIT
-Keywords: cobweb-launcher, cobweb
-Platform: UNKNOWN
-Classifier: Programming Language :: Python :: 3
-Requires-Python: >=3.7
-Description-Content-Type: text/markdown
-License-File: LICENSE
-Requires-Dist: requests (>=2.19.1)
-Requires-Dist: oss2 (>=2.18.1)
-Requires-Dist: redis (>=4.4.4)
-Requires-Dist: aliyun-log-python-sdk
-
-# cobweb
-
-> 通用爬虫框架： 1.单机模式采集框架；2.分布式采集框架
-> 
->  5部分
->  
-> 1. starter -- 启动器
-> 
-> 2. scheduler -- 调度器
-> 
-> 3. distributor -- 分发器
-> 
-> 4. storer -- 存储器
-> 
-> 5. utils -- 工具函数
-> 
-
-need deal
-- 队列优化完善，使用queue的机制wait()同步各模块执行？
-- 日志功能完善，单机模式调度和保存数据写入文件，结构化输出各任务日志
-- 去重过滤（布隆过滤器等）
-- 防丢失（单机模式可以通过日志文件进行检查种子）
-- 自定义数据库的功能
-- excel、mysql、redis数据完善
-
-
-![img.png](https://image-luyuan.oss-cn-hangzhou.aliyuncs.com/image/D2388CDC-B9E5-4CE4-9F2C-7D173763B6A8.png)
-
-

cobweb_launcher-1.0.5.dist-info/RECORD

@@ -1,32 +0,0 @@
-cobweb/__init__.py,sha256=IkGcdTU6fEBNyzWowcJSSMdErntFM1kmu4WUp1BgImU,45
-cobweb/constant.py,sha256=Aw2ES_nohVRLTWylZp6WMiUAlgyw4kLbae7LpwdZ5y4,1867
-cobweb/setting.py,sha256=T693DAwLFLs9P6ZEvugP99UzXn-8PLeMEgdxRmL6cb4,1955
-cobweb/base/__init__.py,sha256=diiK5MygQaWjlWNLbW6eUIg-93O6glMGC9WLNM5jyOc,209
-cobweb/base/common_queue.py,sha256=W7PPZZFl52j3Mc916T0imHj7oAUelA6aKJwW-FecDPE,872
-cobweb/base/decorators.py,sha256=wDCaQ94aAZGxks9Ljc0aXq6omDXT1_yzFy83ZW6VbVI,930
-cobweb/base/item.py,sha256=pMriHStzUXtSvIf5Z3KXsP-bCvjlG1gM3z33wWeuoH8,966
-cobweb/base/log.py,sha256=L01hXdk3L2qEm9X1FOXQ9VmWIoHSELe0cyZvrdAN61A,2003
-cobweb/base/request.py,sha256=b08AtUSZjlFLEFIEw5uGS__FjU6QSldL20-UjZD0LbI,2128
-cobweb/base/response.py,sha256=7h9TwCNqRlwM_fvNmid9zOoRfHbKB8ABSU0eaVUJdVo,405
-cobweb/base/seed.py,sha256=XswH16eEd6iwIBpt71E2S_AsV5UVCcOEOBFoP0r5QRo,2900
-cobweb/crawlers/__init__.py,sha256=1sMhQ0-NJxiff3IqF2aMCXkSXcJFzzoCKIayQ5go4aI,71
-cobweb/crawlers/base_crawler.py,sha256=-3WQSMrMROensTrTulh-eLK61qBsZDPoK2AM-sQwUHA,4893
-cobweb/crawlers/file_crawler.py,sha256=lOLsWxDKUQy0uJZ1NlXIPFiOJ17ufcveJIJD0Wqsqy8,9696
-cobweb/db/__init__.py,sha256=ut0iEyBLjcJL06WNG_5_d4hO5PJWvDrKWMkDOdmgh2M,30
-cobweb/db/redis_db.py,sha256=XE1ebIi_4e0KBKiyPdKX7l2lSgV5vhMwEhNDlAcsccU,4247
-cobweb/exceptions/__init__.py,sha256=E9SHnJBbhD7fOgPFMswqyOf8SKRDrI_i25L0bSpohvk,32
-cobweb/exceptions/oss_db_exception.py,sha256=iP_AImjNHT3-Iv49zCFQ3rdLnlvuHa3h2BXApgrOYpA,636
-cobweb/launchers/__init__.py,sha256=qwlkEJVri7dvCgi45aX3lqAmQS0HrPicAipDvH75kew,69
-cobweb/launchers/launcher.py,sha256=O6Kkvqk-0kOxJom8YO9zW18e_2eMYrA5RTS9Xy4TW5k,5665
-cobweb/launchers/launcher_pro.py,sha256=GlDpyP1XAY2bX5SuSBn3920D5OKNigQgLnJfu6QOmPw,6760
-cobweb/pipelines/__init__.py,sha256=xanY-Z1d7zRR5JhCdW2htzrAywnKBkigiaUlTFa6of0,80
-cobweb/pipelines/base_pipeline.py,sha256=fYnWf79GmhufXpcnMa3te18SbmnVeYLwxfyo-zLd9CY,1577
-cobweb/pipelines/loghub_pipeline.py,sha256=cjPO6w6UJ0jNw2fVvdX0BCdlm58T7dmYXlxzXOBpvfY,1027
-cobweb/utils/__init__.py,sha256=JTE4sBfHnKHhD6w9Auk0MIT7O9BMOamCeryhlHNx3Zg,47
-cobweb/utils/oss.py,sha256=uD5aN2oVYImit3amE6TjxWMaTAcbAh9dCnpIQhf4M9Q,3238
-cobweb/utils/tools.py,sha256=8oLxkzwaYcDqKXXuLI3A_lNabyLBr7HSPgTF6x4xbnk,1239
-cobweb_launcher-1.0.5.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
-cobweb_launcher-1.0.5.dist-info/METADATA,sha256=qZCm5HEq0VbbPTtZbWoRh6a_Jx7JsUtk_VIXU_-zQAA,1245
-cobweb_launcher-1.0.5.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
-cobweb_launcher-1.0.5.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
-cobweb_launcher-1.0.5.dist-info/RECORD,,